gallant/SDL2
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

SDL2: backport SDL_CalculateYUVSize() that checks for YUV size overflow (#6972)

Browse source
This commit is contained in:
Sylvain Becker 2023-01-02 18:09:57 +01:00 committed by GitHub
parent 2fd9e63f1a
commit d234f7a498
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 157 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
src/render/SDL_yuv_sw.c
View file

@ -57,30 +57,11 @@ SDL_SW_CreateYUVTexture(Uint32 format, int w, int h)
swdata->w = w;
swdata->h = h;
{
const int sz_plane = w * h;
const int sz_plane_chroma = ((w + 1) / 2) * ((h + 1) / 2);
const int sz_plane_packed = ((w + 1) / 2) * h;
int dst_size = 0;
switch (format) {
case SDL_PIXELFORMAT_YV12: /**< Planar mode: Y + V + U (3 planes) */
case SDL_PIXELFORMAT_IYUV: /**< Planar mode: Y + U + V (3 planes) */
dst_size = sz_plane + sz_plane_chroma + sz_plane_chroma;
break;
case SDL_PIXELFORMAT_YUY2: /**< Packed mode: Y0+U0+Y1+V0 (1 plane) */
case SDL_PIXELFORMAT_UYVY: /**< Packed mode: U0+Y0+V0+Y1 (1 plane) */
case SDL_PIXELFORMAT_YVYU: /**< Packed mode: Y0+V0+Y1+U0 (1 plane) */
dst_size = 4 * sz_plane_packed;
break;
case SDL_PIXELFORMAT_NV12: /**< Planar mode: Y + U/V interleaved (2 planes) */
case SDL_PIXELFORMAT_NV21: /**< Planar mode: Y + V/U interleaved (2 planes) */
dst_size = sz_plane + sz_plane_chroma + sz_plane_chroma;
break;
default:
SDL_assert(0 && "We should never get here (caught above)");
break;
size_t dst_size;
if (SDL_CalculateYUVSize(format, w, h, &dst_size, NULL) < 0) {
SDL_SW_DestroyYUVTexture(swdata);
SDL_OutOfMemory();
return NULL;
}
swdata->pixels = (Uint8 *)SDL_SIMDAlloc(dst_size);
if (!swdata->pixels) {

24
src/video/SDL_surface.c
View file

@ -39,13 +39,15 @@ SDL_COMPILE_TIME_ASSERT(can_indicate_overflow, SDL_SIZE_MAX > SDL_MAX_SINT32);
/*
* Calculate the pad-aligned scanline width of a surface.
* Return SDL_SIZE_MAX on overflow.
*
* for FOURCC, use SDL_CalculateYUVSize()
*/
static size_t
SDL_CalculatePitch(Uint32 format, size_t width, SDL_bool minimal)
{
size_t pitch;
if (SDL_ISPIXELFORMAT_FOURCC(format) || SDL_BITSPERPIXEL(format) >= 8) {
if (SDL_BITSPERPIXEL(format) >= 8) {
if (SDL_size_mul_overflow(width, SDL_BYTESPERPIXEL(format), &pitch)) {
return SDL_SIZE_MAX;
}
@ -93,11 +95,16 @@ SDL_CreateRGBSurfaceWithFormat(Uint32 flags, int width, int height, int depth,
return NULL;
}
pitch = SDL_CalculatePitch(format, width, SDL_FALSE);
if (pitch > SDL_MAX_SINT32) {
/* Overflow... */
SDL_OutOfMemory();
if (SDL_ISPIXELFORMAT_FOURCC(format)) {
SDL_SetError("invalid format");
return NULL;
} else {
pitch = SDL_CalculatePitch(format, width, SDL_FALSE);
if (pitch > SDL_MAX_SINT32) {
/* Overflow... */
SDL_OutOfMemory();
return NULL;
}
}
/* Allocate the surface */
@ -269,7 +276,12 @@ SDL_CreateRGBSurfaceWithFormatFrom(void *pixels,
return NULL;
}
minimalPitch = SDL_CalculatePitch(format, width, SDL_TRUE);
if (SDL_ISPIXELFORMAT_FOURCC(format)) {
SDL_SetError("invalid format");
return NULL;
} else {
minimalPitch = SDL_CalculatePitch(format, width, SDL_TRUE);
}
if (pitch < 0 || (pitch > 0 && ((size_t)pitch) < minimalPitch)) {
SDL_InvalidParamError("pitch");

131
src/video/SDL_yuv.c
View file

@ -31,6 +31,10 @@
static SDL_YUV_CONVERSION_MODE SDL_YUV_ConversionMode = SDL_YUV_CONVERSION_BT601;
#if SDL_HAVE_YUV
static SDL_bool IsPlanar2x2Format(Uint32 format);
#endif
void SDL_SetYUVConversionMode(SDL_YUV_CONVERSION_MODE mode)
{
SDL_YUV_ConversionMode = mode;
@ -54,6 +58,133 @@ SDL_YUV_CONVERSION_MODE SDL_GetYUVConversionModeForResolution(int width, int hei
return mode;
}
/*
* Calculate YUV size and pitch. Check for overflow.
* Output 'pitch' that can be used with SDL_ConvertPixels()
*
* return 0 on success, -1 on error
*/
int SDL_CalculateYUVSize(Uint32 format, int w, int h, size_t *size, int *pitch)
{
#if SDL_HAVE_YUV
int sz_plane = 0, sz_plane_chroma = 0, sz_plane_packed = 0;
if (IsPlanar2x2Format(format) == SDL_TRUE) {
{
/* sz_plane == w * h; */
size_t s1;
if (SDL_size_mul_overflow(w, h, &s1) < 0) {
return -1;
}
sz_plane = (int) s1;
}
{
/* sz_plane_chroma == ((w + 1) / 2) * ((h + 1) / 2); */
size_t s1, s2, s3;
if (SDL_size_add_overflow(w, 1, &s1) < 0) {
return -1;
}
s1 = s1 / 2;
if (SDL_size_add_overflow(h, 1, &s2) < 0) {
return -1;
}
s2 = s2 / 2;
if (SDL_size_mul_overflow(s1, s2, &s3) < 0) {
return -1;
}
sz_plane_chroma = (int) s3;
}
} else {
/* sz_plane_packed == ((w + 1) / 2) * h; */
size_t s1, s2;
if (SDL_size_add_overflow(w, 1, &s1) < 0) {
return -1;
}
s1 = s1 / 2;
if (SDL_size_mul_overflow(s1, h, &s2) < 0) {
return -1;
}
sz_plane_packed = (int) s2;
}
switch (format) {
case SDL_PIXELFORMAT_YV12: /**< Planar mode: Y + V + U (3 planes) */
case SDL_PIXELFORMAT_IYUV: /**< Planar mode: Y + U + V (3 planes) */
if (pitch) {
*pitch = w;
}
if (size) {
/* dst_size == sz_plane + sz_plane_chroma + sz_plane_chroma; */
size_t s1, s2;
if (SDL_size_add_overflow(sz_plane, sz_plane_chroma, &s1) < 0) {
return -1;
}
if (SDL_size_add_overflow(s1, sz_plane_chroma, &s2) < 0) {
return -1;
}
*size = (int)s2;
}
break;
case SDL_PIXELFORMAT_YUY2: /**< Packed mode: Y0+U0+Y1+V0 (1 plane) */
case SDL_PIXELFORMAT_UYVY: /**< Packed mode: U0+Y0+V0+Y1 (1 plane) */
case SDL_PIXELFORMAT_YVYU: /**< Packed mode: Y0+V0+Y1+U0 (1 plane) */
if (pitch) {
/* pitch == ((w + 1) / 2) * 4; */
size_t p1, p2;
if (SDL_size_add_overflow(w, 1, &p1) < 0) {
return -1;
}
p1 = p1 / 2;
if (SDL_size_mul_overflow(p1, 4, &p2) < 0) {
return -1;
}
*pitch = (int) p2;
}
if (size) {
/* dst_size == 4 * sz_plane_packed; */
size_t s1;
if (SDL_size_mul_overflow(sz_plane_packed, 4, &s1) < 0) {
return -1;
}
*size = (int) s1;
}
break;
case SDL_PIXELFORMAT_NV12: /**< Planar mode: Y + U/V interleaved (2 planes) */
case SDL_PIXELFORMAT_NV21: /**< Planar mode: Y + V/U interleaved (2 planes) */
if (pitch) {
*pitch = w;
}
if (size) {
/* dst_size == sz_plane + sz_plane_chroma + sz_plane_chroma; */
size_t s1, s2;
if (SDL_size_add_overflow(sz_plane, sz_plane_chroma, &s1) < 0) {
return -1;
}
if (SDL_size_add_overflow(s1, sz_plane_chroma, &s2) < 0) {
return -1;
}
*size = (int) s2;
}
break;
default:
return -1;
}
return 0;
#else
return -1;
#endif
}
#if SDL_HAVE_YUV
static int GetYUVConversionType(int width, int height, YCbCrType *yuv_type)

3
src/video/SDL_yuv_c.h
View file

@ -30,6 +30,9 @@ extern int SDL_ConvertPixels_YUV_to_RGB(int width, int height, Uint32 src_format
extern int SDL_ConvertPixels_RGB_to_YUV(int width, int height, Uint32 src_format, const void *src, int src_pitch, Uint32 dst_format, void *dst, int dst_pitch);
extern int SDL_ConvertPixels_YUV_to_YUV(int width, int height, Uint32 src_format, const void *src, int src_pitch, Uint32 dst_format, void *dst, int dst_pitch);
extern int SDL_CalculateYUVSize(Uint32 format, int w, int h, size_t *size, int *pitch);
#endif /* SDL_yuv_c_h_ */
/* vi: set ts=4 sw=4 expandtab: */