gallant/SDL2
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Fixed bug libsdl-org#6745 - Check for overflow in SDL_CalculateYUVSize (#6747)

Browse source 
Fixed bug #6745 - Check for overflow in SDL_CalculateYUVSize
This commit is contained in:
Sylvain Becker 2022-12-03 21:22:14 +01:00 committed by GitHub
parent 5650046f93
commit 4ffa7f868d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 137 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
src/render/SDL_yuv_sw.c
View file

@ -58,7 +58,10 @@ SDL_SW_CreateYUVTexture(Uint32 format, int w, int h)
swdata->h = h;
{
size_t dst_size;
SDL_CalculateYUVSize(format, w, h, &dst_size, NULL);
if (SDL_CalculateYUVSize(format, w, h, &dst_size, NULL) < 0) {
SDL_OutOfMemory();
return NULL;
}
swdata->pixels = (Uint8 *)SDL_SIMDAlloc(dst_size);
if (!swdata->pixels) {
SDL_SW_DestroyYUVTexture(swdata);

54
src/video/SDL_surface.c
View file

@ -39,13 +39,15 @@ SDL_COMPILE_TIME_ASSERT(can_indicate_overflow, SDL_SIZE_MAX > SDL_MAX_SINT32);
/*
* Calculate the pad-aligned scanline width of a surface.
* Return SDL_SIZE_MAX on overflow.
*
* for FOURCC, use SDL_CalculateYUVSize()
*/
static size_t
SDL_CalculatePitch(Uint32 format, size_t width, SDL_bool minimal)
{
size_t pitch;
if (SDL_ISPIXELFORMAT_FOURCC(format) || SDL_BITSPERPIXEL(format) >= 8) {
if (SDL_BITSPERPIXEL(format) >= 8) {
if (SDL_size_mul_overflow(width, SDL_BYTESPERPIXEL(format), &pitch)) {
return SDL_SIZE_MAX;
}
@ -88,11 +90,21 @@ SDL_CreateSurface(int width, int height, Uint32 format)
return NULL;
}
pitch = SDL_CalculatePitch(format, width, SDL_FALSE);
if (pitch > SDL_MAX_SINT32) {
/* Overflow... */
SDL_OutOfMemory();
return NULL;
if (SDL_ISPIXELFORMAT_FOURCC(format)) {
int p;
if (SDL_CalculateYUVSize(format, width, height, NULL, &p) < 0) {
/* Overflow... */
SDL_OutOfMemory();
return NULL;
}
pitch = p;
} else {
pitch = SDL_CalculatePitch(format, width, SDL_FALSE);
if (pitch > SDL_MAX_SINT32) {
/* Overflow... */
SDL_OutOfMemory();
return NULL;
}
}
/* Allocate the surface */
@ -134,20 +146,21 @@ SDL_CreateSurface(int width, int height, Uint32 format)
/* Get the pixels */
if (surface->w && surface->h) {
/* Assumptions checked in surface_size_assumptions assert above */
size_t size;
if (SDL_size_mul_overflow(surface->h, surface->pitch, &size)) {
/* Overflow... */
SDL_FreeSurface(surface);
SDL_OutOfMemory();
return NULL;
}
if (SDL_ISPIXELFORMAT_FOURCC(surface->format->format)) {
/* Get correct size and pitch for YUV formats */
if (SDL_CalculateYUVSize(surface->format->format, surface->w, surface->h, &size, &surface->pitch) < 0) {
SDL_OutOfMemory();
/* Overflow... */
SDL_FreeSurface(surface);
SDL_OutOfMemory();
return NULL;
}
} else {
/* Assumptions checked in surface_size_assumptions assert above */
if (SDL_size_mul_overflow(surface->h, surface->pitch, &size)) {
/* Overflow... */
SDL_FreeSurface(surface);
SDL_OutOfMemory();
return NULL;
}
}
@ -202,7 +215,16 @@ SDL_CreateSurfaceFrom(void *pixels,
return NULL;
}
minimalPitch = SDL_CalculatePitch(format, width, SDL_TRUE);
if (SDL_ISPIXELFORMAT_FOURCC(format)) {
int p;
if (SDL_CalculateYUVSize(format, width, height, NULL, &p) < 0) {
SDL_InvalidParamError("pitch");
return NULL;
}
minimalPitch = p;
} else {
minimalPitch = SDL_CalculatePitch(format, width, SDL_TRUE);
}
if (pitch < 0 || (pitch > 0 && ((size_t)pitch) < minimalPitch)) {
SDL_InvalidParamError("pitch");

107
src/video/SDL_yuv.c
View file

@ -29,6 +29,10 @@
static SDL_YUV_CONVERSION_MODE SDL_YUV_ConversionMode = SDL_YUV_CONVERSION_BT601;
#if SDL_HAVE_YUV
static SDL_bool IsPlanar2x2Format(Uint32 format);
#endif
void SDL_SetYUVConversionMode(SDL_YUV_CONVERSION_MODE mode)
{
SDL_YUV_ConversionMode = mode;
@ -53,51 +57,130 @@ SDL_YUV_CONVERSION_MODE SDL_GetYUVConversionModeForResolution(int width, int hei
}
/*
* Calculate YUV size.
* Calculate YUV size and pitch. Check for overflow.
* Output 'pitch' that can be used with SDL_ConvertPixels()
*
* return 0 on success, -1 on error
*/
int SDL_CalculateYUVSize(Uint32 format, int w, int h, size_t *size, int *pitch)
{
const int sz_plane = w * h;
const int sz_plane_chroma = ((w + 1) / 2) * ((h + 1) / 2);
const int sz_plane_packed = ((w + 1) / 2) * h;
int dst_size = 0;
#if SDL_HAVE_YUV
int sz_plane = 0, sz_plane_chroma = 0, sz_plane_packed = 0;
if (IsPlanar2x2Format(format) == SDL_TRUE) {
{
/* sz_plane == w * h; */
size_t s1;
if (SDL_size_mul_overflow(w, h, &s1) < 0) {
return -1;
}
sz_plane = (int) s1;
}
{
/* sz_plane_chroma == ((w + 1) / 2) * ((h + 1) / 2); */
size_t s1, s2, s3;
if (SDL_size_add_overflow(w, 1, &s1) < 0) {
return -1;
}
s1 = s1 / 2;
if (SDL_size_add_overflow(h, 1, &s2) < 0) {
return -1;
}
s2 = s2 / 2;
if (SDL_size_mul_overflow(s1, s2, &s3) < 0) {
return -1;
}
sz_plane_chroma = (int) s3;
}
} else {
/* sz_plane_packed == ((w + 1) / 2) * h; */
size_t s1, s2;
if (SDL_size_add_overflow(w, 1, &s1) < 0) {
return -1;
}
s1 = s1 / 2;
if (SDL_size_mul_overflow(s1, h, &s2) < 0) {
return -1;
}
sz_plane_packed = (int) s2;
}
switch (format) {
case SDL_PIXELFORMAT_YV12: /**< Planar mode: Y + V + U (3 planes) */
case SDL_PIXELFORMAT_IYUV: /**< Planar mode: Y + U + V (3 planes) */
dst_size = sz_plane + sz_plane_chroma + sz_plane_chroma;
if (pitch) {
*pitch = w;
}
if (size) {
/* dst_size == sz_plane + sz_plane_chroma + sz_plane_chroma; */
size_t s1, s2;
if (SDL_size_add_overflow(sz_plane, sz_plane_chroma, &s1) < 0) {
return -1;
}
if (SDL_size_add_overflow(s1, sz_plane_chroma, &s2) < 0) {
return -1;
}
*size = (int)s2;
}
break;
case SDL_PIXELFORMAT_YUY2: /**< Packed mode: Y0+U0+Y1+V0 (1 plane) */
case SDL_PIXELFORMAT_UYVY: /**< Packed mode: U0+Y0+V0+Y1 (1 plane) */
case SDL_PIXELFORMAT_YVYU: /**< Packed mode: Y0+V0+Y1+U0 (1 plane) */
dst_size = 4 * sz_plane_packed;
if (pitch) {
*pitch = ((w + 1) / 2) * 4;
/* pitch == ((w + 1) / 2) * 4; */
size_t p1, p2;
if (SDL_size_add_overflow(w, 1, &p1) < 0) {
return -1;
}
p1 = p1 / 2;
if (SDL_size_mul_overflow(p1, 4, &p2) < 0) {
return -1;
}
*pitch = (int) p2;
}
if (size) {
/* dst_size == 4 * sz_plane_packed; */
size_t s1;
if (SDL_size_mul_overflow(sz_plane_packed, 4, &s1) < 0) {
return -1;
}
*size = (int) s1;
}
break;
case SDL_PIXELFORMAT_NV12: /**< Planar mode: Y + U/V interleaved (2 planes) */
case SDL_PIXELFORMAT_NV21: /**< Planar mode: Y + V/U interleaved (2 planes) */
dst_size = sz_plane + sz_plane_chroma + sz_plane_chroma;
if (pitch) {
*pitch = w;
}
if (size) {
/* dst_size == sz_plane + sz_plane_chroma + sz_plane_chroma; */
size_t s1, s2;
if (SDL_size_add_overflow(sz_plane, sz_plane_chroma, &s1) < 0) {
return -1;
}
if (SDL_size_add_overflow(s1, sz_plane_chroma, &s2) < 0) {
return -1;
}
*size = (int) s2;
}
break;
default:
return -1;
}
if (size) {
*size = dst_size;
}
return 0;
#else
return -1;
#endif
}
#if SDL_HAVE_YUV