From 60f7cba34a81fdd6ca6873a2f375db529d9b7463 Mon Sep 17 00:00:00 2001 From: "Dexter.k" <164054284+rootvector2@users.noreply.github.com> Date: Sun, 22 Feb 2026 09:54:47 +0000 Subject: [PATCH 1/5] Fix potential integer overflow in rowBytes multiplications Cast the first operand to (size_t) before multiplying two uint32_t values involving rowBytes, alphaRowBytes, or yuvRowBytes to prevent unsigned integer wrap-around on large images. --- src/codec_aom.c | 4 +- src/codec_avm.c | 4 +- src/codec_svt.c | 10 ++-- src/read.c | 8 ++-- src/reformat.c | 120 ++++++++++++++++++++++++------------------------ 5 files changed, 73 insertions(+), 73 deletions(-) diff --git a/src/codec_aom.c b/src/codec_aom.c index 212d822d81..6846ec4105 100644 --- a/src/codec_aom.c +++ b/src/codec_aom.c @@ -1209,7 +1209,7 @@ static avifResult aomCodecEncodeImage(avifCodec * codec, if (aomImageAllocated) { const uint32_t bytesPerRow = ((image->depth > 8) ? 2 : 1) * image->width; for (uint32_t j = 0; j < image->height; ++j) { - const uint8_t * srcAlphaRow = &image->alphaPlane[j * image->alphaRowBytes]; + const uint8_t * srcAlphaRow = &image->alphaPlane[(size_t)j * image->alphaRowBytes]; uint8_t * dstAlphaRow = &aomImage.planes[0][j * aomImage.stride[0]]; memcpy(dstAlphaRow, srcAlphaRow, bytesPerRow); } @@ -1233,7 +1233,7 @@ static avifResult aomCodecEncodeImage(avifCodec * codec, uint32_t bytesPerRow = bytesPerPixel * planeWidth; for (uint32_t j = 0; j < planeHeight; ++j) { - const uint8_t * srcRow = &image->yuvPlanes[yuvPlane][j * image->yuvRowBytes[yuvPlane]]; + const uint8_t * srcRow = &image->yuvPlanes[yuvPlane][(size_t)j * image->yuvRowBytes[yuvPlane]]; uint8_t * dstRow = &aomImage.planes[yuvPlane][j * aomImage.stride[yuvPlane]]; memcpy(dstRow, srcRow, bytesPerRow); } diff --git a/src/codec_avm.c b/src/codec_avm.c index e1627724ad..9e09ed0c28 100644 --- a/src/codec_avm.c +++ b/src/codec_avm.c @@ -903,7 +903,7 @@ static avifResult avmCodecEncodeImage(avifCodec * codec, if (avmImageAllocated) { const uint32_t bytesPerRow = ((image->depth > 8) ? 2 : 1) * image->width; for (uint32_t j = 0; j < image->height; ++j) { - const uint8_t * srcAlphaRow = &image->alphaPlane[j * image->alphaRowBytes]; + const uint8_t * srcAlphaRow = &image->alphaPlane[(size_t)j * image->alphaRowBytes]; uint8_t * dstAlphaRow = &avmImage.planes[0][j * avmImage.stride[0]]; memcpy(dstAlphaRow, srcAlphaRow, bytesPerRow); } @@ -927,7 +927,7 @@ static avifResult avmCodecEncodeImage(avifCodec * codec, uint32_t bytesPerRow = bytesPerPixel * planeWidth; for (uint32_t j = 0; j < planeHeight; ++j) { - const uint8_t * srcRow = &image->yuvPlanes[yuvPlane][j * image->yuvRowBytes[yuvPlane]]; + const uint8_t * srcRow = &image->yuvPlanes[yuvPlane][(size_t)j * image->yuvRowBytes[yuvPlane]]; uint8_t * dstRow = &avmImage.planes[yuvPlane][j * avmImage.stride[yuvPlane]]; memcpy(dstRow, srcRow, bytesPerRow); } diff --git a/src/codec_svt.c b/src/codec_svt.c index e893e46ceb..dccc388be2 100644 --- a/src/codec_svt.c +++ b/src/codec_svt.c @@ -274,13 +274,13 @@ static avifResult svtCodecEncodeImage(avifCodec * codec, if (alpha) { input_picture_buffer->y_stride = image->alphaRowBytes / bytesPerPixel; input_picture_buffer->luma = image->alphaPlane; - input_buffer->n_filled_len = image->alphaRowBytes * image->height; + input_buffer->n_filled_len = (size_t)image->alphaRowBytes * image->height; #if SVT_AV1_CHECK_VERSION(1, 8, 0) // Simulate 4:2:0 UV planes. SVT-AV1 does not support 4:0:0 samples. const uint32_t uvWidth = (image->width + y_shift) >> y_shift; const uint32_t uvRowBytes = uvWidth * bytesPerPixel; - const uint32_t uvSize = uvRowBytes * uvHeight; + const size_t uvSize = (size_t)uvRowBytes * uvHeight; uvPlanes = avifAlloc(uvSize); if (uvPlanes == NULL) { goto cleanup; @@ -300,11 +300,11 @@ static avifResult svtCodecEncodeImage(avifCodec * codec, } else { input_picture_buffer->y_stride = image->yuvRowBytes[0] / bytesPerPixel; input_picture_buffer->luma = image->yuvPlanes[0]; - input_buffer->n_filled_len = image->yuvRowBytes[0] * image->height; + input_buffer->n_filled_len = (size_t)image->yuvRowBytes[0] * image->height; input_picture_buffer->cb = image->yuvPlanes[1]; - input_buffer->n_filled_len += image->yuvRowBytes[1] * uvHeight; + input_buffer->n_filled_len += (size_t)image->yuvRowBytes[1] * uvHeight; input_picture_buffer->cr = image->yuvPlanes[2]; - input_buffer->n_filled_len += image->yuvRowBytes[2] * uvHeight; + input_buffer->n_filled_len += (size_t)image->yuvRowBytes[2] * uvHeight; input_picture_buffer->cb_stride = image->yuvRowBytes[1] / bytesPerPixel; input_picture_buffer->cr_stride = image->yuvRowBytes[2] / bytesPerPixel; } diff --git a/src/read.c b/src/read.c index a5512d0470..e65eb66372 100644 --- a/src/read.c +++ b/src/read.c @@ -6599,8 +6599,8 @@ static avifResult avifImageLimitedToFullAlpha(avifImage * image) if (image->depth > 8) { for (uint32_t j = 0; j < image->height; ++j) { - const uint8_t * srcRow = &alphaPlane[j * alphaRowBytes]; - uint8_t * dstRow = &image->alphaPlane[j * image->alphaRowBytes]; + const uint8_t * srcRow = &alphaPlane[(size_t)j * alphaRowBytes]; + uint8_t * dstRow = &image->alphaPlane[(size_t)j * image->alphaRowBytes]; for (uint32_t i = 0; i < image->width; ++i) { int srcAlpha = *((const uint16_t *)&srcRow[i * 2]); int dstAlpha = avifLimitedToFullY(image->depth, srcAlpha); @@ -6609,8 +6609,8 @@ static avifResult avifImageLimitedToFullAlpha(avifImage * image) } } else { for (uint32_t j = 0; j < image->height; ++j) { - const uint8_t * srcRow = &alphaPlane[j * alphaRowBytes]; - uint8_t * dstRow = &image->alphaPlane[j * image->alphaRowBytes]; + const uint8_t * srcRow = &alphaPlane[(size_t)j * alphaRowBytes]; + uint8_t * dstRow = &image->alphaPlane[(size_t)j * image->alphaRowBytes]; for (uint32_t i = 0; i < image->width; ++i) { int srcAlpha = srcRow[i]; int dstAlpha = avifLimitedToFullY(image->depth, srcAlpha); diff --git a/src/reformat.c b/src/reformat.c index 9d4260b3a5..e3c8932da4 100644 --- a/src/reformat.c +++ b/src/reformat.c @@ -469,10 +469,10 @@ avifResult avifImageRGBToYUV(avifImage * image, const avifRGBImage * rgb) const uint32_t grayPixelBytes = state.rgb.pixelBytes; const uint32_t offsetBytesGray = state.rgb.offsetBytesGray; const uint32_t offsetBytesA = state.rgb.offsetBytesA; - const uint32_t grayRowBytes = rgb->rowBytes; + const size_t grayRowBytes = rgb->rowBytes; const float grayMaxChannelF = state.rgb.maxChannelF; uint8_t * yPlane = image->yuvPlanes[AVIF_CHAN_Y]; - const uint32_t yRowBytes = image->yuvRowBytes[AVIF_CHAN_Y]; + const size_t yRowBytes = image->yuvRowBytes[AVIF_CHAN_Y]; for (uint32_t j = 0; j < image->height; ++j) { for (uint32_t i = 0; i < image->width; ++i) { float g; @@ -521,7 +521,7 @@ avifResult avifImageRGBToYUV(avifImage * image, const avifRGBImage * rgb) const int half = 1 << (image->depth - 1); if (image->yuvPlanes[AVIF_CHAN_U]) { uint8_t * uPlane = image->yuvPlanes[AVIF_CHAN_U]; - const uint32_t uRowBytes = image->yuvRowBytes[AVIF_CHAN_U]; + const size_t uRowBytes = image->yuvRowBytes[AVIF_CHAN_U]; if (state.yuv.channelBytes > 1) { avifMemset16(uPlane, half, shiftedH * uRowBytes / 2); } else { @@ -530,7 +530,7 @@ avifResult avifImageRGBToYUV(avifImage * image, const avifRGBImage * rgb) } if (image->yuvPlanes[AVIF_CHAN_V]) { uint8_t * vPlane = image->yuvPlanes[AVIF_CHAN_V]; - const uint32_t vRowBytes = image->yuvRowBytes[AVIF_CHAN_V]; + const size_t vRowBytes = image->yuvRowBytes[AVIF_CHAN_V]; if (state.yuv.channelBytes > 1) { avifMemset16(vPlane, half, shiftedH * vRowBytes / 2); } else { @@ -664,10 +664,10 @@ static avifResult avifImageYUVAnyToRGBAnySlow(const avifImage * image, const uint8_t * uPlane = image->yuvPlanes[AVIF_CHAN_U]; const uint8_t * vPlane = image->yuvPlanes[AVIF_CHAN_V]; const uint8_t * aPlane = image->alphaPlane; - const uint32_t yRowBytes = image->yuvRowBytes[AVIF_CHAN_Y]; - const uint32_t uRowBytes = image->yuvRowBytes[AVIF_CHAN_U]; - const uint32_t vRowBytes = image->yuvRowBytes[AVIF_CHAN_V]; - const uint32_t aRowBytes = image->alphaRowBytes; + const size_t yRowBytes = image->yuvRowBytes[AVIF_CHAN_Y]; + const size_t uRowBytes = image->yuvRowBytes[AVIF_CHAN_U]; + const size_t vRowBytes = image->yuvRowBytes[AVIF_CHAN_V]; + const size_t aRowBytes = image->alphaRowBytes; // Various observations and limits const avifBool yuvHasColor = (uPlane && vPlane && (image->yuvFormat != AVIF_PIXEL_FORMAT_YUV400)); @@ -692,10 +692,10 @@ static avifResult avifImageYUVAnyToRGBAnySlow(const avifImage * image, const uint16_t * ptrV16 = (const uint16_t *)ptrV8; const uint16_t * ptrA16 = (const uint16_t *)ptrA8; - uint8_t * ptrR = &rgb->pixels[state->rgb.offsetBytesR + (j * rgb->rowBytes)]; - uint8_t * ptrG = &rgb->pixels[state->rgb.offsetBytesG + (j * rgb->rowBytes)]; - uint8_t * ptrB = &rgb->pixels[state->rgb.offsetBytesB + (j * rgb->rowBytes)]; - uint8_t * ptrGray = &rgb->pixels[state->rgb.offsetBytesGray + (j * rgb->rowBytes)]; + uint8_t * ptrR = &rgb->pixels[state->rgb.offsetBytesR + ((size_t)j * rgb->rowBytes)]; + uint8_t * ptrG = &rgb->pixels[state->rgb.offsetBytesG + ((size_t)j * rgb->rowBytes)]; + uint8_t * ptrB = &rgb->pixels[state->rgb.offsetBytesB + ((size_t)j * rgb->rowBytes)]; + uint8_t * ptrGray = &rgb->pixels[state->rgb.offsetBytesGray + ((size_t)j * rgb->rowBytes)]; for (uint32_t i = 0; i < image->width; ++i) { float Y, Cb = 0.5f, Cr = 0.5f; @@ -986,12 +986,12 @@ static avifResult avifImageYUV16ToRGB16Color(const avifImage * image, avifRGBIma const float rgbMaxChannelF = state->rgb.maxChannelF; for (uint32_t j = 0; j < image->height; ++j) { const uint32_t uvJ = j >> state->yuv.formatInfo.chromaShiftY; - const uint16_t * const ptrY = (uint16_t *)&image->yuvPlanes[AVIF_CHAN_Y][(j * image->yuvRowBytes[AVIF_CHAN_Y])]; - const uint16_t * const ptrU = (uint16_t *)&image->yuvPlanes[AVIF_CHAN_U][(uvJ * image->yuvRowBytes[AVIF_CHAN_U])]; - const uint16_t * const ptrV = (uint16_t *)&image->yuvPlanes[AVIF_CHAN_V][(uvJ * image->yuvRowBytes[AVIF_CHAN_V])]; - uint8_t * ptrR = &rgb->pixels[state->rgb.offsetBytesR + (j * rgb->rowBytes)]; - uint8_t * ptrG = &rgb->pixels[state->rgb.offsetBytesG + (j * rgb->rowBytes)]; - uint8_t * ptrB = &rgb->pixels[state->rgb.offsetBytesB + (j * rgb->rowBytes)]; + const uint16_t * const ptrY = (uint16_t *)&image->yuvPlanes[AVIF_CHAN_Y][((size_t)j * image->yuvRowBytes[AVIF_CHAN_Y])]; + const uint16_t * const ptrU = (uint16_t *)&image->yuvPlanes[AVIF_CHAN_U][((size_t)uvJ * image->yuvRowBytes[AVIF_CHAN_U])]; + const uint16_t * const ptrV = (uint16_t *)&image->yuvPlanes[AVIF_CHAN_V][((size_t)uvJ * image->yuvRowBytes[AVIF_CHAN_V])]; + uint8_t * ptrR = &rgb->pixels[state->rgb.offsetBytesR + ((size_t)j * rgb->rowBytes)]; + uint8_t * ptrG = &rgb->pixels[state->rgb.offsetBytesG + ((size_t)j * rgb->rowBytes)]; + uint8_t * ptrB = &rgb->pixels[state->rgb.offsetBytesB + ((size_t)j * rgb->rowBytes)]; for (uint32_t i = 0; i < image->width; ++i) { uint32_t uvI = i >> state->yuv.formatInfo.chromaShiftX; @@ -1038,10 +1038,10 @@ static avifResult avifImageYUV16ToRGB16Mono(const avifImage * image, avifRGBImag const uint16_t maxChannel = (uint16_t)state->yuv.maxChannel; const float maxChannelF = state->rgb.maxChannelF; for (uint32_t j = 0; j < image->height; ++j) { - const uint16_t * const ptrY = (uint16_t *)&image->yuvPlanes[AVIF_CHAN_Y][(j * image->yuvRowBytes[AVIF_CHAN_Y])]; - uint8_t * ptrR = &rgb->pixels[state->rgb.offsetBytesR + (j * rgb->rowBytes)]; - uint8_t * ptrG = &rgb->pixels[state->rgb.offsetBytesG + (j * rgb->rowBytes)]; - uint8_t * ptrB = &rgb->pixels[state->rgb.offsetBytesB + (j * rgb->rowBytes)]; + const uint16_t * const ptrY = (uint16_t *)&image->yuvPlanes[AVIF_CHAN_Y][((size_t)j * image->yuvRowBytes[AVIF_CHAN_Y])]; + uint8_t * ptrR = &rgb->pixels[state->rgb.offsetBytesR + ((size_t)j * rgb->rowBytes)]; + uint8_t * ptrG = &rgb->pixels[state->rgb.offsetBytesG + ((size_t)j * rgb->rowBytes)]; + uint8_t * ptrB = &rgb->pixels[state->rgb.offsetBytesB + ((size_t)j * rgb->rowBytes)]; for (uint32_t i = 0; i < image->width; ++i) { // clamp incoming data to protect against bad LUT lookups @@ -1086,12 +1086,12 @@ static avifResult avifImageYUV16ToRGB8Color(const avifImage * image, avifRGBImag const float rgbMaxChannelF = state->rgb.maxChannelF; for (uint32_t j = 0; j < image->height; ++j) { const uint32_t uvJ = j >> state->yuv.formatInfo.chromaShiftY; - const uint16_t * const ptrY = (uint16_t *)&image->yuvPlanes[AVIF_CHAN_Y][(j * image->yuvRowBytes[AVIF_CHAN_Y])]; - const uint16_t * const ptrU = (uint16_t *)&image->yuvPlanes[AVIF_CHAN_U][(uvJ * image->yuvRowBytes[AVIF_CHAN_U])]; - const uint16_t * const ptrV = (uint16_t *)&image->yuvPlanes[AVIF_CHAN_V][(uvJ * image->yuvRowBytes[AVIF_CHAN_V])]; - uint8_t * ptrR = &rgb->pixels[state->rgb.offsetBytesR + (j * rgb->rowBytes)]; - uint8_t * ptrG = &rgb->pixels[state->rgb.offsetBytesG + (j * rgb->rowBytes)]; - uint8_t * ptrB = &rgb->pixels[state->rgb.offsetBytesB + (j * rgb->rowBytes)]; + const uint16_t * const ptrY = (uint16_t *)&image->yuvPlanes[AVIF_CHAN_Y][((size_t)j * image->yuvRowBytes[AVIF_CHAN_Y])]; + const uint16_t * const ptrU = (uint16_t *)&image->yuvPlanes[AVIF_CHAN_U][((size_t)uvJ * image->yuvRowBytes[AVIF_CHAN_U])]; + const uint16_t * const ptrV = (uint16_t *)&image->yuvPlanes[AVIF_CHAN_V][((size_t)uvJ * image->yuvRowBytes[AVIF_CHAN_V])]; + uint8_t * ptrR = &rgb->pixels[state->rgb.offsetBytesR + ((size_t)j * rgb->rowBytes)]; + uint8_t * ptrG = &rgb->pixels[state->rgb.offsetBytesG + ((size_t)j * rgb->rowBytes)]; + uint8_t * ptrB = &rgb->pixels[state->rgb.offsetBytesB + ((size_t)j * rgb->rowBytes)]; for (uint32_t i = 0; i < image->width; ++i) { uint32_t uvI = i >> state->yuv.formatInfo.chromaShiftX; @@ -1142,10 +1142,10 @@ static avifResult avifImageYUV16ToRGB8Mono(const avifImage * image, avifRGBImage const uint16_t yuvMaxChannel = (uint16_t)state->yuv.maxChannel; const float rgbMaxChannelF = state->rgb.maxChannelF; for (uint32_t j = 0; j < image->height; ++j) { - const uint16_t * const ptrY = (uint16_t *)&image->yuvPlanes[AVIF_CHAN_Y][(j * image->yuvRowBytes[AVIF_CHAN_Y])]; - uint8_t * ptrR = &rgb->pixels[state->rgb.offsetBytesR + (j * rgb->rowBytes)]; - uint8_t * ptrG = &rgb->pixels[state->rgb.offsetBytesG + (j * rgb->rowBytes)]; - uint8_t * ptrB = &rgb->pixels[state->rgb.offsetBytesB + (j * rgb->rowBytes)]; + const uint16_t * const ptrY = (uint16_t *)&image->yuvPlanes[AVIF_CHAN_Y][((size_t)j * image->yuvRowBytes[AVIF_CHAN_Y])]; + uint8_t * ptrR = &rgb->pixels[state->rgb.offsetBytesR + ((size_t)j * rgb->rowBytes)]; + uint8_t * ptrG = &rgb->pixels[state->rgb.offsetBytesG + ((size_t)j * rgb->rowBytes)]; + uint8_t * ptrB = &rgb->pixels[state->rgb.offsetBytesB + ((size_t)j * rgb->rowBytes)]; for (uint32_t i = 0; i < image->width; ++i) { // clamp incoming data to protect against bad LUT lookups @@ -1193,12 +1193,12 @@ static avifResult avifImageYUV8ToRGB16Color(const avifImage * image, avifRGBImag const float rgbMaxChannelF = state->rgb.maxChannelF; for (uint32_t j = 0; j < image->height; ++j) { const uint32_t uvJ = j >> state->yuv.formatInfo.chromaShiftY; - const uint8_t * const ptrY = &image->yuvPlanes[AVIF_CHAN_Y][(j * image->yuvRowBytes[AVIF_CHAN_Y])]; - const uint8_t * const ptrU = &image->yuvPlanes[AVIF_CHAN_U][(uvJ * image->yuvRowBytes[AVIF_CHAN_U])]; - const uint8_t * const ptrV = &image->yuvPlanes[AVIF_CHAN_V][(uvJ * image->yuvRowBytes[AVIF_CHAN_V])]; - uint8_t * ptrR = &rgb->pixels[state->rgb.offsetBytesR + (j * rgb->rowBytes)]; - uint8_t * ptrG = &rgb->pixels[state->rgb.offsetBytesG + (j * rgb->rowBytes)]; - uint8_t * ptrB = &rgb->pixels[state->rgb.offsetBytesB + (j * rgb->rowBytes)]; + const uint8_t * const ptrY = &image->yuvPlanes[AVIF_CHAN_Y][((size_t)j * image->yuvRowBytes[AVIF_CHAN_Y])]; + const uint8_t * const ptrU = &image->yuvPlanes[AVIF_CHAN_U][((size_t)uvJ * image->yuvRowBytes[AVIF_CHAN_U])]; + const uint8_t * const ptrV = &image->yuvPlanes[AVIF_CHAN_V][((size_t)uvJ * image->yuvRowBytes[AVIF_CHAN_V])]; + uint8_t * ptrR = &rgb->pixels[state->rgb.offsetBytesR + ((size_t)j * rgb->rowBytes)]; + uint8_t * ptrG = &rgb->pixels[state->rgb.offsetBytesG + ((size_t)j * rgb->rowBytes)]; + uint8_t * ptrB = &rgb->pixels[state->rgb.offsetBytesB + ((size_t)j * rgb->rowBytes)]; for (uint32_t i = 0; i < image->width; ++i) { uint32_t uvI = i >> state->yuv.formatInfo.chromaShiftX; @@ -1239,10 +1239,10 @@ static avifResult avifImageYUV8ToRGB16Mono(const avifImage * image, avifRGBImage const float rgbMaxChannelF = state->rgb.maxChannelF; for (uint32_t j = 0; j < image->height; ++j) { - const uint8_t * const ptrY = &image->yuvPlanes[AVIF_CHAN_Y][(j * image->yuvRowBytes[AVIF_CHAN_Y])]; - uint8_t * ptrR = &rgb->pixels[state->rgb.offsetBytesR + (j * rgb->rowBytes)]; - uint8_t * ptrG = &rgb->pixels[state->rgb.offsetBytesG + (j * rgb->rowBytes)]; - uint8_t * ptrB = &rgb->pixels[state->rgb.offsetBytesB + (j * rgb->rowBytes)]; + const uint8_t * const ptrY = &image->yuvPlanes[AVIF_CHAN_Y][((size_t)j * image->yuvRowBytes[AVIF_CHAN_Y])]; + uint8_t * ptrR = &rgb->pixels[state->rgb.offsetBytesR + ((size_t)j * rgb->rowBytes)]; + uint8_t * ptrG = &rgb->pixels[state->rgb.offsetBytesG + ((size_t)j * rgb->rowBytes)]; + uint8_t * ptrB = &rgb->pixels[state->rgb.offsetBytesB + ((size_t)j * rgb->rowBytes)]; for (uint32_t i = 0; i < image->width; ++i) { // Convert unorm to float (no clamp necessary, the full uint8_t range is a legal lookup) @@ -1274,12 +1274,12 @@ static avifResult avifImageIdentity8ToRGB8ColorFullRange(const avifImage * image { const uint32_t rgbPixelBytes = state->rgb.pixelBytes; for (uint32_t j = 0; j < image->height; ++j) { - const uint8_t * const ptrY = &image->yuvPlanes[AVIF_CHAN_Y][(j * image->yuvRowBytes[AVIF_CHAN_Y])]; - const uint8_t * const ptrU = &image->yuvPlanes[AVIF_CHAN_U][(j * image->yuvRowBytes[AVIF_CHAN_U])]; - const uint8_t * const ptrV = &image->yuvPlanes[AVIF_CHAN_V][(j * image->yuvRowBytes[AVIF_CHAN_V])]; - uint8_t * ptrR = &rgb->pixels[state->rgb.offsetBytesR + (j * rgb->rowBytes)]; - uint8_t * ptrG = &rgb->pixels[state->rgb.offsetBytesG + (j * rgb->rowBytes)]; - uint8_t * ptrB = &rgb->pixels[state->rgb.offsetBytesB + (j * rgb->rowBytes)]; + const uint8_t * const ptrY = &image->yuvPlanes[AVIF_CHAN_Y][((size_t)j * image->yuvRowBytes[AVIF_CHAN_Y])]; + const uint8_t * const ptrU = &image->yuvPlanes[AVIF_CHAN_U][((size_t)j * image->yuvRowBytes[AVIF_CHAN_U])]; + const uint8_t * const ptrV = &image->yuvPlanes[AVIF_CHAN_V][((size_t)j * image->yuvRowBytes[AVIF_CHAN_V])]; + uint8_t * ptrR = &rgb->pixels[state->rgb.offsetBytesR + ((size_t)j * rgb->rowBytes)]; + uint8_t * ptrG = &rgb->pixels[state->rgb.offsetBytesG + ((size_t)j * rgb->rowBytes)]; + uint8_t * ptrB = &rgb->pixels[state->rgb.offsetBytesB + ((size_t)j * rgb->rowBytes)]; // This is intentionally a per-row conditional instead of a per-pixel // conditional. This makes the "else" path (much more common than the @@ -1316,12 +1316,12 @@ static avifResult avifImageYUV8ToRGB8Color(const avifImage * image, avifRGBImage const float rgbMaxChannelF = state->rgb.maxChannelF; for (uint32_t j = 0; j < image->height; ++j) { const uint32_t uvJ = j >> state->yuv.formatInfo.chromaShiftY; - const uint8_t * const ptrY = &image->yuvPlanes[AVIF_CHAN_Y][(j * image->yuvRowBytes[AVIF_CHAN_Y])]; - const uint8_t * const ptrU = &image->yuvPlanes[AVIF_CHAN_U][(uvJ * image->yuvRowBytes[AVIF_CHAN_U])]; - const uint8_t * const ptrV = &image->yuvPlanes[AVIF_CHAN_V][(uvJ * image->yuvRowBytes[AVIF_CHAN_V])]; - uint8_t * ptrR = &rgb->pixels[state->rgb.offsetBytesR + (j * rgb->rowBytes)]; - uint8_t * ptrG = &rgb->pixels[state->rgb.offsetBytesG + (j * rgb->rowBytes)]; - uint8_t * ptrB = &rgb->pixels[state->rgb.offsetBytesB + (j * rgb->rowBytes)]; + const uint8_t * const ptrY = &image->yuvPlanes[AVIF_CHAN_Y][((size_t)j * image->yuvRowBytes[AVIF_CHAN_Y])]; + const uint8_t * const ptrU = &image->yuvPlanes[AVIF_CHAN_U][((size_t)uvJ * image->yuvRowBytes[AVIF_CHAN_U])]; + const uint8_t * const ptrV = &image->yuvPlanes[AVIF_CHAN_V][((size_t)uvJ * image->yuvRowBytes[AVIF_CHAN_V])]; + uint8_t * ptrR = &rgb->pixels[state->rgb.offsetBytesR + ((size_t)j * rgb->rowBytes)]; + uint8_t * ptrG = &rgb->pixels[state->rgb.offsetBytesG + ((size_t)j * rgb->rowBytes)]; + uint8_t * ptrB = &rgb->pixels[state->rgb.offsetBytesB + ((size_t)j * rgb->rowBytes)]; for (uint32_t i = 0; i < image->width; ++i) { uint32_t uvI = i >> state->yuv.formatInfo.chromaShiftX; @@ -1366,10 +1366,10 @@ static avifResult avifImageYUV8ToRGB8Mono(const avifImage * image, avifRGBImage const float rgbMaxChannelF = state->rgb.maxChannelF; for (uint32_t j = 0; j < image->height; ++j) { - const uint8_t * const ptrY = &image->yuvPlanes[AVIF_CHAN_Y][(j * image->yuvRowBytes[AVIF_CHAN_Y])]; - uint8_t * ptrR = &rgb->pixels[state->rgb.offsetBytesR + (j * rgb->rowBytes)]; - uint8_t * ptrG = &rgb->pixels[state->rgb.offsetBytesG + (j * rgb->rowBytes)]; - uint8_t * ptrB = &rgb->pixels[state->rgb.offsetBytesB + (j * rgb->rowBytes)]; + const uint8_t * const ptrY = &image->yuvPlanes[AVIF_CHAN_Y][((size_t)j * image->yuvRowBytes[AVIF_CHAN_Y])]; + uint8_t * ptrR = &rgb->pixels[state->rgb.offsetBytesR + ((size_t)j * rgb->rowBytes)]; + uint8_t * ptrG = &rgb->pixels[state->rgb.offsetBytesG + ((size_t)j * rgb->rowBytes)]; + uint8_t * ptrB = &rgb->pixels[state->rgb.offsetBytesB + ((size_t)j * rgb->rowBytes)]; for (uint32_t i = 0; i < image->width; ++i) { // Convert unorm to float (no clamp necessary, the full uint8_t range is a legal lookup) @@ -1854,7 +1854,7 @@ void avifGetRGBAPixel(const avifRGBImage * src, uint32_t x, uint32_t y, const av assert(!src->isFloat || src->depth == 16); assert(src->format != AVIF_RGB_FORMAT_RGB_565 || src->depth == 8); - const uint8_t * const srcPixel = &src->pixels[y * src->rowBytes + x * info->pixelBytes]; + const uint8_t * const srcPixel = &src->pixels[(size_t)y * src->rowBytes + x * info->pixelBytes]; if (info->channelBytes > 1) { uint16_t r = *((const uint16_t *)(&srcPixel[info->offsetBytesR])); uint16_t g = *((const uint16_t *)(&srcPixel[info->offsetBytesG])); @@ -1897,7 +1897,7 @@ void avifSetRGBAPixel(const avifRGBImage * dst, uint32_t x, uint32_t y, const av assert(rgbaPixel[1] >= 0.0f && rgbaPixel[1] <= 1.0f); assert(rgbaPixel[2] >= 0.0f && rgbaPixel[2] <= 1.0f); - uint8_t * const dstPixel = &dst->pixels[y * dst->rowBytes + x * info->pixelBytes]; + uint8_t * const dstPixel = &dst->pixels[(size_t)y * dst->rowBytes + x * info->pixelBytes]; uint8_t * const ptrR = &dstPixel[info->offsetBytesR]; uint8_t * const ptrG = &dstPixel[info->offsetBytesG]; From 5bf8fa21f3521d7fabf074ddf7ac02e8eb3a55b0 Mon Sep 17 00:00:00 2001 From: "Dexter.k" <164054284+rootvector2@users.noreply.github.com> Date: Mon, 23 Feb 2026 10:35:40 +0000 Subject: [PATCH 2/5] prevent potential size_t overflow in UV plane size calculations --- src/codec_svt.c | 21 ++++++++++++++++++--- src/reformat.c | 3 ++- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/codec_svt.c b/src/codec_svt.c index dccc388be2..fe155ade5b 100644 --- a/src/codec_svt.c +++ b/src/codec_svt.c @@ -278,9 +278,24 @@ static avifResult svtCodecEncodeImage(avifCodec * codec, #if SVT_AV1_CHECK_VERSION(1, 8, 0) // Simulate 4:2:0 UV planes. SVT-AV1 does not support 4:0:0 samples. - const uint32_t uvWidth = (image->width + y_shift) >> y_shift; - const uint32_t uvRowBytes = uvWidth * bytesPerPixel; - const size_t uvSize = (size_t)uvRowBytes * uvHeight; + const size_t uvWidth = ((size_t)image->width + y_shift) >> y_shift; + + // Use size_t to avoid 32-bit overflow + const size_t uvRowBytes = (size_t)uvWidth * (size_t)bytesPerPixel; + + // Verify multiplication overflow + if (uvWidth != 0 && + uvRowBytes / (size_t)uvWidth != (size_t)bytesPerPixel) { + goto cleanup; + } + + const size_t uvSize = uvRowBytes * (size_t)uvHeight; + + // Verify second multiplication overflow + if (uvHeight != 0 && + uvSize / (size_t)uvHeight != uvRowBytes) { + goto cleanup; + } uvPlanes = avifAlloc(uvSize); if (uvPlanes == NULL) { goto cleanup; diff --git a/src/reformat.c b/src/reformat.c index e3c8932da4..3f89c6f444 100644 --- a/src/reformat.c +++ b/src/reformat.c @@ -1854,7 +1854,8 @@ void avifGetRGBAPixel(const avifRGBImage * src, uint32_t x, uint32_t y, const av assert(!src->isFloat || src->depth == 16); assert(src->format != AVIF_RGB_FORMAT_RGB_565 || src->depth == 8); - const uint8_t * const srcPixel = &src->pixels[(size_t)y * src->rowBytes + x * info->pixelBytes]; + const size_t offset = (size_t)y * (size_t)src->rowBytes + (size_t)x * (size_t)info->pixelBytes; + const uint8_t * const srcPixel = &src->pixels[offset]; if (info->channelBytes > 1) { uint16_t r = *((const uint16_t *)(&srcPixel[info->offsetBytesR])); uint16_t g = *((const uint16_t *)(&srcPixel[info->offsetBytesG])); From dc8d99cc35f9da6c98f4c35f5d8b861c1705dc98 Mon Sep 17 00:00:00 2001 From: "Dexter.k" <164054284+rootvector2@users.noreply.github.com> Date: Tue, 24 Feb 2026 19:44:36 +0000 Subject: [PATCH 3/5] conversion warnings in codec_svt.c --- src/codec_svt.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/codec_svt.c b/src/codec_svt.c index fe155ade5b..353bec4806 100644 --- a/src/codec_svt.c +++ b/src/codec_svt.c @@ -274,7 +274,7 @@ static avifResult svtCodecEncodeImage(avifCodec * codec, if (alpha) { input_picture_buffer->y_stride = image->alphaRowBytes / bytesPerPixel; input_picture_buffer->luma = image->alphaPlane; - input_buffer->n_filled_len = (size_t)image->alphaRowBytes * image->height; + input_buffer->n_filled_len = (uint32_t)((size_t)image->alphaRowBytes * image->height); #if SVT_AV1_CHECK_VERSION(1, 8, 0) // Simulate 4:2:0 UV planes. SVT-AV1 does not support 4:0:0 samples. @@ -302,11 +302,11 @@ static avifResult svtCodecEncodeImage(avifCodec * codec, } memset(uvPlanes, 0, uvSize); input_picture_buffer->cb = uvPlanes; - input_buffer->n_filled_len += uvSize; + input_buffer->n_filled_len += (uint32_t)uvSize; input_picture_buffer->cr = uvPlanes; - input_buffer->n_filled_len += uvSize; - input_picture_buffer->cb_stride = uvWidth; - input_picture_buffer->cr_stride = uvWidth; + input_buffer->n_filled_len += (uint32_t)uvSize; + input_picture_buffer->cb_stride = (uint32_t)uvWidth; + input_picture_buffer->cr_stride = (uint32_t)uvWidth; #else // This workaround was not needed before SVT-AV1 1.8.0. // See https://github.com/AOMediaCodec/libavif/issues/1992. @@ -315,11 +315,11 @@ static avifResult svtCodecEncodeImage(avifCodec * codec, } else { input_picture_buffer->y_stride = image->yuvRowBytes[0] / bytesPerPixel; input_picture_buffer->luma = image->yuvPlanes[0]; - input_buffer->n_filled_len = (size_t)image->yuvRowBytes[0] * image->height; + input_buffer->n_filled_len = (uint32_t)((size_t)image->yuvRowBytes[0] * image->height); input_picture_buffer->cb = image->yuvPlanes[1]; - input_buffer->n_filled_len += (size_t)image->yuvRowBytes[1] * uvHeight; + input_buffer->n_filled_len += (uint32_t)((size_t)image->yuvRowBytes[1] * uvHeight); input_picture_buffer->cr = image->yuvPlanes[2]; - input_buffer->n_filled_len += (size_t)image->yuvRowBytes[2] * uvHeight; + input_buffer->n_filled_len += (uint32_t)((size_t)image->yuvRowBytes[2] * uvHeight); input_picture_buffer->cb_stride = image->yuvRowBytes[1] / bytesPerPixel; input_picture_buffer->cr_stride = image->yuvRowBytes[2] / bytesPerPixel; } From 8ea8f533a39f6ebea511628591c208f5998616e6 Mon Sep 17 00:00:00 2001 From: "Dexter.k" <164054284+rootvector2@users.noreply.github.com> Date: Wed, 25 Feb 2026 07:01:30 +0000 Subject: [PATCH 4/5] overflow checks and size_t casts --- src/codec_aom.c | 4 ++-- src/codec_avm.c | 4 ++-- src/codec_svt.c | 47 +++++++++++++++++++++++++++-------------------- src/reformat.c | 5 ++--- 4 files changed, 33 insertions(+), 27 deletions(-) diff --git a/src/codec_aom.c b/src/codec_aom.c index 6846ec4105..0ed48635aa 100644 --- a/src/codec_aom.c +++ b/src/codec_aom.c @@ -1210,7 +1210,7 @@ static avifResult aomCodecEncodeImage(avifCodec * codec, const uint32_t bytesPerRow = ((image->depth > 8) ? 2 : 1) * image->width; for (uint32_t j = 0; j < image->height; ++j) { const uint8_t * srcAlphaRow = &image->alphaPlane[(size_t)j * image->alphaRowBytes]; - uint8_t * dstAlphaRow = &aomImage.planes[0][j * aomImage.stride[0]]; + uint8_t * dstAlphaRow = &aomImage.planes[0][(size_t)j * aomImage.stride[0]]; memcpy(dstAlphaRow, srcAlphaRow, bytesPerRow); } } else { @@ -1234,7 +1234,7 @@ static avifResult aomCodecEncodeImage(avifCodec * codec, for (uint32_t j = 0; j < planeHeight; ++j) { const uint8_t * srcRow = &image->yuvPlanes[yuvPlane][(size_t)j * image->yuvRowBytes[yuvPlane]]; - uint8_t * dstRow = &aomImage.planes[yuvPlane][j * aomImage.stride[yuvPlane]]; + uint8_t * dstRow = &aomImage.planes[yuvPlane][(size_t)j * aomImage.stride[yuvPlane]]; memcpy(dstRow, srcRow, bytesPerRow); } } diff --git a/src/codec_avm.c b/src/codec_avm.c index 9e09ed0c28..c016eeb13d 100644 --- a/src/codec_avm.c +++ b/src/codec_avm.c @@ -904,7 +904,7 @@ static avifResult avmCodecEncodeImage(avifCodec * codec, const uint32_t bytesPerRow = ((image->depth > 8) ? 2 : 1) * image->width; for (uint32_t j = 0; j < image->height; ++j) { const uint8_t * srcAlphaRow = &image->alphaPlane[(size_t)j * image->alphaRowBytes]; - uint8_t * dstAlphaRow = &avmImage.planes[0][j * avmImage.stride[0]]; + uint8_t * dstAlphaRow = &avmImage.planes[0][(size_t)j * avmImage.stride[0]]; memcpy(dstAlphaRow, srcAlphaRow, bytesPerRow); } } else { @@ -928,7 +928,7 @@ static avifResult avmCodecEncodeImage(avifCodec * codec, for (uint32_t j = 0; j < planeHeight; ++j) { const uint8_t * srcRow = &image->yuvPlanes[yuvPlane][(size_t)j * image->yuvRowBytes[yuvPlane]]; - uint8_t * dstRow = &avmImage.planes[yuvPlane][j * avmImage.stride[yuvPlane]]; + uint8_t * dstRow = &avmImage.planes[yuvPlane][(size_t)j * avmImage.stride[yuvPlane]]; memcpy(dstRow, srcRow, bytesPerRow); } } diff --git a/src/codec_svt.c b/src/codec_svt.c index 353bec4806..2b887e85ef 100644 --- a/src/codec_svt.c +++ b/src/codec_svt.c @@ -274,26 +274,21 @@ static avifResult svtCodecEncodeImage(avifCodec * codec, if (alpha) { input_picture_buffer->y_stride = image->alphaRowBytes / bytesPerPixel; input_picture_buffer->luma = image->alphaPlane; - input_buffer->n_filled_len = (uint32_t)((size_t)image->alphaRowBytes * image->height); + const size_t alphaSize = (size_t)image->alphaRowBytes * image->height; + if (alphaSize > UINT32_MAX) { + goto cleanup; + } + input_buffer->n_filled_len = (uint32_t)alphaSize; #if SVT_AV1_CHECK_VERSION(1, 8, 0) // Simulate 4:2:0 UV planes. SVT-AV1 does not support 4:0:0 samples. - const size_t uvWidth = ((size_t)image->width + y_shift) >> y_shift; - - // Use size_t to avoid 32-bit overflow - const size_t uvRowBytes = (size_t)uvWidth * (size_t)bytesPerPixel; - - // Verify multiplication overflow - if (uvWidth != 0 && - uvRowBytes / (size_t)uvWidth != (size_t)bytesPerPixel) { + const uint32_t uvWidth = (image->width + y_shift) >> y_shift; + const uint32_t uvRowBytes = uvWidth * bytesPerPixel; + const size_t uvSize = (size_t)uvRowBytes * uvHeight; + if (uvSize > UINT32_MAX / 2) { goto cleanup; } - - const size_t uvSize = uvRowBytes * (size_t)uvHeight; - - // Verify second multiplication overflow - if (uvHeight != 0 && - uvSize / (size_t)uvHeight != uvRowBytes) { + if (uvSize * 2 > UINT32_MAX - input_buffer->n_filled_len) { goto cleanup; } uvPlanes = avifAlloc(uvSize); @@ -305,8 +300,8 @@ static avifResult svtCodecEncodeImage(avifCodec * codec, input_buffer->n_filled_len += (uint32_t)uvSize; input_picture_buffer->cr = uvPlanes; input_buffer->n_filled_len += (uint32_t)uvSize; - input_picture_buffer->cb_stride = (uint32_t)uvWidth; - input_picture_buffer->cr_stride = (uint32_t)uvWidth; + input_picture_buffer->cb_stride = uvWidth; + input_picture_buffer->cr_stride = uvWidth; #else // This workaround was not needed before SVT-AV1 1.8.0. // See https://github.com/AOMediaCodec/libavif/issues/1992. @@ -315,11 +310,23 @@ static avifResult svtCodecEncodeImage(avifCodec * codec, } else { input_picture_buffer->y_stride = image->yuvRowBytes[0] / bytesPerPixel; input_picture_buffer->luma = image->yuvPlanes[0]; - input_buffer->n_filled_len = (uint32_t)((size_t)image->yuvRowBytes[0] * image->height); + const size_t ySize = (size_t)image->yuvRowBytes[0] * image->height; + if (ySize > UINT32_MAX) { + goto cleanup; + } + input_buffer->n_filled_len = (uint32_t)ySize; input_picture_buffer->cb = image->yuvPlanes[1]; - input_buffer->n_filled_len += (uint32_t)((size_t)image->yuvRowBytes[1] * uvHeight); + const size_t uSize = (size_t)image->yuvRowBytes[1] * uvHeight; + if (uSize > UINT32_MAX - input_buffer->n_filled_len) { + goto cleanup; + } + input_buffer->n_filled_len += (uint32_t)uSize; input_picture_buffer->cr = image->yuvPlanes[2]; - input_buffer->n_filled_len += (uint32_t)((size_t)image->yuvRowBytes[2] * uvHeight); + const size_t vSize = (size_t)image->yuvRowBytes[2] * uvHeight; + if (vSize > UINT32_MAX - input_buffer->n_filled_len) { + goto cleanup; + } + input_buffer->n_filled_len += (uint32_t)vSize; input_picture_buffer->cb_stride = image->yuvRowBytes[1] / bytesPerPixel; input_picture_buffer->cr_stride = image->yuvRowBytes[2] / bytesPerPixel; } diff --git a/src/reformat.c b/src/reformat.c index 3f89c6f444..ec16bdf7e0 100644 --- a/src/reformat.c +++ b/src/reformat.c @@ -1854,8 +1854,7 @@ void avifGetRGBAPixel(const avifRGBImage * src, uint32_t x, uint32_t y, const av assert(!src->isFloat || src->depth == 16); assert(src->format != AVIF_RGB_FORMAT_RGB_565 || src->depth == 8); - const size_t offset = (size_t)y * (size_t)src->rowBytes + (size_t)x * (size_t)info->pixelBytes; - const uint8_t * const srcPixel = &src->pixels[offset]; + const uint8_t * const srcPixel = &src->pixels[(size_t)y * src->rowBytes + (size_t)x * info->pixelBytes]; if (info->channelBytes > 1) { uint16_t r = *((const uint16_t *)(&srcPixel[info->offsetBytesR])); uint16_t g = *((const uint16_t *)(&srcPixel[info->offsetBytesG])); @@ -1898,7 +1897,7 @@ void avifSetRGBAPixel(const avifRGBImage * dst, uint32_t x, uint32_t y, const av assert(rgbaPixel[1] >= 0.0f && rgbaPixel[1] <= 1.0f); assert(rgbaPixel[2] >= 0.0f && rgbaPixel[2] <= 1.0f); - uint8_t * const dstPixel = &dst->pixels[(size_t)y * dst->rowBytes + x * info->pixelBytes]; + uint8_t * const dstPixel = &dst->pixels[(size_t)y * dst->rowBytes + (size_t)x * info->pixelBytes]; uint8_t * const ptrR = &dstPixel[info->offsetBytesR]; uint8_t * const ptrG = &dstPixel[info->offsetBytesG]; From d2ea0793a8a432c385d1b9d8ee45dfa68b61547c Mon Sep 17 00:00:00 2001 From: "Dexter.k" <164054284+rootvector2@users.noreply.github.com> Date: Wed, 25 Feb 2026 20:07:57 +0000 Subject: [PATCH 5/5] undo the changes to these four lines --- src/reformat.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/reformat.c b/src/reformat.c index ec16bdf7e0..7b53addfcb 100644 --- a/src/reformat.c +++ b/src/reformat.c @@ -664,10 +664,10 @@ static avifResult avifImageYUVAnyToRGBAnySlow(const avifImage * image, const uint8_t * uPlane = image->yuvPlanes[AVIF_CHAN_U]; const uint8_t * vPlane = image->yuvPlanes[AVIF_CHAN_V]; const uint8_t * aPlane = image->alphaPlane; - const size_t yRowBytes = image->yuvRowBytes[AVIF_CHAN_Y]; - const size_t uRowBytes = image->yuvRowBytes[AVIF_CHAN_U]; - const size_t vRowBytes = image->yuvRowBytes[AVIF_CHAN_V]; - const size_t aRowBytes = image->alphaRowBytes; + const uint32_t yRowBytes = image->yuvRowBytes[AVIF_CHAN_Y]; + const uint32_t uRowBytes = image->yuvRowBytes[AVIF_CHAN_U]; + const uint32_t vRowBytes = image->yuvRowBytes[AVIF_CHAN_V]; + const uint32_t aRowBytes = image->alphaRowBytes; // Various observations and limits const avifBool yuvHasColor = (uPlane && vPlane && (image->yuvFormat != AVIF_PIXEL_FORMAT_YUV400));