Skip to content

Fix potential integer overflow in rowBytes multiplications #3053

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
wantehchang merged 5 commits into from
Feb 25, 2026
+97 −75
Merged

Fix potential integer overflow in rowBytes multiplications #3053

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
60f7cba
Fix potential integer overflow in rowBytes multiplications
rootvector2 Feb 22, 2026
5bf8fa2
prevent potential size_t overflow in UV plane size calculations
rootvector2 Feb 23, 2026
dc8d99c
conversion warnings in codec_svt.c
rootvector2 Feb 24, 2026
8ea8f53
overflow checks and size_t casts
rootvector2 Feb 25, 2026
d2ea079
undo the changes to these four lines
rootvector2 Feb 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions src/codec_aom.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1209,8 +1209,8 @@ static avifResult aomCodecEncodeImage(avifCodec * codec,
if (aomImageAllocated) {
const uint32_t bytesPerRow = ((image->depth > 8) ? 2 : 1) * image->width;
for (uint32_t j = 0; j < image->height; ++j) {
const uint8_t * srcAlphaRow = &image->alphaPlane[j * image->alphaRowBytes];
uint8_t * dstAlphaRow = &aomImage.planes[0][j * aomImage.stride[0]];
const uint8_t * srcAlphaRow = &image->alphaPlane[(size_t)j * image->alphaRowBytes];
uint8_t * dstAlphaRow = &aomImage.planes[0][(size_t)j * aomImage.stride[0]];
Copy link
Collaborator

@wantehchang wantehchang Feb 25, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Dexter.k: For future reference: This for loop could be also be rewritten as follows, which avoids multiplying rowBytes / stride with the loop index j:

            const uint8_t * srcAlphaRow = image->alphaPlane;
            uint8_t * dstAlphaRow = aomImage.planes[0];
            for (uint32_t j = 0; j < image->height; ++j) {
                memcpy(dstAlphaRow, srcAlphaRow, bytesPerRow);
                srcAlphaRow += image->alphaRowBytes;
                dstAlphaRow += aomImage.stride[0];
            }

memcpy(dstAlphaRow, srcAlphaRow, bytesPerRow);
}
} else {
Expand All @@ -1233,8 +1233,8 @@ static avifResult aomCodecEncodeImage(avifCodec * codec,
uint32_t bytesPerRow = bytesPerPixel * planeWidth;

for (uint32_t j = 0; j < planeHeight; ++j) {
const uint8_t * srcRow = &image->yuvPlanes[yuvPlane][j * image->yuvRowBytes[yuvPlane]];
uint8_t * dstRow = &aomImage.planes[yuvPlane][j * aomImage.stride[yuvPlane]];
const uint8_t * srcRow = &image->yuvPlanes[yuvPlane][(size_t)j * image->yuvRowBytes[yuvPlane]];
uint8_t * dstRow = &aomImage.planes[yuvPlane][(size_t)j * aomImage.stride[yuvPlane]];
memcpy(dstRow, srcRow, bytesPerRow);
}
}
Expand Down
8 changes: 4 additions & 4 deletions src/codec_avm.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -903,8 +903,8 @@ static avifResult avmCodecEncodeImage(avifCodec * codec,
if (avmImageAllocated) {
const uint32_t bytesPerRow = ((image->depth > 8) ? 2 : 1) * image->width;
for (uint32_t j = 0; j < image->height; ++j) {
const uint8_t * srcAlphaRow = &image->alphaPlane[j * image->alphaRowBytes];
uint8_t * dstAlphaRow = &avmImage.planes[0][j * avmImage.stride[0]];
const uint8_t * srcAlphaRow = &image->alphaPlane[(size_t)j * image->alphaRowBytes];
uint8_t * dstAlphaRow = &avmImage.planes[0][(size_t)j * avmImage.stride[0]];
memcpy(dstAlphaRow, srcAlphaRow, bytesPerRow);
}
} else {
Expand All @@ -927,8 +927,8 @@ static avifResult avmCodecEncodeImage(avifCodec * codec,
uint32_t bytesPerRow = bytesPerPixel * planeWidth;

for (uint32_t j = 0; j < planeHeight; ++j) {
const uint8_t * srcRow = &image->yuvPlanes[yuvPlane][j * image->yuvRowBytes[yuvPlane]];
uint8_t * dstRow = &avmImage.planes[yuvPlane][j * avmImage.stride[yuvPlane]];
const uint8_t * srcRow = &image->yuvPlanes[yuvPlane][(size_t)j * image->yuvRowBytes[yuvPlane]];
uint8_t * dstRow = &avmImage.planes[yuvPlane][(size_t)j * avmImage.stride[yuvPlane]];
memcpy(dstRow, srcRow, bytesPerRow);
}
}
Expand Down
36 changes: 29 additions & 7 deletions src/codec_svt.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -274,22 +274,32 @@ static avifResult svtCodecEncodeImage(avifCodec * codec,
if (alpha) {
input_picture_buffer->y_stride = image->alphaRowBytes / bytesPerPixel;
input_picture_buffer->luma = image->alphaPlane;
input_buffer->n_filled_len = image->alphaRowBytes * image->height;
const size_t alphaSize = (size_t)image->alphaRowBytes * image->height;
if (alphaSize > UINT32_MAX) {
goto cleanup;
}
input_buffer->n_filled_len = (uint32_t)alphaSize;

#if SVT_AV1_CHECK_VERSION(1, 8, 0)
// Simulate 4:2:0 UV planes. SVT-AV1 does not support 4:0:0 samples.
const uint32_t uvWidth = (image->width + y_shift) >> y_shift;
const uint32_t uvRowBytes = uvWidth * bytesPerPixel;
const uint32_t uvSize = uvRowBytes * uvHeight;
const size_t uvSize = (size_t)uvRowBytes * uvHeight;
if (uvSize > UINT32_MAX / 2) {
goto cleanup;
}
if (uvSize * 2 > UINT32_MAX - input_buffer->n_filled_len) {
goto cleanup;
}
uvPlanes = avifAlloc(uvSize);
if (uvPlanes == NULL) {
goto cleanup;
}
memset(uvPlanes, 0, uvSize);
input_picture_buffer->cb = uvPlanes;
input_buffer->n_filled_len += uvSize;
input_buffer->n_filled_len += (uint32_t)uvSize;
input_picture_buffer->cr = uvPlanes;
input_buffer->n_filled_len += uvSize;
input_buffer->n_filled_len += (uint32_t)uvSize;
input_picture_buffer->cb_stride = uvWidth;
input_picture_buffer->cr_stride = uvWidth;
#else
Expand All @@ -300,11 +310,23 @@ static avifResult svtCodecEncodeImage(avifCodec * codec,
} else {
input_picture_buffer->y_stride = image->yuvRowBytes[0] / bytesPerPixel;
input_picture_buffer->luma = image->yuvPlanes[0];
input_buffer->n_filled_len = image->yuvRowBytes[0] * image->height;
const size_t ySize = (size_t)image->yuvRowBytes[0] * image->height;
if (ySize > UINT32_MAX) {
goto cleanup;
}
input_buffer->n_filled_len = (uint32_t)ySize;
input_picture_buffer->cb = image->yuvPlanes[1];
input_buffer->n_filled_len += image->yuvRowBytes[1] * uvHeight;
const size_t uSize = (size_t)image->yuvRowBytes[1] * uvHeight;
if (uSize > UINT32_MAX - input_buffer->n_filled_len) {
goto cleanup;
}
input_buffer->n_filled_len += (uint32_t)uSize;
input_picture_buffer->cr = image->yuvPlanes[2];
input_buffer->n_filled_len += image->yuvRowBytes[2] * uvHeight;
const size_t vSize = (size_t)image->yuvRowBytes[2] * uvHeight;
if (vSize > UINT32_MAX - input_buffer->n_filled_len) {
goto cleanup;
}
input_buffer->n_filled_len += (uint32_t)vSize;
input_picture_buffer->cb_stride = image->yuvRowBytes[1] / bytesPerPixel;
input_picture_buffer->cr_stride = image->yuvRowBytes[2] / bytesPerPixel;
}
Expand Down
8 changes: 4 additions & 4 deletions src/read.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6599,8 +6599,8 @@ static avifResult avifImageLimitedToFullAlpha(avifImage * image)

if (image->depth > 8) {
for (uint32_t j = 0; j < image->height; ++j) {
const uint8_t * srcRow = &alphaPlane[j * alphaRowBytes];
uint8_t * dstRow = &image->alphaPlane[j * image->alphaRowBytes];
const uint8_t * srcRow = &alphaPlane[(size_t)j * alphaRowBytes];
uint8_t * dstRow = &image->alphaPlane[(size_t)j * image->alphaRowBytes];
for (uint32_t i = 0; i < image->width; ++i) {
int srcAlpha = *((const uint16_t *)&srcRow[i * 2]);
int dstAlpha = avifLimitedToFullY(image->depth, srcAlpha);
Expand All @@ -6609,8 +6609,8 @@ static avifResult avifImageLimitedToFullAlpha(avifImage * image)
}
} else {
for (uint32_t j = 0; j < image->height; ++j) {
const uint8_t * srcRow = &alphaPlane[j * alphaRowBytes];
uint8_t * dstRow = &image->alphaPlane[j * image->alphaRowBytes];
const uint8_t * srcRow = &alphaPlane[(size_t)j * alphaRowBytes];
uint8_t * dstRow = &image->alphaPlane[(size_t)j * image->alphaRowBytes];
for (uint32_t i = 0; i < image->width; ++i) {
int srcAlpha = srcRow[i];
int dstAlpha = avifLimitedToFullY(image->depth, srcAlpha);
Expand Down
Loading
Loading