From efa7c281b17ab5649b98980dd6f795be7058dede Mon Sep 17 00:00:00 2001 From: Anthony Hurtado Date: Mon, 4 May 2026 11:14:03 -0500 Subject: [PATCH 1/5] Fix NaN bypass of AVIF_CLAMP in gain map pixel clamping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit powf() in avifRGBImageComputeGainMap() can return NaN when processing degenerate gain map gamma fractions. The NaN value bypasses AVIF_CLAMP(v, 0.0f, 1.0f) because IEEE 754 defines all comparisons with NaN as false, so the macro evaluates to NaN. The unclamped NaN then reaches avifSetRGBAPixel(), triggering the assertion that pixel values are in [0.0, 1.0]. Replace AVIF_CLAMP with fminf/fmaxf at the three call sites that feed pixel data into avifSetRGBAPixel(). Per C99 §7.12.12, fmaxf/fminf return the non-NaN argument when one argument is NaN, which correctly clamps NaN to 0.0f. Found via AFL++ fuzzing of the experimental gain map API. --- src/gainmap.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/gainmap.c b/src/gainmap.c index 40a332fc2f..f832fda769 100644 --- a/src/gainmap.c +++ b/src/gainmap.c @@ -153,7 +153,11 @@ avifResult avifRGBImageApplyGainMap(const avifRGBImage * baseImage, avifLinearRGBConvertColorSpace(basePixelRGBA, conversionCoeffs); } for (int c = 0; c < 3; ++c) { - basePixelRGBA[c] = AVIF_CLAMP(linearToGamma(basePixelRGBA[c]), 0.0f, 1.0f); + // Use fminf/fmaxf instead of AVIF_CLAMP for NaN safety: + // AVIF_CLAMP passes NaN through because IEEE 754 comparisons + // with NaN always return false. fmaxf/fminf return the non-NaN + // argument per C99 §7.12.12. + basePixelRGBA[c] = fminf(1.0f, fmaxf(0.0f, linearToGamma(basePixelRGBA[c]))); } } avifSetRGBAPixel(toneMappedImage, i, j, &toneMappedPixelRGBInfo, basePixelRGBA); @@ -270,7 +274,8 @@ avifResult avifRGBImageApplyGainMap(const avifRGBImage * baseImage, } for (int c = 0; c < 3; ++c) { - toneMappedPixelRGBA[c] = AVIF_CLAMP(linearToGamma(toneMappedPixelRGBA[c]), 0.0f, 1.0f); + // NaN-safe clamp: fmaxf/fminf return the non-NaN argument per C99. + toneMappedPixelRGBA[c] = fminf(1.0f, fmaxf(0.0f, linearToGamma(toneMappedPixelRGBA[c]))); } toneMappedPixelRGBA[3] = basePixelRGBA[3]; // Alpha is unaffected by tone mapping. @@ -762,7 +767,11 @@ avifResult avifRGBImageComputeGainMap(const avifRGBImage * baseRgbImage, float v = gainMapF[c][(size_t)j * width + i]; v = AVIF_CLAMP(v, gainMapMinLog2[c], gainMapMaxLog2[c]); v = powf((v - gainMapMinLog2[c]) / range, gainMapGamma); - gainMapF[c][(size_t)j * width + i] = AVIF_CLAMP(v, 0.0f, 1.0f); + // NaN-safe clamp: powf() can return NaN for degenerate gamma + // values, and AVIF_CLAMP passes NaN through because IEEE 754 + // comparisons with NaN return false. fmaxf/fminf return the + // non-NaN argument per C99 §7.12.12. + gainMapF[c][(size_t)j * width + i] = fminf(1.0f, fmaxf(0.0f, v)); } } } From 1a816c3ad09c70ddd269270ea165daba60da532e Mon Sep 17 00:00:00 2001 From: Anthony Hurtado Date: Mon, 4 May 2026 11:32:41 -0500 Subject: [PATCH 2/5] Add CHANGELOG entry for NaN-safe clamp fix in gain map Co-Authored-By: Claude Opus 4.6 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe4099b395..79b3dbe45c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ The changes are relative to the previous release, unless the baseline is specifi * Update LocalAvm.cmake: research-v15.0.0-rc1 * Update svt.cmd/svt.sh/LocalSvt.cmake: v4.1.0 * Fix decoding layered image with multiple scaled alpha layers +* Fix NaN bypass of AVIF_CLAMP in gain map tone mapping (use fminf/fmaxf) ## [1.4.1] - 2026-03-20 From 1d659320a5de79dd83d81e388702382723023c02 Mon Sep 17 00:00:00 2001 From: Anthony Hurtado Date: Tue, 5 May 2026 12:29:27 -0500 Subject: [PATCH 3/5] Add standalone reproducer for NaN crash in gain map tone mapping Demonstrates the apply-path NaN from 0 * Inf: a black base pixel with baseOffset=0 and gainMapMax=1000 causes exp2f(1000) to overflow to +Inf, and (0.0 + 0.0) * +Inf = NaN per IEEE 754. Co-Authored-By: Claude Opus 4.6 --- tests/reproduce_gainmap_nan.c | 143 ++++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 tests/reproduce_gainmap_nan.c diff --git a/tests/reproduce_gainmap_nan.c b/tests/reproduce_gainmap_nan.c new file mode 100644 index 0000000000..2a0da24901 --- /dev/null +++ b/tests/reproduce_gainmap_nan.c @@ -0,0 +1,143 @@ +/* + * reproduce_gainmap_nan.c — Demonstrates NaN crash in gain map tone mapping. + * + * Without the fminf/fmaxf fix, this triggers an assertion failure + * in avifSetRGBAPixel() (debug builds) or undefined float-to-int + * conversion (release builds). + * + * The NaN arises from IEEE 754 indeterminate form 0 * Inf: + * - baseOffset = 0, so (baseLinear + baseOffset) = 0 for a black pixel + * - gainMapMax = 1000, so exp2f(lerp(0, 1000, 1.0) * 1.0) = +Inf + * - 0.0f * +Inf = NaN + * - AVIF_CLAMP(NaN, 0, 1) = NaN (ternary comparisons with NaN are false) + * + * Build (from libavif root): + * + * mkdir build && cd build + * cmake .. -DAVIF_CODEC_AOM=LOCAL -DAVIF_LIBYUV=LOCAL \ + * -DCMAKE_BUILD_TYPE=Debug -DBUILD_SHARED_LIBS=OFF + * cmake --build . --target avif -j$(nproc) + * cd .. + * + * cc -g -O1 -I include tests/reproduce_gainmap_nan.c \ + * build/libavif.a build/_deps/libyuv-build/libyuv.a \ + * build/_deps/aom-build/libaom.a -lstdc++ -lm -lpthread \ + * -o reproduce_gainmap_nan + * + * ./reproduce_gainmap_nan + * + * Expected without fix: assertion failure in avifSetRGBAPixel + * Expected with fix: "PASS: no crash" + */ + +#include +#include +#include +#include + +int main(void) { + /* 2x2 black base image (sRGB, BT.709) */ + avifImage *base = avifImageCreate(2, 2, 8, AVIF_PIXEL_FORMAT_YUV444); + if (!base) { + fprintf(stderr, "Failed to create base image\n"); + return 1; + } + base->colorPrimaries = AVIF_COLOR_PRIMARIES_SRGB; + base->transferCharacteristics = AVIF_TRANSFER_CHARACTERISTICS_SRGB; + base->matrixCoefficients = AVIF_MATRIX_COEFFICIENTS_BT709; + base->yuvRange = AVIF_RANGE_FULL; + if (avifImageAllocatePlanes(base, AVIF_PLANES_YUV) != AVIF_RESULT_OK) { + fprintf(stderr, "Failed to allocate base planes\n"); + avifImageDestroy(base); + return 1; + } + /* Y=0 (black), U=128/V=128 (neutral chroma) */ + memset(base->yuvPlanes[0], 0, (size_t)base->yuvRowBytes[0] * 2); + memset(base->yuvPlanes[1], 128, (size_t)base->yuvRowBytes[1] * 2); + memset(base->yuvPlanes[2], 128, (size_t)base->yuvRowBytes[2] * 2); + + /* 2x2 gain map image — all pixels at maximum (255 -> 1.0 normalized) */ + avifGainMap *gainMap = avifGainMapCreate(); + if (!gainMap) { + fprintf(stderr, "Failed to create gain map\n"); + avifImageDestroy(base); + return 1; + } + gainMap->image = avifImageCreate(2, 2, 8, AVIF_PIXEL_FORMAT_YUV444); + if (!gainMap->image) { + fprintf(stderr, "Failed to create gain map image\n"); + avifGainMapDestroy(gainMap); + avifImageDestroy(base); + return 1; + } + gainMap->image->yuvRange = AVIF_RANGE_FULL; + gainMap->image->matrixCoefficients = AVIF_MATRIX_COEFFICIENTS_IDENTITY; + if (avifImageAllocatePlanes(gainMap->image, AVIF_PLANES_YUV) != AVIF_RESULT_OK) { + fprintf(stderr, "Failed to allocate gain map planes\n"); + avifGainMapDestroy(gainMap); + avifImageDestroy(base); + return 1; + } + memset(gainMap->image->yuvPlanes[0], 255, (size_t)gainMap->image->yuvRowBytes[0] * 2); + memset(gainMap->image->yuvPlanes[1], 255, (size_t)gainMap->image->yuvRowBytes[1] * 2); + memset(gainMap->image->yuvPlanes[2], 255, (size_t)gainMap->image->yuvRowBytes[2] * 2); + + /* + * Gain map metadata crafted to trigger NaN: + * gainMapMin = 0 -> lerp lower bound + * gainMapMax = 1000 -> lerp upper bound + * gamma = 1 -> no gamma distortion + * baseOffset = 0 -> (baseLinear + 0) = 0 for black pixels + * altOffset = 0 + * + * The math: lerp(0, 1000, powf(1.0, 1.0)) = 1000 + * exp2f(1000 * weight) = +Inf + * (0.0 + 0.0) * +Inf = NaN (IEEE 754) + */ + for (int c = 0; c < 3; ++c) { + gainMap->gainMapMin[c] = (avifSignedFraction){ 0, 1 }; + gainMap->gainMapMax[c] = (avifSignedFraction){ 1000, 1 }; + gainMap->gainMapGamma[c] = (avifUnsignedFraction){ 1, 1 }; + gainMap->baseOffset[c] = (avifSignedFraction){ 0, 1 }; + gainMap->alternateOffset[c] = (avifSignedFraction){ 0, 1 }; + } + gainMap->baseHdrHeadroom = (avifUnsignedFraction){ 0, 1 }; + gainMap->alternateHdrHeadroom = (avifUnsignedFraction){ 6, 1 }; + gainMap->useBaseColorSpace = 1; + gainMap->altColorPrimaries = AVIF_COLOR_PRIMARIES_SRGB; + gainMap->altTransferCharacteristics = AVIF_TRANSFER_CHARACTERISTICS_SRGB; + gainMap->altMatrixCoefficients = AVIF_MATRIX_COEFFICIENTS_BT709; + gainMap->altYUVRange = AVIF_RANGE_FULL; + gainMap->altDepth = 8; + gainMap->altPlaneCount = 3; + + /* Output tone-mapped image — set format/depth only. + * avifRGBImageApplyGainMap sets width/height and allocates pixels internally. */ + avifRGBImage toneMap; + memset(&toneMap, 0, sizeof(toneMap)); + toneMap.depth = 8; + toneMap.format = AVIF_RGB_FORMAT_RGBA; + + avifContentLightLevelInformationBox clli; + memset(&clli, 0, sizeof(clli)); + avifDiagnostics diag; + avifDiagnosticsClearError(&diag); + + /* Apply with full HDR headroom (weight = 1.0) */ + avifResult result = avifImageApplyGainMap(base, gainMap, 6.0f, + AVIF_COLOR_PRIMARIES_SRGB, + AVIF_TRANSFER_CHARACTERISTICS_SRGB, + &toneMap, &clli, &diag); + + if (result == AVIF_RESULT_OK) { + printf("Result: OK\n"); + } else { + printf("Result: %s (%s)\n", avifResultToString(result), diag.error); + } + printf("PASS: no crash\n"); + + avifRGBImageFreePixels(&toneMap); + avifGainMapDestroy(gainMap); + avifImageDestroy(base); + return 0; +} From 608ca50e7f454a82c9a6865312ab8c9d5d587f0d Mon Sep 17 00:00:00 2001 From: Anthony Hurtado Date: Tue, 5 May 2026 12:41:32 -0500 Subject: [PATCH 4/5] Fix reproducer: use LINEAR transfer to expose NaN crash sRGB's avifToGammaSRGB() absorbs NaN because all branch conditions (< 0, < 0.003, < 1.0) are false for NaN, causing it to fall through to "return 1.0f". This masks the bug as silent data corruption instead of triggering the assertion in avifSetRGBAPixel(). Switch output transfer to LINEAR, whose transfer function uses AVIF_CLAMP (which also passes NaN through), allowing NaN to reach the assertion. Also fix build instructions (libavif_internal.a, libaom path). Co-Authored-By: Claude Opus 4.6 --- tests/reproduce_gainmap_nan.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tests/reproduce_gainmap_nan.c b/tests/reproduce_gainmap_nan.c index 2a0da24901..61e49ea62f 100644 --- a/tests/reproduce_gainmap_nan.c +++ b/tests/reproduce_gainmap_nan.c @@ -11,6 +11,13 @@ * - 0.0f * +Inf = NaN * - AVIF_CLAMP(NaN, 0, 1) = NaN (ternary comparisons with NaN are false) * + * We use LINEAR output transfer because sRGB's avifToGammaSRGB() absorbs + * NaN: all branch conditions (< 0, < 0.003, < 1.0) are false for NaN, + * so it falls through to "return 1.0f" — masking the bug as silent data + * corruption (black becomes white) instead of a crash. LINEAR's transfer + * function uses AVIF_CLAMP which passes NaN through, allowing it to reach + * the assertion in avifSetRGBAPixel(). + * * Build (from libavif root): * * mkdir build && cd build @@ -20,8 +27,8 @@ * cd .. * * cc -g -O1 -I include tests/reproduce_gainmap_nan.c \ - * build/libavif.a build/_deps/libyuv-build/libyuv.a \ - * build/_deps/aom-build/libaom.a -lstdc++ -lm -lpthread \ + * build/libavif_internal.a build/_deps/libyuv-build/libyuv.a \ + * build/_deps/libaom-build/libaom.a -lstdc++ -lm -lpthread \ * -o reproduce_gainmap_nan * * ./reproduce_gainmap_nan @@ -123,10 +130,12 @@ int main(void) { avifDiagnostics diag; avifDiagnosticsClearError(&diag); - /* Apply with full HDR headroom (weight = 1.0) */ + /* Apply with full HDR headroom (weight = 1.0). + * Use LINEAR transfer so NaN propagates through to avifSetRGBAPixel. + * (sRGB's gamma function absorbs NaN to 1.0f, hiding the crash.) */ avifResult result = avifImageApplyGainMap(base, gainMap, 6.0f, AVIF_COLOR_PRIMARIES_SRGB, - AVIF_TRANSFER_CHARACTERISTICS_SRGB, + AVIF_TRANSFER_CHARACTERISTICS_LINEAR, &toneMap, &clli, &diag); if (result == AVIF_RESULT_OK) { From 9f8480f5b9074a673833e83720b0d2f1d3f8b389 Mon Sep 17 00:00:00 2001 From: Anthony Hurtado Date: Thu, 7 May 2026 10:08:53 -0500 Subject: [PATCH 5/5] Refactor NaN handling: add avifNanSafeClamp(), detect and reject NaN in gain map apply Address review feedback: - Extract fminf/fmaxf NaN-safe clamp pattern into avifNanSafeClamp() helper - Detect NaN in avifImageApplyGainMap() and return AVIF_RESULT_INVALID_TONE_MAPPED_IMAGE with diagnostic message - Move standalone reproducer into avifgainmaptest.cc as TEST(GainMapTest, ApplyGainMapNaN) - Delete tests/reproduce_gainmap_nan.c Co-Authored-By: Claude Opus 4.6 --- src/gainmap.c | 28 +++--- tests/gtest/avifgainmaptest.cc | 89 +++++++++++++++++++ tests/reproduce_gainmap_nan.c | 152 --------------------------------- 3 files changed, 105 insertions(+), 164 deletions(-) delete mode 100644 tests/reproduce_gainmap_nan.c diff --git a/src/gainmap.c b/src/gainmap.c index f832fda769..f56484d435 100644 --- a/src/gainmap.c +++ b/src/gainmap.c @@ -7,6 +7,14 @@ #include #include +// NaN-safe clamp to [0, 1]. AVIF_CLAMP passes NaN through because IEEE 754 +// comparisons with NaN always return false. fmaxf/fminf return the non-NaN +// argument per C99 §7.12.12, so this clamps NaN to 0. +static float avifNanSafeClamp(float val) +{ + return fminf(1.0f, fmaxf(0.0f, val)); +} + static void avifGainMapSetEncodingDefaults(avifGainMap * gainMap) { for (int i = 0; i < 3; ++i) { @@ -153,11 +161,7 @@ avifResult avifRGBImageApplyGainMap(const avifRGBImage * baseImage, avifLinearRGBConvertColorSpace(basePixelRGBA, conversionCoeffs); } for (int c = 0; c < 3; ++c) { - // Use fminf/fmaxf instead of AVIF_CLAMP for NaN safety: - // AVIF_CLAMP passes NaN through because IEEE 754 comparisons - // with NaN always return false. fmaxf/fminf return the non-NaN - // argument per C99 §7.12.12. - basePixelRGBA[c] = fminf(1.0f, fmaxf(0.0f, linearToGamma(basePixelRGBA[c]))); + basePixelRGBA[c] = avifNanSafeClamp(linearToGamma(basePixelRGBA[c])); } } avifSetRGBAPixel(toneMappedImage, i, j, &toneMappedPixelRGBInfo, basePixelRGBA); @@ -274,8 +278,12 @@ avifResult avifRGBImageApplyGainMap(const avifRGBImage * baseImage, } for (int c = 0; c < 3; ++c) { - // NaN-safe clamp: fmaxf/fminf return the non-NaN argument per C99. - toneMappedPixelRGBA[c] = fminf(1.0f, fmaxf(0.0f, linearToGamma(toneMappedPixelRGBA[c]))); + if (isnan(toneMappedPixelRGBA[c])) { + avifDiagnosticsPrintf(diag, "Degenerate gain map parameters produce NaN at pixel (%u, %u)", i, j); + res = AVIF_RESULT_INVALID_TONE_MAPPED_IMAGE; + goto cleanup; + } + toneMappedPixelRGBA[c] = avifNanSafeClamp(linearToGamma(toneMappedPixelRGBA[c])); } toneMappedPixelRGBA[3] = basePixelRGBA[3]; // Alpha is unaffected by tone mapping. @@ -767,11 +775,7 @@ avifResult avifRGBImageComputeGainMap(const avifRGBImage * baseRgbImage, float v = gainMapF[c][(size_t)j * width + i]; v = AVIF_CLAMP(v, gainMapMinLog2[c], gainMapMaxLog2[c]); v = powf((v - gainMapMinLog2[c]) / range, gainMapGamma); - // NaN-safe clamp: powf() can return NaN for degenerate gamma - // values, and AVIF_CLAMP passes NaN through because IEEE 754 - // comparisons with NaN return false. fmaxf/fminf return the - // non-NaN argument per C99 §7.12.12. - gainMapF[c][(size_t)j * width + i] = fminf(1.0f, fmaxf(0.0f, v)); + gainMapF[c][(size_t)j * width + i] = avifNanSafeClamp(v); } } } diff --git a/tests/gtest/avifgainmaptest.cc b/tests/gtest/avifgainmaptest.cc index 8abd2265b0..7871bf4c32 100644 --- a/tests/gtest/avifgainmaptest.cc +++ b/tests/gtest/avifgainmaptest.cc @@ -1593,6 +1593,95 @@ TEST(FindMinMaxWithoutOutliers, Test) { } } +// Verify that applying a gain map with degenerate parameters that produce NaN +// (0 * Inf from black pixels with baseOffset=0 and large gainMapMax) returns +// AVIF_RESULT_INVALID_TONE_MAPPED_IMAGE instead of crashing or producing +// garbage output. +TEST(GainMapTest, ApplyGainMapNaN) { + // 2x2 black base image (sRGB, BT.709). + ImagePtr base(avifImageCreate(2, 2, 8, AVIF_PIXEL_FORMAT_YUV444)); + ASSERT_NE(base, nullptr); + base->colorPrimaries = AVIF_COLOR_PRIMARIES_SRGB; + base->transferCharacteristics = AVIF_TRANSFER_CHARACTERISTICS_SRGB; + base->matrixCoefficients = AVIF_MATRIX_COEFFICIENTS_BT709; + base->yuvRange = AVIF_RANGE_FULL; + ASSERT_EQ(avifImageAllocatePlanes(base.get(), AVIF_PLANES_YUV), + AVIF_RESULT_OK) + << "Failed to allocate base planes"; + // Y=0 (black), U=128/V=128 (neutral chroma). + memset(base->yuvPlanes[0], 0, (size_t)base->yuvRowBytes[0] * 2); + memset(base->yuvPlanes[1], 128, (size_t)base->yuvRowBytes[1] * 2); + memset(base->yuvPlanes[2], 128, (size_t)base->yuvRowBytes[2] * 2); + + // 2x2 gain map image — all pixels at maximum (255 -> 1.0 normalized). + GainMapPtr gainMap(avifGainMapCreate()); + ASSERT_NE(gainMap, nullptr); + gainMap->image = avifImageCreate(2, 2, 8, AVIF_PIXEL_FORMAT_YUV444); + ASSERT_NE(gainMap->image, nullptr) << "Failed to create gain map image"; + gainMap->image->yuvRange = AVIF_RANGE_FULL; + gainMap->image->matrixCoefficients = AVIF_MATRIX_COEFFICIENTS_IDENTITY; + ASSERT_EQ(avifImageAllocatePlanes(gainMap->image, AVIF_PLANES_YUV), + AVIF_RESULT_OK) + << "Failed to allocate gain map planes"; + memset(gainMap->image->yuvPlanes[0], 255, + (size_t)gainMap->image->yuvRowBytes[0] * 2); + memset(gainMap->image->yuvPlanes[1], 255, + (size_t)gainMap->image->yuvRowBytes[1] * 2); + memset(gainMap->image->yuvPlanes[2], 255, + (size_t)gainMap->image->yuvRowBytes[2] * 2); + + // Gain map metadata crafted to trigger NaN: + // gainMapMin = 0 -> lerp lower bound + // gainMapMax = 1000 -> lerp upper bound + // gamma = 1 -> no gamma distortion + // baseOffset = 0 -> (baseLinear + 0) = 0 for black pixels + // altOffset = 0 + // + // The math: lerp(0, 1000, powf(1.0, 1.0)) = 1000 + // exp2f(1000 * weight) = +Inf + // (0.0 + 0.0) * +Inf = NaN (IEEE 754) + for (int c = 0; c < 3; ++c) { + gainMap->gainMapMin[c] = {0, 1}; + gainMap->gainMapMax[c] = {1000, 1}; + gainMap->gainMapGamma[c] = {1, 1}; + gainMap->baseOffset[c] = {0, 1}; + gainMap->alternateOffset[c] = {0, 1}; + } + gainMap->baseHdrHeadroom = {0, 1}; + gainMap->alternateHdrHeadroom = {6, 1}; + gainMap->useBaseColorSpace = 1; + gainMap->altColorPrimaries = AVIF_COLOR_PRIMARIES_SRGB; + gainMap->altTransferCharacteristics = AVIF_TRANSFER_CHARACTERISTICS_SRGB; + gainMap->altMatrixCoefficients = AVIF_MATRIX_COEFFICIENTS_BT709; + gainMap->altYUVRange = AVIF_RANGE_FULL; + gainMap->altDepth = 8; + gainMap->altPlaneCount = 3; + + // Output tone-mapped image. + avifRGBImage toneMap; + memset(&toneMap, 0, sizeof(toneMap)); + toneMap.depth = 8; + toneMap.format = AVIF_RGB_FORMAT_RGBA; + + avifContentLightLevelInformationBox clli; + memset(&clli, 0, sizeof(clli)); + avifDiagnostics diag; + avifDiagnosticsClearError(&diag); + + // Apply with full HDR headroom (weight = 1.0). + // Use LINEAR transfer so NaN propagates through to the clamp check. + // (sRGB's gamma function absorbs NaN to 1.0f, hiding the issue.) + avifResult result = avifImageApplyGainMap(base.get(), gainMap.get(), 6.0f, + AVIF_COLOR_PRIMARIES_SRGB, + AVIF_TRANSFER_CHARACTERISTICS_LINEAR, + &toneMap, &clli, &diag); + + EXPECT_EQ(result, AVIF_RESULT_INVALID_TONE_MAPPED_IMAGE) + << avifResultToString(result) << " (" << diag.error << ")"; + + avifRGBImageFreePixels(&toneMap); +} + } // namespace } // namespace avif diff --git a/tests/reproduce_gainmap_nan.c b/tests/reproduce_gainmap_nan.c deleted file mode 100644 index 61e49ea62f..0000000000 --- a/tests/reproduce_gainmap_nan.c +++ /dev/null @@ -1,152 +0,0 @@ -/* - * reproduce_gainmap_nan.c — Demonstrates NaN crash in gain map tone mapping. - * - * Without the fminf/fmaxf fix, this triggers an assertion failure - * in avifSetRGBAPixel() (debug builds) or undefined float-to-int - * conversion (release builds). - * - * The NaN arises from IEEE 754 indeterminate form 0 * Inf: - * - baseOffset = 0, so (baseLinear + baseOffset) = 0 for a black pixel - * - gainMapMax = 1000, so exp2f(lerp(0, 1000, 1.0) * 1.0) = +Inf - * - 0.0f * +Inf = NaN - * - AVIF_CLAMP(NaN, 0, 1) = NaN (ternary comparisons with NaN are false) - * - * We use LINEAR output transfer because sRGB's avifToGammaSRGB() absorbs - * NaN: all branch conditions (< 0, < 0.003, < 1.0) are false for NaN, - * so it falls through to "return 1.0f" — masking the bug as silent data - * corruption (black becomes white) instead of a crash. LINEAR's transfer - * function uses AVIF_CLAMP which passes NaN through, allowing it to reach - * the assertion in avifSetRGBAPixel(). - * - * Build (from libavif root): - * - * mkdir build && cd build - * cmake .. -DAVIF_CODEC_AOM=LOCAL -DAVIF_LIBYUV=LOCAL \ - * -DCMAKE_BUILD_TYPE=Debug -DBUILD_SHARED_LIBS=OFF - * cmake --build . --target avif -j$(nproc) - * cd .. - * - * cc -g -O1 -I include tests/reproduce_gainmap_nan.c \ - * build/libavif_internal.a build/_deps/libyuv-build/libyuv.a \ - * build/_deps/libaom-build/libaom.a -lstdc++ -lm -lpthread \ - * -o reproduce_gainmap_nan - * - * ./reproduce_gainmap_nan - * - * Expected without fix: assertion failure in avifSetRGBAPixel - * Expected with fix: "PASS: no crash" - */ - -#include -#include -#include -#include - -int main(void) { - /* 2x2 black base image (sRGB, BT.709) */ - avifImage *base = avifImageCreate(2, 2, 8, AVIF_PIXEL_FORMAT_YUV444); - if (!base) { - fprintf(stderr, "Failed to create base image\n"); - return 1; - } - base->colorPrimaries = AVIF_COLOR_PRIMARIES_SRGB; - base->transferCharacteristics = AVIF_TRANSFER_CHARACTERISTICS_SRGB; - base->matrixCoefficients = AVIF_MATRIX_COEFFICIENTS_BT709; - base->yuvRange = AVIF_RANGE_FULL; - if (avifImageAllocatePlanes(base, AVIF_PLANES_YUV) != AVIF_RESULT_OK) { - fprintf(stderr, "Failed to allocate base planes\n"); - avifImageDestroy(base); - return 1; - } - /* Y=0 (black), U=128/V=128 (neutral chroma) */ - memset(base->yuvPlanes[0], 0, (size_t)base->yuvRowBytes[0] * 2); - memset(base->yuvPlanes[1], 128, (size_t)base->yuvRowBytes[1] * 2); - memset(base->yuvPlanes[2], 128, (size_t)base->yuvRowBytes[2] * 2); - - /* 2x2 gain map image — all pixels at maximum (255 -> 1.0 normalized) */ - avifGainMap *gainMap = avifGainMapCreate(); - if (!gainMap) { - fprintf(stderr, "Failed to create gain map\n"); - avifImageDestroy(base); - return 1; - } - gainMap->image = avifImageCreate(2, 2, 8, AVIF_PIXEL_FORMAT_YUV444); - if (!gainMap->image) { - fprintf(stderr, "Failed to create gain map image\n"); - avifGainMapDestroy(gainMap); - avifImageDestroy(base); - return 1; - } - gainMap->image->yuvRange = AVIF_RANGE_FULL; - gainMap->image->matrixCoefficients = AVIF_MATRIX_COEFFICIENTS_IDENTITY; - if (avifImageAllocatePlanes(gainMap->image, AVIF_PLANES_YUV) != AVIF_RESULT_OK) { - fprintf(stderr, "Failed to allocate gain map planes\n"); - avifGainMapDestroy(gainMap); - avifImageDestroy(base); - return 1; - } - memset(gainMap->image->yuvPlanes[0], 255, (size_t)gainMap->image->yuvRowBytes[0] * 2); - memset(gainMap->image->yuvPlanes[1], 255, (size_t)gainMap->image->yuvRowBytes[1] * 2); - memset(gainMap->image->yuvPlanes[2], 255, (size_t)gainMap->image->yuvRowBytes[2] * 2); - - /* - * Gain map metadata crafted to trigger NaN: - * gainMapMin = 0 -> lerp lower bound - * gainMapMax = 1000 -> lerp upper bound - * gamma = 1 -> no gamma distortion - * baseOffset = 0 -> (baseLinear + 0) = 0 for black pixels - * altOffset = 0 - * - * The math: lerp(0, 1000, powf(1.0, 1.0)) = 1000 - * exp2f(1000 * weight) = +Inf - * (0.0 + 0.0) * +Inf = NaN (IEEE 754) - */ - for (int c = 0; c < 3; ++c) { - gainMap->gainMapMin[c] = (avifSignedFraction){ 0, 1 }; - gainMap->gainMapMax[c] = (avifSignedFraction){ 1000, 1 }; - gainMap->gainMapGamma[c] = (avifUnsignedFraction){ 1, 1 }; - gainMap->baseOffset[c] = (avifSignedFraction){ 0, 1 }; - gainMap->alternateOffset[c] = (avifSignedFraction){ 0, 1 }; - } - gainMap->baseHdrHeadroom = (avifUnsignedFraction){ 0, 1 }; - gainMap->alternateHdrHeadroom = (avifUnsignedFraction){ 6, 1 }; - gainMap->useBaseColorSpace = 1; - gainMap->altColorPrimaries = AVIF_COLOR_PRIMARIES_SRGB; - gainMap->altTransferCharacteristics = AVIF_TRANSFER_CHARACTERISTICS_SRGB; - gainMap->altMatrixCoefficients = AVIF_MATRIX_COEFFICIENTS_BT709; - gainMap->altYUVRange = AVIF_RANGE_FULL; - gainMap->altDepth = 8; - gainMap->altPlaneCount = 3; - - /* Output tone-mapped image — set format/depth only. - * avifRGBImageApplyGainMap sets width/height and allocates pixels internally. */ - avifRGBImage toneMap; - memset(&toneMap, 0, sizeof(toneMap)); - toneMap.depth = 8; - toneMap.format = AVIF_RGB_FORMAT_RGBA; - - avifContentLightLevelInformationBox clli; - memset(&clli, 0, sizeof(clli)); - avifDiagnostics diag; - avifDiagnosticsClearError(&diag); - - /* Apply with full HDR headroom (weight = 1.0). - * Use LINEAR transfer so NaN propagates through to avifSetRGBAPixel. - * (sRGB's gamma function absorbs NaN to 1.0f, hiding the crash.) */ - avifResult result = avifImageApplyGainMap(base, gainMap, 6.0f, - AVIF_COLOR_PRIMARIES_SRGB, - AVIF_TRANSFER_CHARACTERISTICS_LINEAR, - &toneMap, &clli, &diag); - - if (result == AVIF_RESULT_OK) { - printf("Result: OK\n"); - } else { - printf("Result: %s (%s)\n", avifResultToString(result), diag.error); - } - printf("PASS: no crash\n"); - - avifRGBImageFreePixels(&toneMap); - avifGainMapDestroy(gainMap); - avifImageDestroy(base); - return 0; -}