Fix GH-19666: imageconvolution() unexpected nan filter value.

undefined behavior occurs during cast from float to int when the element
of the filter matrix is -INF. Port the libgd clamp helper which treats
non-finite accumulators as 0 (negative/NaN) or 255 (positive infinity)
instead of bailing out, matching upstream libgd 81d7f2e.

close GH-22129

Author: devnexen

NEWS

@@ -5,6 +5,8 @@ PHP                                                                        NEWS
 - GD:
   . Fixed bug GH-22121 (Double free in gdImageSetStyle() after
     overflow-triggered early return). (iliaal)
+  . Fixed bug GH-19666 (imageconvolution() unexpected nan filter value).
+    (David Carlier)
 
 - Intl:
   . Fix incorrect argument positions for uninitialized calendar arguments in

ext/gd/libgd/gd_filter.c

@@ -7,6 +7,7 @@
 #else
 # include <unistd.h>
 #endif
+#include <math.h>
 #include <stdlib.h>
 #include <time.h>
 
@@ -20,6 +21,20 @@
 /* Begin filters function */
 #define GET_PIXEL_FUNCTION(src)(src->trueColor?gdImageGetTrueColorPixel:gdImageGetPixel)
 
+static int gdClampFloatToByte(float value)
+{
+	if (!isfinite(value)) {
+		return value > 0.0f ? 255 : 0;
+	}
+	if (value > 255.0f) {
+		return 255;
+	}
+	if (value < 0.0f) {
+		return 0;
+	}
+	return (int)value;
+}
+
 #ifdef _WIN32
 # define GD_SCATTER_SEED() (unsigned int)(time(0) * GetCurrentProcessId())
 #else
@@ -361,6 +376,7 @@ int gdImageConvolution(gdImagePtr src, float filter[3][3], float filter_div, flo
 
 	for ( y=0; y<src->sy; y++) {
 		for(x=0; x<src->sx; x++) {
+			int new_ri, new_gi, new_bi;
 			new_r = new_g = new_b = 0;
 			pxl = f(srcback, x, y);
 			new_a = gdImageAlpha(srcback, pxl);
@@ -379,13 +395,13 @@ int gdImageConvolution(gdImagePtr src, float filter[3][3], float filter_div, flo
 			new_g = (new_g/filter_div)+offset;
 			new_b = (new_b/filter_div)+offset;
 
-			new_r = (new_r > 255.0f)? 255.0f : ((new_r < 0.0f)? 0.0f:new_r);
-			new_g = (new_g > 255.0f)? 255.0f : ((new_g < 0.0f)? 0.0f:new_g);
-			new_b = (new_b > 255.0f)? 255.0f : ((new_b < 0.0f)? 0.0f:new_b);
+			new_ri = gdClampFloatToByte(new_r);
+			new_gi = gdClampFloatToByte(new_g);
+			new_bi = gdClampFloatToByte(new_b);
 
-			new_pxl = gdImageColorAllocateAlpha(src, (int)new_r, (int)new_g, (int)new_b, new_a);
+			new_pxl = gdImageColorAllocateAlpha(src, new_ri, new_gi, new_bi, new_a);
 			if (new_pxl == -1) {
-				new_pxl = gdImageColorClosestAlpha(src, (int)new_r, (int)new_g, (int)new_b, new_a);
+				new_pxl = gdImageColorClosestAlpha(src, new_ri, new_gi, new_bi, new_a);
 			}
 			gdImageSetPixel (src, x, y, new_pxl);
 		}

ext/gd/tests/gh19666.phpt

@@ -0,0 +1,34 @@
+--TEST--
+GH-19666 (Unexpected nan value in imageconvolution)
+--EXTENSIONS--
+gd
+--FILE--
+<?php
+$image = imagecreatetruecolor(180, 30);
+
+$gaussian = array(
+    array(1.0, 2.0, 1.0),
+    array(2.0, 4.0, 2.0),
+    array(1.0, 2.0, -INF)
+);
+
+var_dump(imageconvolution($image, $gaussian, 16, 0));
+
+$rgba = imagecolorat($image, 0, 0);
+$colors = imagecolorsforindex($image, $rgba);
+var_dump($colors);
+
+imagedestroy($image);
+?>
+--EXPECT--
+bool(true)
+array(4) {
+  ["red"]=>
+  int(0)
+  ["green"]=>
+  int(0)
+  ["blue"]=>
+  int(0)
+  ["alpha"]=>
+  int(0)
+}