Fix GH-20660: imageellipse()/imagefilledellipse() overflow

Author: devnexen

ext/gd/libgd/gd.c

@@ -1723,6 +1723,13 @@ void gdImageEllipse(gdImagePtr im, int mx, int my, int w, int h, int c)
 	b=h>>1;
 	gdImageSetPixel(im,mx+a, my, c);
 	gdImageSetPixel(im,mx-a, my, c);
+	if (a > (INT64_MAX >> 1) / b / b) {
+		return;
+	}
+
+	if (b > (INT64_MAX >> 1) / a / a) {
+		return;
+	}
 	mx1 = mx-a;my1 = my;
 	mx2 = mx+a;my2 = my;
 
@@ -1762,6 +1769,13 @@ void gdImageFilledEllipse (gdImagePtr im, int mx, int my, int w, int h, int c)
 
 	a=w>>1;
 	b=h>>1;
+	if (a > (INT64_MAX >> 1) / b / b) {
+		return;
+	}
+
+	if (b > (INT64_MAX >> 1) / a / a) {
+		return;
+	}
 
 	for (x = mx-a; x <= mx+a; x++) {
 		gdImageSetPixel(im, x, my, c);

ext/gd/tests/gh20660-2.phpt

@@ -0,0 +1,15 @@
+--TEST--
+GH-20660 (imagefilleellipse() overflow)
+--EXTENSIONS--
+gd
+--SKIPIF--
+<?php if (getenv("SKIP_SLOW_TESTS")) die("skip slow test"); ?>
+--FILE--
+<?php
+$im = imagecreate(8, 8);
+imageellipse($im, 255, 255, "1234567890", 64, 0);
+imagefilledellipse($im, 255, 255, "1234567890", 64, 0);
+echo "OK";
+?>
+--EXPECT--
+OK

ext/gd/tests/gh20660.phpt

@@ -0,0 +1,14 @@
+--TEST--
+GH-20660 (imageellipse() overflow)
+--EXTENSIONS--
+gd
+--SKIPIF--
+<?php if (getenv("SKIP_SLOW_TESTS")) die("skip slow test"); ?>
+--FILE--
+<?php
+$im = imagecreate(8, 8);
+imageellipse($im, 255, 255, "1234567890", 64, 0);
+echo "OK";
+?>
+--EXPECT--
+OK