feat: iou calculation for elements

change_analyzer/spaces/discrete_app_action_space.py

@@ -10,6 +10,7 @@
 
 from change_analyzer.spaces.actions.app_action import AppAction
 from change_analyzer.spaces.actions.click_app_action import ClickAppAction
+from change_analyzer.utils.elements import iou
 
 
 class DiscreteAppActionSpace(Space):
@@ -62,9 +63,9 @@ def update_actionable_elements(self):
                 if self._is_el_clickable(el)
             }
             for el0, el1 in itertools.combinations(self.actionable_elements, 2):
-                if self._overlap_percentage(el0, el1) > self.OVERLAPING_PERCENTAGE:
+                if iou(el0.rect, el1.rect) > self.OVERLAPING_PERCENTAGE:
                     self.actionable_elements.remove(el1)
-                if self._overlap_percentage(el1, el0) > self.OVERLAPING_PERCENTAGE:
+                if iou(el1.rect, el0.rect) > self.OVERLAPING_PERCENTAGE:
                     self.actionable_elements.remove(el0)
         except Exception as e:
             self._logger.info(
@@ -99,12 +100,6 @@ def __eq__(self, other: Space) -> bool:
             isinstance(other, DiscreteAppActionSpace) and self.actions == other.actions
         )
 
-    def _overlap_percentage(self, el0, el1) -> float:
-        # TODO check that the elements are not overlaping
-        # self._logger.info(f"checking rects: {el0.rect}, {el1.rect}")
-        # https://stackoverflow.com/a/42874377 or more compact: https://stackoverflow.com/a/57247833
-        return 0
-
     def _is_el_clickable(self, el: WebElement) -> bool:
         try:
             return (

change_analyzer/tests/test_utils.py

@@ -1,6 +1,6 @@
 from PIL import ImageGrab
 
-from change_analyzer.utils import image_pad_resize_to
+from change_analyzer.utils import image_pad_resize_to, iou
 
 
 def test_image_pad_resize_to_wide():
@@ -22,3 +22,15 @@ def test_image_pad_resize_to_wide():
 #     im = image_pad_resize_to(ImageGrab.grab(), size)
 #     # im.show()
 #     assert im.size == size
+
+
+def test_zero_iou():
+    rect0 = {"x": 0, "y": 0, "width": 100, "height": 100}
+    rect1 = {"x": 101, "y": 101, "width": 100, "height": 100}
+    assert iou(rect0, rect1) == 0
+
+
+def test_full_iou():
+    rect0 = {"x": 0, "y": 0, "width": 100, "height": 100}
+    rect1 = {"x": 0, "y": 0, "width": 100, "height": 100}
+    assert iou(rect0, rect1) == 1

change_analyzer/utils.py

@@ -1,6 +1,7 @@
-from typing import Tuple
+from typing import Tuple, Dict
 
 from PIL import Image, ImageOps
+from shapely.geometry import Polygon
 
 
 def image_pad_resize_to(im: Image, new_dims: Tuple[int, int]) -> Image:
@@ -17,3 +18,21 @@ def image_pad_resize_to(im: Image, new_dims: Tuple[int, int]) -> Image:
 
     padding = (round(delta_w / 2), round(delta_h / 2))
     return ImageOps.expand(im, padding, "black").resize(new_dims, Image.ANTIALIAS)
+
+
+def iou(rect0: Dict[str, int], rect1: Dict[str, int]) -> float:
+    # self._logger.info(f"checking rects: {el0.rect}, {el1.rect}")
+    # https://stackoverflow.com/a/42874377 or more compact: https://stackoverflow.com/a/57247833
+    poly0 = Polygon([
+        [rect0["x"], rect0["y"]],
+        [rect0["x"] + rect0["width"], rect0["y"]],
+        [rect0["x"] + rect0["width"], rect0["y"] + rect0["height"]],
+        [rect0["x"], rect0["y"] + rect0["height"]]
+    ])
+    poly1 = Polygon([
+        [rect1["x"], rect1["y"]],
+        [rect1["x"] + rect1["width"], rect1["y"]],
+        [rect1["x"] + rect1["width"], rect1["y"] + rect1["height"]],
+        [rect1["x"], rect1["y"] + rect1["height"]]
+    ])
+    return poly0.intersection(poly1).area / poly0.union(poly1).area

setup.py

@@ -20,7 +20,8 @@ def _read_long_description():
     "jinja2",
     "pandas",
     "numpy",
-    "matplotlib"
+    "matplotlib",
+    "shapely",
 ]
 DEV_REQUIRE = [
     "black",