From 89aeb70f60d3ff2afa25ed55840275023505f1db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Dubois?= Date: Tue, 5 May 2026 09:14:02 +0200 Subject: [PATCH 1/3] fix: use RateLimiter in geocode_object to respect Nominatim 1 req/s policy and catch GeocoderRateLimited so bulk imports are not aborted on 429 WEB-4423 --- CHANGES.rst | 4 +++- src/imio/smartweb/common/tests/test_utils.py | 14 ++++++++++++++ src/imio/smartweb/common/utils.py | 9 ++++++--- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 3d9011a..037800e 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -5,7 +5,9 @@ Changelog 1.2.53 (unreleased) ------------------- -- Nothing changed yet. +- WEB-4423 : Use ``RateLimiter`` in ``geocode_object`` to respect Nominatim's 1 req/s + policy and catch ``GeocoderRateLimited`` so bulk imports are not aborted on 429. + [remdub] 1.2.52 (2026-04-22) diff --git a/src/imio/smartweb/common/tests/test_utils.py b/src/imio/smartweb/common/tests/test_utils.py index ea0fe44..46050e8 100644 --- a/src/imio/smartweb/common/tests/test_utils.py +++ b/src/imio/smartweb/common/tests/test_utils.py @@ -118,6 +118,20 @@ def test_geocode_object_geocoder_unavailable(self): result = geocode_object(obj) self.assertFalse(result) + def test_geocode_object_geocoder_rate_limited(self): + obj = GeolocatedObject() + obj.street = "Test Street" + obj.number = "1" + obj.complement = "" + obj.zipcode = "12345" + obj.city = "Testville" + obj.country = "be" + with patch("geopy.geocoders.Nominatim") as mock_nominatim: + instance = mock_nominatim.return_value + instance.geocode.side_effect = geopy.exc.GeocoderRateLimited + result = geocode_object(obj) + self.assertFalse(result) + def test_get_uncroppable_scales_infos(self): folder = api.content.create( container=self.portal, diff --git a/src/imio/smartweb/common/utils.py b/src/imio/smartweb/common/utils.py index e1a6197..cf755e8 100644 --- a/src/imio/smartweb/common/utils.py +++ b/src/imio/smartweb/common/utils.py @@ -80,14 +80,17 @@ def geocode_object(obj): address = " ".join(filter(None, [street, entity, country])) if not address: return + from geopy.extra.rate_limiter import RateLimiter + geolocator = geopy.geocoders.Nominatim(user_agent="contact@imio.be", timeout=3) + geocode = RateLimiter(geolocator.geocode, min_delay_seconds=1) location = None try: - location = geolocator.geocode(address) - except geopy.exc.GeocoderUnavailable: + location = geocode(address) + except (geopy.exc.GeocoderUnavailable, geopy.exc.GeocoderRateLimited): api.portal.show_message( _( - "Error: Geolocation service is unavailable. Your content is not geocoded." + "Error: Geolocation service is unavailable or rate limited. Your content is not geocoded." ), request=getRequest(), type="warning", From 76dbaffe3cb6ae7b7f6f574ab3c93b07e3637309 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Dubois?= Date: Tue, 5 May 2026 09:50:06 +0200 Subject: [PATCH 2/3] fix: tests --- src/imio/smartweb/common/tests/test_utils.py | 19 +++++++------------ src/imio/smartweb/common/utils.py | 11 ++++++----- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/src/imio/smartweb/common/tests/test_utils.py b/src/imio/smartweb/common/tests/test_utils.py index 46050e8..b08e00c 100644 --- a/src/imio/smartweb/common/tests/test_utils.py +++ b/src/imio/smartweb/common/tests/test_utils.py @@ -85,9 +85,6 @@ def test_translate_vocabulary_term(self): ) def test_geolocation(self): - attr = {"geocode.return_value": mock.Mock(latitude=1, longitude=2)} - geopy.geocoders.Nominatim = mock.Mock(return_value=mock.Mock(**attr)) - obj = GeolocatedObject() obj.geolocation = Geolocation(0, 0) @@ -97,7 +94,9 @@ def test_geolocation(self): self.assertEqual(obj.geolocation.longitude, 0) obj.street = "My beautiful street" - geocoded = geocode_object(obj) + with patch("imio.smartweb.common.utils._geocode") as mock_geocode: + mock_geocode.return_value = mock.Mock(latitude=1, longitude=2) + geocoded = geocode_object(obj) self.assertTrue(geocoded) self.assertEqual(obj.geolocation.latitude, 1) self.assertEqual(obj.geolocation.longitude, 2) @@ -110,11 +109,8 @@ def test_geocode_object_geocoder_unavailable(self): obj.zipcode = "12345" obj.city = "Testville" obj.country = "be" - with patch("geopy.geocoders.Nominatim") as mock_nominatim, patch( - "geopy.exc.GeocoderUnavailable", new=geopy.exc.GeocoderUnavailable - ): - instance = mock_nominatim.return_value - instance.geocode.side_effect = geopy.exc.GeocoderUnavailable + with patch("imio.smartweb.common.utils._geocode") as mock_geocode: + mock_geocode.side_effect = geopy.exc.GeocoderUnavailable result = geocode_object(obj) self.assertFalse(result) @@ -126,9 +122,8 @@ def test_geocode_object_geocoder_rate_limited(self): obj.zipcode = "12345" obj.city = "Testville" obj.country = "be" - with patch("geopy.geocoders.Nominatim") as mock_nominatim: - instance = mock_nominatim.return_value - instance.geocode.side_effect = geopy.exc.GeocoderRateLimited + with patch("imio.smartweb.common.utils._geocode") as mock_geocode: + mock_geocode.side_effect = geopy.exc.GeocoderRateLimited("rate limited") result = geocode_object(obj) self.assertFalse(result) diff --git a/src/imio/smartweb/common/utils.py b/src/imio/smartweb/common/utils.py index cf755e8..e3f43de 100644 --- a/src/imio/smartweb/common/utils.py +++ b/src/imio/smartweb/common/utils.py @@ -24,6 +24,11 @@ import geopy import re import unicodedata +from geopy.extra.rate_limiter import RateLimiter + + +_geolocator = geopy.geocoders.Nominatim(user_agent="contact@imio.be", timeout=3) +_geocode = RateLimiter(_geolocator.geocode, min_delay_seconds=1, swallow_exceptions=False) def get_vocabulary(voc_name, obj=None): @@ -80,13 +85,9 @@ def geocode_object(obj): address = " ".join(filter(None, [street, entity, country])) if not address: return - from geopy.extra.rate_limiter import RateLimiter - - geolocator = geopy.geocoders.Nominatim(user_agent="contact@imio.be", timeout=3) - geocode = RateLimiter(geolocator.geocode, min_delay_seconds=1) location = None try: - location = geocode(address) + location = _geocode(address) except (geopy.exc.GeocoderUnavailable, geopy.exc.GeocoderRateLimited): api.portal.show_message( _( From f378a89c372b036f13dd9b6b7d7e2eccb3023b4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Dubois?= Date: Tue, 5 May 2026 10:04:26 +0200 Subject: [PATCH 3/3] chore: black --- src/imio/smartweb/common/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/imio/smartweb/common/utils.py b/src/imio/smartweb/common/utils.py index e3f43de..91b48a1 100644 --- a/src/imio/smartweb/common/utils.py +++ b/src/imio/smartweb/common/utils.py @@ -28,7 +28,9 @@ _geolocator = geopy.geocoders.Nominatim(user_agent="contact@imio.be", timeout=3) -_geocode = RateLimiter(_geolocator.geocode, min_delay_seconds=1, swallow_exceptions=False) +_geocode = RateLimiter( + _geolocator.geocode, min_delay_seconds=1, swallow_exceptions=False +) def get_vocabulary(voc_name, obj=None):