Enable re-checking after rate limit was hit

Author: timobrembeck

CHANGELOG

@@ -1,6 +1,7 @@
 Unreleased
 
-* Ignore raw `post_save` signal (Timo Ludwig, #106).
+* Enable re-checking after rate limit was hit (Timo Ludwig, #153)
+* Ignore raw `post_save` signal (Timo Ludwig, #106)
 * Retry with fallback user agent on forbidden response (Timo Ludwig, #159)
 * Also set `redirect_to` on internal redirects (Timo Ludwig, #163)
 * Add new fields to `Url` model:

linkcheck/models.py

@@ -428,7 +428,14 @@ def check_external(self, external_recheck_interval=EXTERNAL_RECHECK_INTERVAL):
             if not request_params['verify']:
                 self.message += ', SSL certificate could not be verified'
 
-        self.last_checked = now()
+        # When a rate limit was hit or the server returned an internal error, do not update
+        # the last_checked date so the result is not cached for EXTERNAL_RECHECK_INTERVAL minutes
+        if (
+            not self.status_code or
+            self.status_code != HTTPStatus.TOO_MANY_REQUESTS and
+            self.status_code < 500
+        ):
+            self.last_checked = now()
         self.save()
         return self.status
 

linkcheck/tests/test_linkcheck.py

@@ -541,6 +541,20 @@ def test_external_check_timedout(self):
         self.assertEqual(uv.redirect_to, '')
         self.assertEqual(uv.type, 'external')
 
+    def test_external_check_rate_limit(self):
+        uv = Url(url=f"{self.live_server_url}/http/429/")
+        uv.check_url()
+        self.assertEqual(uv.status, False)
+        self.assertEqual(uv.last_checked, None)
+        self.assertEqual(uv.message, '429 Too Many Requests')
+        self.assertEqual(uv.anchor_message, '')
+        self.assertEqual(uv.ssl_status, None)
+        self.assertEqual(uv.ssl_message, 'Insecure link')
+        self.assertEqual(uv.get_status_code_display(), '429 Too Many Requests')
+        self.assertEqual(uv.get_redirect_status_code_display(), None)
+        self.assertEqual(uv.redirect_to, '')
+        self.assertEqual(uv.type, 'external')
+
     def test_working_external_anchor(self):
         uv = Url(url=f"{self.live_server_url}/http/anchor/#anchor")
         uv.check_url()