Fix encoding of utf-8 domain names

Author: timobrembeck

CHANGELOG

@@ -1,5 +1,6 @@
 Unreleased
 
+* Fix encoding of utf-8 domain names (Timo Brembeck, #190)
 * Move coverage view to management command (Timo Brembeck, #187)
   * Add new management command `linkcheck_suggest_config`
   * Delete coverage view

linkcheck/models.py

@@ -3,7 +3,7 @@
 import re
 from datetime import timedelta
 from http import HTTPStatus
-from urllib.parse import unquote
+from urllib.parse import unquote, urlparse
 
 import requests
 from django.conf import settings
@@ -230,13 +230,14 @@ def external_url(self):
         if self.internal:
             return None
 
-        # Remove URL fragment identifiers
-        external_url = self.url.rsplit('#')[0]
-        # Check that non-ascii chars are properly encoded
-        try:
-            external_url.encode('ascii')
-        except UnicodeEncodeError:
-            external_url = iri_to_uri(external_url)
+        # Encode path and query and remove anchor fragment
+        parsed = urlparse(self.url)
+        external_url = parsed._replace(
+            path=iri_to_uri(parsed.path),
+            query=iri_to_uri(parsed.query),
+            fragment=""
+        ).geturl()
+
         logger.debug('External URL: %s', external_url)
         return external_url
 

linkcheck/tests/test_linkcheck.py

@@ -484,6 +484,37 @@ def test_external_check_200_utf8_not_encoded(self):
         self.assertEqual(uv.redirect_to, '')
         self.assertEqual(uv.type, 'external')
 
+    def test_external_check_200_utf8_domain(self):
+        uv = Url(url='https://bafög.de/')
+        uv.check_url()
+        self.assertEqual(uv.status, True)
+        self.assertEqual(uv.message, '302 Found')
+        self.assertEqual(uv.get_message, 'Working temporary redirect')
+        self.assertEqual(uv.error_message, '')
+        self.assertEqual(uv.anchor_message, '')
+        self.assertEqual(uv.ssl_status, True)
+        self.assertEqual(uv.ssl_message, 'Valid SSL certificate')
+        self.assertEqual(uv.get_status_code_display(), '302 Found')
+        self.assertEqual(uv.get_redirect_status_code_display(), '200 OK')
+        self.assertEqual(uv.type, 'external')
+        # The actual redirect URL might be subject to change
+        self.assertNotEqual(uv.redirect_to, '')
+
+    def test_external_check_200_punycode_domain(self):
+        uv = Url(url='https://www.xn--jobbrse-stellenangebote-blc.de/')
+        uv.check_url()
+        self.assertEqual(uv.status, True)
+        self.assertEqual(uv.message, '200 OK')
+        self.assertEqual(uv.get_message, 'Working external link')
+        self.assertEqual(uv.error_message, '')
+        self.assertEqual(uv.anchor_message, '')
+        self.assertEqual(uv.ssl_status, True)
+        self.assertEqual(uv.ssl_message, 'Valid SSL certificate')
+        self.assertEqual(uv.get_status_code_display(), '200 OK')
+        self.assertEqual(uv.get_redirect_status_code_display(), None)
+        self.assertEqual(uv.redirect_to, '')
+        self.assertEqual(uv.type, 'external')
+
     def test_external_check_301(self):
         uv = Url(url=f"{self.live_server_url}/http/301/")
         uv.check_url()