fix: preservation of url encoded hash signs.

* issue #91
* description: don't decode `%23` to `#` when it appears in a url
* tests-new: test_url.CanonicalizeUrlTest.test_preserve_nonfragment_hash
* tests-pass: py27, py36
* notes: adjustment to _safe_chars suggested by @Gallaecio

Author: jvanasco

tests/test_url.py

@@ -650,6 +650,21 @@ def test_canonicalize_url_idna_exceptions(self):
             "http://www.{label}.com/r%C3%A9sum%C3%A9?q=r%C3%A9sum%C3%A9".format(
                     label=u"example"*11))
 
+    def test_preserve_nonfragment_hash(self):
+        # don't decode `%23` to `#`
+        self.assertEqual(canonicalize_url("http://www.example.com/path/to/%23/foo/bar"),
+                                          "http://www.example.com/path/to/%23/foo/bar")
+        self.assertEqual(canonicalize_url("http://www.example.com/path/to/%23/foo/bar#frag"),
+                                          "http://www.example.com/path/to/%23/foo/bar")
+        self.assertEqual(canonicalize_url("http://www.example.com/path/to/%23/foo/bar#frag", keep_fragments=True),
+                                          "http://www.example.com/path/to/%23/foo/bar#frag")
+        self.assertEqual(canonicalize_url("http://www.example.com/path/to/%23/foo/bar?url=http%3A%2F%2Fwww.example.com%2Fpath%2Fto%2F%23%2Fbar%2Ffoo"),
+                                          "http://www.example.com/path/to/%23/foo/bar?url=http%3A%2F%2Fwww.example.com%2Fpath%2Fto%2F%23%2Fbar%2Ffoo")
+        self.assertEqual(canonicalize_url("http://www.example.com/path/to/%23/foo/bar?url=http%3A%2F%2Fwww.example.com%2F%2Fpath%2Fto%2F%23%2Fbar%2Ffoo#frag"),
+                                          "http://www.example.com/path/to/%23/foo/bar?url=http%3A%2F%2Fwww.example.com%2F%2Fpath%2Fto%2F%23%2Fbar%2Ffoo")
+        self.assertEqual(canonicalize_url("http://www.example.com/path/to/%23/foo/bar?url=http%3A%2F%2Fwww.example.com%2F%2Fpath%2Fto%2F%23%2Fbar%2Ffoo#frag", keep_fragments=True),
+                                          "http://www.example.com/path/to/%23/foo/bar?url=http%3A%2F%2Fwww.example.com%2F%2Fpath%2Fto%2F%23%2Fbar%2Ffoo#frag")
+
 
 class DataURITests(unittest.TestCase):
 

w3lib/url.py

@@ -34,6 +34,9 @@ def _quote_byte(error):
 
 _safe_chars = RFC3986_RESERVED + RFC3986_UNRESERVED + EXTRA_SAFE_CHARS + b'%'
 
+# see https://github.com/scrapy/w3lib/issues/91
+_safe_chars = _safe_chars.replace(b'#', b'')
+
 _ascii_tab_newline_re = re.compile(r'[\t\n\r]')  # see https://infra.spec.whatwg.org/#ascii-tab-or-newline
 
 def safe_url_string(url, encoding='utf8', path_encoding='utf8', quote_path=True):