Update of chardet to 4.0.0

Author: stamparm

data/txt/sha256sums.txt

@@ -188,7 +188,7 @@ c03dc585f89642cfd81b087ac2723e3e1bb3bfa8c60e6f5fe58ef3b0113ebfe6  lib/core/data.
 48797d6c34dd9bb8a53f7f3794c85f4288d82a9a1d6be7fcf317d388cb20d4b3  lib/core/replication.py
 0b8c38a01bb01f843d94a6c5f2075ee47520d0c4aa799cecea9c3e2c5a4a23a6  lib/core/revision.py
 888daba83fd4a34e9503fe21f01fef4cc730e5cde871b1d40e15d4cbc847d56c  lib/core/session.py
-b035d4789fe95557807a3e0b3e16822a31dba2f46cb952cd46c9bce270fa7368  lib/core/settings.py
+6e5a916581e1e7032cdd451737be69af0bbc42cfc7e46d44a43df0ee272c361d  lib/core/settings.py
 cd5a66deee8963ba8e7e9af3dd36eb5e8127d4d68698811c29e789655f507f82  lib/core/shell.py
 bcb5d8090d5e3e0ef2a586ba09ba80eef0c6d51feb0f611ed25299fbb254f725  lib/core/subprocessng.py
 70ea3768f1b3062b22d20644df41c86238157ec80dd43da40545c620714273c6  lib/core/target.py
@@ -208,7 +208,7 @@ c5b258be7485089fac9d9cd179960e774fbd85e62836dc67cce76cc028bb6aeb  lib/parse/hand
 d2e771cdacef25ee3fdc0e0355b92e7cd1b68f5edc2756ffc19f75d183ba2c73  lib/parse/payloads.py
 455ab0ec63e55cd56ce4a884b85bdc089223155008cab0f3696da5a33118f95b  lib/parse/sitemap.py
 1be3da334411657461421b8a26a0f2ff28e1af1e28f1e963c6c92768f9b0847c  lib/request/basicauthhandler.py
-132abf563aeaaf0108b7e3932cfcc9680c8f445e992de4ee71ceed1ddf60bc29  lib/request/basic.py
+08e31008a089f8ff058afaccc06f42d6ca103408f9167cfb1831428e608374a4  lib/request/basic.py
 bc61bc944b81a7670884f82231033a6ac703324b34b071c9834886a92e249d0e  lib/request/chunkedhandler.py
 09c2d8786fb5280f5f14a7b4345ecb2e7c2ca836ee06a6cf9b51770df923d94c  lib/request/comparison.py
 9236db2abad1b1d368a3c5a5beb655055fd2445faba57a4172db264b06105bd4  lib/request/connect.py
@@ -573,10 +573,10 @@ f862301288d2ba2f913860bb901cd5197e72c0461e3330164f90375f713b8199  thirdparty/bot
 0ffccae46cb3a15b117acd0790b2738a5b45417d1b2822ceac57bdff10ef3bff  thirdparty/chardet/big5freq.py
 901c476dd7ad0693deef1ae56fe7bdf748a8b7ae20fde1922dddf6941eff8773  thirdparty/chardet/big5prober.py
 df0a164bad8aac6a282b2ab3e334129e315b2696ba57b834d9d68089b4f0725f  thirdparty/chardet/chardistribution.py
-e9b0eef1822246e49c5f871af4881bd14ebd4c0d8f1975c37a3e82738ffd90ee  thirdparty/chardet/charsetgroupprober.py
+1992d17873fa151467e3786f48ea060b161a984acacf2a7a460390c55782de48  thirdparty/chardet/charsetgroupprober.py
 2929b0244ae3ca9ca3d1b459982e45e5e33b73c61080b6088d95e29ed64db2d8  thirdparty/chardet/charsetprober.py
 558a7fe9ccb2922e6c1e05c34999d75b8ab5a1e94773772ef40c904d7eeeba0f  thirdparty/chardet/codingstatemachine.py
-3ca4f31e449bb5b1c3a92f4fcae8cc6d7ef8ab56bc98ca5e4130d5b10859311c  thirdparty/chardet/compat.py
+e34cebeb0202670927c72b8b18670838fcaf7bc0d379b0426dbbedb6f9e6a794  thirdparty/chardet/compat.py
 4d9e37e105fccf306c9d4bcbffcc26e004154d9d9992a10440bfe5370f5ff68c  thirdparty/chardet/cp949prober.py
 0229b075bf5ab357492996853541f63a158854155de9990927f58ae6c358f1c5  thirdparty/chardet/enums.py
 924caa560d58c370c8380309d9b765c9081415086e1c05bc7541ac913a0d5927  thirdparty/chardet/escprober.py
@@ -589,26 +589,25 @@ d77a7a10fe3245ac6a9cfe221edc47389e91db3c47ab5fe6f214d18f3559f797  thirdparty/cha
 257f25b3078a2e69c2c2693c507110b0b824affacffe411bbe2bc2e2a3ceae57  thirdparty/chardet/gb2312freq.py
 806bc85a2f568438c4fb14171ef348cab9cbbc46cc01883251267ae4751fca5c  thirdparty/chardet/gb2312prober.py
 737499f8aee1bf2cc663a251019c4983027fb144bd93459892f318d34601605a  thirdparty/chardet/hebrewprober.py
-62c3f9c1096c1c9d9ab85d516497f2a624ab080eff6d08919b7112fcd23bebe6  thirdparty/chardet/__init__.py
+99665a5a6bd9921c1f044013f4ed58ea74537cace14fb1478504d302e8dba940  thirdparty/chardet/__init__.py
 be9989bf606ed09f209cc5513c730579f4d1be8fe16b59abc8b8a0f0207080e8  thirdparty/chardet/jisfreq.py
 3d894da915104fc2ccddc4f91661c63f48a2b1c1654d6103f763002ef06e9e0a  thirdparty/chardet/jpcntx.py
-d47a904bd3dbb678f5c508318ad24cbf0f17ea42abe4ea1c90d09959f110acf1  thirdparty/chardet/langbulgarianmodel.py
-2ce0da8efb1eb47f3bc980c340a0360942d7507f3bb48db6ddd85f8e1f59c7d7  thirdparty/chardet/langcyrillicmodel.py
-f18016edb53c6304896a9d2420949b3ccc35044ab31a35b3a9ca9fd168142800  thirdparty/chardet/langgreekmodel.py
-2529ea984e44eb6b432d33d3bcba50b20e6038c3b83db75646f57b02f91cd070  thirdparty/chardet/langhebrewmodel.py
-4616a96121b997465a3be555e056a7e6c5b4591190aa1c0133ad72c77cb1c8e0  thirdparty/chardet/langhungarianmodel.py
-f25d35ef71aefd6e86f26c6640e4c417896cd98744ec5c567f74244b11065c94  thirdparty/chardet/langthaimodel.py
-5b6d9e44d26ca88eae5807f05d22955969c27ab62aac8f1d6504e6fccd254459  thirdparty/chardet/langturkishmodel.py
+afab6f3ad3bc16a8676d6041e55e1ccdc9757d6338a41f651a259053ef20becc  thirdparty/chardet/langbulgarianmodel.py
+d5c32edb05203c1f1b43645b5634782cdc020844e043e0f0a34120dbfb81d75b  thirdparty/chardet/langgreekmodel.py
+bab3262471c85ed0b069602acb5cc463fe129b0c0dcedef7d1b0ceb635f3463b  thirdparty/chardet/langhebrewmodel.py
+383022b2fa827deb3c07815ec8cfcf83d1d8dd90e7132682893e01c72ce873ac  thirdparty/chardet/langhungarianmodel.py
+a69a0a3862fd38f763f40e025321bc478f336e75edf4c37559778261ea5aeac7  thirdparty/chardet/langthaimodel.py
+1f795d89c23fae196fd2bdd5169556b542fa5f7d16cb9f7abfbfd81f3dac11d8  thirdparty/chardet/langturkishmodel.py
 4b6228391845937f451053a54855ad815c9b4623fa87b0652e574755c94d914f  thirdparty/chardet/latin1prober.py
 011f797851fdbeea927ef2d064df8be628de6b6e4d3810a85eac3cb393bdc4b4  thirdparty/chardet/mbcharsetprober.py
 87a4d19e762ad8ec46d56743e493b2c5c755a67edd1b4abebc1f275abe666e1e  thirdparty/chardet/mbcsgroupprober.py
 498df6c15205dc7cdc8d8dc1684b29cbd99eb5b3522b120807444a3e7eed8e92  thirdparty/chardet/mbcssm.py
-2c34a90a5743085958c149069300f6a05c4b94f5885974f4f5a907ff63e263be  thirdparty/chardet/sbcharsetprober.py
-d48a6b70207f935a9f9a7c460ba3016f110b94aa83dec716e92f1823075ec970  thirdparty/chardet/sbcsgroupprober.py
+9e6c8ccaec731bcec337a2b7464d8c53324b30b47af4cad6a5d9c7ccec155304  thirdparty/chardet/sbcharsetprober.py
+86a79f42e5e6885c83040ace8ee8c7ea177a5855e5383d64582b310e18f1e557  thirdparty/chardet/sbcsgroupprober.py
 208b7e9598f4589a8ae2b9946732993f8189944f0a504b45615b98f7a7a4e4c4  thirdparty/chardet/sjisprober.py
-a8bd35ef8952644e38d9e076d679e4b53f7f55c0327b4ee5685594794ae3b6d6  thirdparty/chardet/universaldetector.py
+0e96535c25f49d41d7c6443db2be06671181fe1bde67a856b77b8cf7872058ab  thirdparty/chardet/universaldetector.py
 21d0fcbf7cd63ac07c38b8b23e2fb2fdfab08a9445c55f4d73578a04b4ae204c  thirdparty/chardet/utf8prober.py
-b29dc1d3c9ab0d707ea5fdcaf5fa89ff37831ce08b0bc46b9e04320c56a9ffb8  thirdparty/chardet/version.py
+0380882c501df0c4551b51e85cfa78e622bd44b956c95ef76b512dc04f13be7f  thirdparty/chardet/version.py
 1c1ee8a91eb20f8038ace6611610673243d0f71e2b7566111698462182c7efdd  thirdparty/clientform/clientform.py
 e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855  thirdparty/clientform/__init__.py
 4e8a7811e12e69074159db5e28c11c18e4de29e175f50f96a3febf0a3e643b34  thirdparty/colorama/ansi.py

lib/core/settings.py

@@ -20,7 +20,7 @@
 from thirdparty import six
 
 # sqlmap version (<major>.<minor>.<month>.<monthly commit>)
-VERSION = "1.10.6.56"
+VERSION = "1.10.6.57"
 TYPE = "dev" if VERSION.count('.') > 2 and VERSION.split('.')[-1] != '0' else "stable"
 TYPE_COLORS = {"dev": 33, "stable": 90, "pip": 34}
 VERSION_STRING = "sqlmap/%s#%s" % ('.'.join(VERSION.split('.')[:-1]) if VERSION.count('.') > 2 and VERSION.split('.')[-1] == '0' else VERSION, TYPE)

lib/request/basic.py

@@ -258,10 +258,11 @@ def getHeuristicCharEncoding(page):
 
     >>> getHeuristicCharEncoding(b"<html></html>")
     'ascii'
+    >>> getHeuristicCharEncoding(b'<!DOCTYPE html><html><head><meta charset="windows-1251"><title>\\xd2\\xe5\\xf1\\xf2</title></head><body>\\xc2 \\xf1\\xee\\xee\\xf2\\xe2\\xe5\\xf2\\xf1\\xf2\\xe2\\xe8\\xe8 \\xf1 \\xef\\xf0\\xe8\\xed\\xf6\\xe8\\xef\\xe0\\xec\\xe8 \\xf0\\xe0\\xe1\\xee\\xf2\\xfb \\xf3\\xf2\\xe8\\xeb\\xe8\\xf2\\xfb \\xe0\\xe2\\xf2\\xee\\xec\\xe0\\xf2\\xe8\\xf7\\xe5\\xf1\\xea\\xee\\xe3\\xee \\xee\\xef\\xf0\\xe5\\xe4\\xe5\\xeb\\xe5\\xed\\xe8\\xff \\xea\\xee\\xe4\\xe8\\xf0\\xee\\xe2\\xea\\xe8, \\xed\\xe0\\xec \\xf2\\xf0\\xe5\\xe1\\xf3\\xe5\\xf2\\xf1\\xff \\xef\\xf0\\xe5\\xe4\\xee\\xf1\\xf2\\xe0\\xe2\\xe8\\xf2\\xfc \\xe7\\xed\\xe0\\xf7\\xe8\\xf2\\xe5\\xeb\\xfc\\xed\\xee \\xe1\\xee\\xeb\\xe5\\xe5 \\xe4\\xeb\\xe8\\xed\\xed\\xfb\\xe9 \\xf4\\xf0\\xe0\\xe3\\xec\\xe5\\xed\\xf2 \\xf2\\xe5\\xea\\xf1\\xf2\\xe0 \\xed\\xe0 \\xf0\\xf3\\xf1\\xf1\\xea\\xee\\xec \\xff\\xe7\\xfb\\xea\\xe5. \\xdd\\xf2\\xee \\xed\\xe5\\xee\\xb1\\xf5\\xee\\xe4\\xe8\\xec\\xee \\xe4\\xeb\\xff \\xf2\\xee\\xe3\\xee, \\xf7\\xf2\\xee\\xf1\\xfb \\xf1\\xf2\\xe0\\xf2\\xe8\\xf1\\xf2\\xe8\\xf7\\xe5\\xf1\\xea\\xe8\\xe9 \\xe0\\xed\\xe0\\xeb\\xe8\\xe7\\xe0\\xf2\\xee\\xf0 \\xf7\\xe0\\xf1\\xf2\\xee\\xf2\\xed\\xee\\xf1\\xf2\\xe8 \\xf1\\xe8\\xec\\xe2\\xee\\xeb\\xee\\xe2 \\xe8 \\xe4\\xe2\\xf3\\xf5\\xe1\\xf3\\xea\\xe2\\xe5\\xed\\xed\\xfb\\xf5 \\xf1\\xee\\xf7\\xe5\\xf2\\xe0\\xed\\xe8\\xb9 \\xf1\\xec\\xee\\xe3 \\xf1 \\xe2\\xfb\\xf1\\xee\\xea\\xee\\xb9 \\xf1\\xf2\\xe5\\xef\\xe5\\xed\\xfc\\xf2 \\xf3\\xe2\\xe5\\xf0\\xe5\\xed\\xed\\xee\\xf1\\xf2\\xe8 \\xe7\\xe0\\xf4\\xe8\\xea\\xf1\\xe8\\xf0\\xee\\xe2\\xe0\\xf2\\xfc \\xe8\\xec\\xe5\\xed\\xed\\xee \\xf1\\xf2\\xe0\\xed\\xe4\\xe0\\xf0\\xf2 Windows-1251, \\xe0 \\xed\\xe5 MacCyrillic \\xe8\\xeb\\xe8 ISO-8859-5. \\xd0\\xf3\\xf1\\xf1\\xea\\xe8\\xb9 \\xff\\xe7\\xfb\\xea \\xee\\xe1\\xbb\\xe0\\xe4\\xe0\\xe5\\xf2 \\xf3\\xed\\xe8\\xea\\xe0\\xeb\\xfc\\xed\\xfb\\xec \\xf0\\xe0\\xf1\\xef\\xf0\\xe5\\xe4\\xe5\\xeb\\xe5\\xed\\xe8\\xe5\\xec \\xe3\\xeb\\xe0\\xf1\\xed\\xfb\\xf5 \\xe8 \\xf1\\xee\\xe3\\xeb\\xe0\\xf1\\xed\\xfb\\xf5 \\xe1\\xf3\\xea\\xe2, \\xf2\\xe0\\xea\\xe8\\xf5 \\xea\\xe0\\xea \\xee, \\xe5, \\xe0, \\xe8, \\xed, \\xf2, \\xea\\xee\\xf2\\xee\\xf0\\xfb\\xe5 \\xe2 \\xf0\\xe0\\xe7\\xed\\xfb\\xf5 \\xea\\xee\\xe4\\xee\\xe2\\xfb\\xf5 \\xf1\\xf2\\xf0\\xe0\\xed\\xe8\\xf6\\xe0\\xf5 \\xe7\\xe0\\xed\\xe8\\xec\\xe0\\xf3\\xf2 \\xf1\\xee\\xe2\\xe5\\xf0\\xf8\\xe5\\xed\\xed\\xee \\xf0\\xe0\\xe7\\xed\\xfb\\xe5 \\xef\\xee\\xf7\\xe8\\xf6\\xe8\\xe8 \\xe2 \\xf2\\xe0\\xe1\\xeb\\xe8\\xf6\\xe5 \\xe1\\xe0\\xb9\\xf2\\xee\\xe2. \\xca\\xee\\xe3\\xe4\\xe0 \\xf2\\xe5\\xea\\xf1\\xf2\\xe0 \\xf1\\xf2\\xe0\\xed\\xee\\xe2\\xe8\\xf2\\xf1\\xff \\xe4\\xee\\xf1\\xf2\\xe0\\xf2\\xee\\xf7\\xed\\xee \\xec\\xed\\xee\\xe3\\xee, \\xe2\\xe5\\xf0\\xee\\xff\\xf2\\xed\\xee\\xf1\\xf2\\xfc \\xee\\xf8\\xe8\\xe1\\xea\\xe8 \\xf1\\xed\\xe8\\xe6\\xe0\\xe5\\xf2\\xf1\\xff \\xef\\xf0\\xe0\\xea\\xf2\\xe8\\xf7\\xe5\\xf1\\xea\\xe8 \\xe4\\xee \\xed\\xf3\\xeb\\xff. \\xcc\\xfb \\xe4\\xee\\xe1\\xe0\\xe2\\xeb\\xff\\xe5\\xec \\xe5\\xf9\\xe5 \\xed\\xe5\\xf1\\xea\\xee\\xeb\\xfc\\xea\\xee \\xef\\xf0\\xe5\\xe4\\xeb\\xee\\xe6\\xe5\\xed\\xe8\\xb9, \\xf7\\xf2\\xee\\xf1\\xfb \\xf0\\xe0\\xf1\\xf8\\xe8\\xf0\\xe8\\xf2\\xfc \\xe2\\xfb\\xe1\\xee\\xf0\\xea\\xf3 \\xe4\\xe0\\xed\\xed\\xfb\\xf5 \\xe4\\xeb\\xff \\xea\\xee\\xf0\\xf0\\xe5\\xea\\xf2\\xed\\xee\\xe3\\xee \\xf2\\xe5\\xf1\\xf2\\xe8\\xf0\\xee\\xe2\\xe0\\xed\\xe8\\xff \\xe2\\xe0\\xf8\\xe5\\xb9 \\xe1\\xe8\\xe1\\xeb\\xe8\\xee\\xf2\\xe5\\xea\\xe8 \\xe2 \\xf1\\xf0\\xe5\\xe4\\xe5 Python.</body></html>')
+    'windows-1251'
     """
 
     key = (len(page), hash(page))
-
     retVal = kb.cache.encoding.get(key)
     if retVal is None:
         retVal = detect(page[:HEURISTIC_PAGE_SIZE_THRESHOLD])["encoding"]

thirdparty/chardet/__init__.py

@@ -16,11 +16,14 @@
 ######################### END LICENSE BLOCK #########################
 
 
-from .compat import PY2, PY3
 from .universaldetector import UniversalDetector
+from .enums import InputState
 from .version import __version__, VERSION
 
 
+__all__ = ['UniversalDetector', 'detect', 'detect_all', '__version__', 'VERSION']
+
+
 def detect(byte_str):
     """
     Detect the encoding of the given byte string.
@@ -31,9 +34,50 @@ def detect(byte_str):
     if not isinstance(byte_str, bytearray):
         if not isinstance(byte_str, bytes):
             raise TypeError('Expected object of type bytes or bytearray, got: '
-                            '{0}'.format(type(byte_str)))
+                            '{}'.format(type(byte_str)))
         else:
             byte_str = bytearray(byte_str)
     detector = UniversalDetector()
     detector.feed(byte_str)
     return detector.close()
+
+
+def detect_all(byte_str):
+    """
+    Detect all the possible encodings of the given byte string.
+
+    :param byte_str:     The byte sequence to examine.
+    :type byte_str:      ``bytes`` or ``bytearray``
+    """
+    if not isinstance(byte_str, bytearray):
+        if not isinstance(byte_str, bytes):
+            raise TypeError('Expected object of type bytes or bytearray, got: '
+                            '{}'.format(type(byte_str)))
+        else:
+            byte_str = bytearray(byte_str)
+
+    detector = UniversalDetector()
+    detector.feed(byte_str)
+    detector.close()
+
+    if detector._input_state == InputState.HIGH_BYTE:
+        results = []
+        for prober in detector._charset_probers:
+            if prober.get_confidence() > detector.MINIMUM_THRESHOLD:
+                charset_name = prober.charset_name
+                lower_charset_name = prober.charset_name.lower()
+                # Use Windows encoding name instead of ISO-8859 if we saw any
+                # extra Windows-specific bytes
+                if lower_charset_name.startswith('iso-8859'):
+                    if detector._has_win_bytes:
+                        charset_name = detector.ISO_WIN_MAP.get(lower_charset_name,
+                                                            charset_name)
+                results.append({
+                    'encoding': charset_name,
+                    'confidence': prober.get_confidence(),
+                    'language': prober.language,
+                })
+        if len(results) > 0:
+            return sorted(results, key=lambda result: -result['confidence'])
+
+    return [detector.result]

thirdparty/chardet/charsetgroupprober.py

@@ -73,6 +73,7 @@ def feed(self, byte_str):
                 continue
             if state == ProbingState.FOUND_IT:
                 self._best_guess_prober = prober
+                self._state = ProbingState.FOUND_IT
                 return self.state
             elif state == ProbingState.NOT_ME:
                 prober.active = False

thirdparty/chardet/compat.py

@@ -25,10 +25,12 @@
 if sys.version_info < (3, 0):
     PY2 = True
     PY3 = False
-    base_str = (str, unicode)
+    string_types = (str, unicode)
     text_type = unicode
+    iteritems = dict.iteritems
 else:
     PY2 = False
     PY3 = True
-    base_str = (bytes, str)
+    string_types = (bytes, str)
     text_type = str
+    iteritems = dict.items