upgrade to charade v1.0.3

diff --git a/requests/packages/charade/__init__.py b/requests/packages/charade/__init__.py
index 5d580b3..1aadf3e 100644 (file)
--- a/requests/packages/charade/__init__.py
+++ b/requests/packages/charade/__init__.py
@@ -15,10 +15,15 @@
 # 02110-1301  USA\r
 ######################### END LICENSE BLOCK #########################\r
 \r
-__version__ = "1.0.1"\r
+__version__ = "1.0.3"\r
+from sys import version_info\r
 \r
 \r
 def detect(aBuf):\r
+    if ((version_info < (3, 0) and isinstance(aBuf, unicode)) or\r
+            (version_info >= (3, 0) and not isinstance(aBuf, bytes))):\r
+        raise ValueError('Expected a bytes object, not a unicode object')\r
+\r
     from . import universaldetector\r
     u = universaldetector.UniversalDetector()\r
     u.reset()\r

diff --git a/requests/packages/charade/chardistribution.py b/requests/packages/charade/chardistribution.py
index 981bd1a..dfd3355 100644 (file)
--- a/requests/packages/charade/chardistribution.py
+++ b/requests/packages/charade/chardistribution.py
@@ -40,6 +40,7 @@ from .compat import wrap_ord
 ENOUGH_DATA_THRESHOLD = 1024\r
 SURE_YES = 0.99\r
 SURE_NO = 0.01\r
+MINIMUM_DATA_THRESHOLD = 3\r
 \r
 \r
 class CharDistributionAnalysis:\r
@@ -82,7 +83,7 @@ class CharDistributionAnalysis:
         """return confidence based on existing data"""\r
         # if we didn't receive any character in our consideration range,\r
         # return negative answer\r
-        if self._mTotalChars <= 0:\r
+        if self._mTotalChars <= 0 or self._mFreqChars <= MINIMUM_DATA_THRESHOLD:\r
             return SURE_NO\r
 \r
         if self._mTotalChars != self._mFreqChars:\r

diff --git a/requests/packages/charade/compat.py b/requests/packages/charade/compat.py
index f86c46b..d9e30ad 100644 (file)
--- a/requests/packages/charade/compat.py
+++ b/requests/packages/charade/compat.py
@@ -18,9 +18,17 @@
 # 02110-1301  USA
 ######################### END LICENSE BLOCK #########################
 
+import sys
+
+
+if sys.version_info < (3, 0):
+    base_str = (str, unicode)
+else:
+    base_str = (bytes, str)
+
 
 def wrap_ord(a):
-    if isinstance(a, str):
+    if sys.version_info < (3, 0) and isinstance(a, base_str):
         return ord(a)
-    elif isinstance(a, int):
+    else:
         return a

diff --git a/requests/packages/charade/langcyrillicmodel.py b/requests/packages/charade/langcyrillicmodel.py
index 4b69c82..15e338f 100644 (file)
--- a/requests/packages/charade/langcyrillicmodel.py
+++ b/requests/packages/charade/langcyrillicmodel.py
@@ -25,8 +25,6 @@
 # 02110-1301  USA\r
 ######################### END LICENSE BLOCK #########################\r
 \r
-from . import constants\r
-\r
 # KOI8-R language model\r
 # Character Mapping Table:\r
 KOI8R_CharToOrderMap = (\r

diff --git a/requests/packages/charade/langgreekmodel.py b/requests/packages/charade/langgreekmodel.py
index 78e9ce6..93241ce 100644 (file)
--- a/requests/packages/charade/langgreekmodel.py
+++ b/requests/packages/charade/langgreekmodel.py
@@ -25,8 +25,6 @@
 # 02110-1301  USA\r
 ######################### END LICENSE BLOCK #########################\r
 \r
-from . import constants\r
-\r
 # 255: Control characters that usually does not exist in any text\r
 # 254: Carriage/Return\r
 # 253: symbol (punctuation) that does not belong to word\r

diff --git a/requests/packages/charade/langhebrewmodel.py b/requests/packages/charade/langhebrewmodel.py
index 4c6b3ce..d871324 100644 (file)
--- a/requests/packages/charade/langhebrewmodel.py
+++ b/requests/packages/charade/langhebrewmodel.py
@@ -27,8 +27,6 @@
 # 02110-1301  USA\r
 ######################### END LICENSE BLOCK #########################\r
 \r
-from . import constants\r
-\r
 # 255: Control characters that usually does not exist in any text\r
 # 254: Carriage/Return\r
 # 253: symbol (punctuation) that does not belong to word\r

diff --git a/requests/packages/charade/langhungarianmodel.py b/requests/packages/charade/langhungarianmodel.py
index bd7f505..6f59c61 100644 (file)
--- a/requests/packages/charade/langhungarianmodel.py
+++ b/requests/packages/charade/langhungarianmodel.py
@@ -25,8 +25,6 @@
 # 02110-1301  USA\r
 ######################### END LICENSE BLOCK #########################\r
 \r
-from . import constants\r
-\r
 # 255: Control characters that usually does not exist in any text\r
 # 254: Carriage/Return\r
 # 253: symbol (punctuation) that does not belong to word\r

diff --git a/requests/packages/charade/mbcsgroupprober.py b/requests/packages/charade/mbcsgroupprober.py
index ebe93d0..2f6f5e8 100644 (file)
--- a/requests/packages/charade/mbcsgroupprober.py
+++ b/requests/packages/charade/mbcsgroupprober.py
@@ -33,6 +33,7 @@ from .sjisprober import SJISProber
 from .eucjpprober import EUCJPProber\r
 from .gb2312prober import GB2312Prober\r
 from .euckrprober import EUCKRProber\r
+from .cp949prober import CP949Prober\r
 from .big5prober import Big5Prober\r
 from .euctwprober import EUCTWProber\r
 \r
@@ -46,6 +47,7 @@ class MBCSGroupProber(CharSetGroupProber):
             EUCJPProber(),\r
             GB2312Prober(),\r
             EUCKRProber(),\r
+            CP949Prober(),\r
             Big5Prober(),\r
             EUCTWProber()\r
         ]\r

diff --git a/requests/packages/charade/mbcssm.py b/requests/packages/charade/mbcssm.py
index 3a720c9..55c02f0 100644 (file)
--- a/requests/packages/charade/mbcssm.py
+++ b/requests/packages/charade/mbcssm.py
@@ -78,6 +78,46 @@ Big5SMModel = {'classTable': BIG5_cls,
                'charLenTable': Big5CharLenTable,\r
                'name': 'Big5'}\r
 \r
+# CP949\r
+\r
+CP949_cls  = (\r
+    1,1,1,1,1,1,1,1, 1,1,1,1,1,1,0,0,  # 00 - 0f\r
+    1,1,1,1,1,1,1,1, 1,1,1,0,1,1,1,1,  # 10 - 1f\r
+    1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,  # 20 - 2f\r
+    1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,  # 30 - 3f\r
+    1,4,4,4,4,4,4,4, 4,4,4,4,4,4,4,4,  # 40 - 4f\r
+    4,4,5,5,5,5,5,5, 5,5,5,1,1,1,1,1,  # 50 - 5f\r
+    1,5,5,5,5,5,5,5, 5,5,5,5,5,5,5,5,  # 60 - 6f\r
+    5,5,5,5,5,5,5,5, 5,5,5,1,1,1,1,1,  # 70 - 7f\r
+    0,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6,  # 80 - 8f\r
+    6,6,6,6,6,6,6,6, 6,6,6,6,6,6,6,6,  # 90 - 9f\r
+    6,7,7,7,7,7,7,7, 7,7,7,7,7,8,8,8,  # a0 - af\r
+    7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7,  # b0 - bf\r
+    7,7,7,7,7,7,9,2, 2,3,2,2,2,2,2,2,  # c0 - cf\r
+    2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,  # d0 - df\r
+    2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,  # e0 - ef\r
+    2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,0,  # f0 - ff\r
+)\r
+\r
+CP949_st = (\r
+#cls=    0      1      2      3      4      5      6      7      8      9  # previous state =\r
+    eError,eStart,     3,eError,eStart,eStart,     4,     5,eError,     6, # eStart\r
+    eError,eError,eError,eError,eError,eError,eError,eError,eError,eError, # eError\r
+    eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe,eItsMe, # eItsMe\r
+    eError,eError,eStart,eStart,eError,eError,eError,eStart,eStart,eStart, # 3\r
+    eError,eError,eStart,eStart,eStart,eStart,eStart,eStart,eStart,eStart, # 4\r
+    eError,eStart,eStart,eStart,eStart,eStart,eStart,eStart,eStart,eStart, # 5\r
+    eError,eStart,eStart,eStart,eStart,eError,eError,eStart,eStart,eStart, # 6\r
+)\r
+\r
+CP949CharLenTable = (0, 1, 2, 0, 1, 1, 2, 2, 0, 2)\r
+\r
+CP949SMModel = {'classTable': CP949_cls,\r
+                'classFactor': 10,\r
+                'stateTable': CP949_st,\r
+                'charLenTable': CP949CharLenTable,\r
+                'name': 'CP949'}\r
+\r
 # EUC-JP\r
 \r
 EUCJP_cls = (\r

diff --git a/requests/packages/charade/universaldetector.py b/requests/packages/charade/universaldetector.py
index adaae72..6175bfb 100644 (file)
--- a/requests/packages/charade/universaldetector.py
+++ b/requests/packages/charade/universaldetector.py
@@ -28,6 +28,7 @@
 \r
 from . import constants\r
 import sys\r
+import codecs\r
 from .latin1prober import Latin1Prober  # windows-1252\r
 from .mbcsgroupprober import MBCSGroupProber  # multi-byte character sets\r
 from .sbcsgroupprober import SBCSGroupProber  # single-byte character sets\r
@@ -70,31 +71,31 @@ class UniversalDetector:
 \r
         if not self._mGotData:\r
             # If the data starts with BOM, we know it is UTF\r
-            if aBuf[:3] == '\xEF\xBB\xBF':\r
+            if aBuf[:3] == codecs.BOM:\r
                 # EF BB BF  UTF-8 with BOM\r
                 self.result = {'encoding': "UTF-8", 'confidence': 1.0}\r
-            elif aBuf[:4] == '\xFF\xFE\x00\x00':\r
+            elif aBuf[:4] == codecs.BOM_UTF32_LE:\r
                 # FF FE 00 00  UTF-32, little-endian BOM\r
                 self.result = {'encoding': "UTF-32LE", 'confidence': 1.0}\r
-            elif aBuf[:4] == '\x00\x00\xFE\xFF':\r
+            elif aBuf[:4] == codecs.BOM_UTF32_BE:\r
                 # 00 00 FE FF  UTF-32, big-endian BOM\r
                 self.result = {'encoding': "UTF-32BE", 'confidence': 1.0}\r
-            elif aBuf[:4] == '\xFE\xFF\x00\x00':\r
+            elif aBuf[:4] == b'\xFE\xFF\x00\x00':\r
                 # FE FF 00 00  UCS-4, unusual octet order BOM (3412)\r
                 self.result = {\r
                     'encoding': "X-ISO-10646-UCS-4-3412",\r
                     'confidence': 1.0\r
                 }\r
-            elif aBuf[:4] == '\x00\x00\xFF\xFE':\r
+            elif aBuf[:4] == b'\x00\x00\xFF\xFE':\r
                 # 00 00 FF FE  UCS-4, unusual octet order BOM (2143)\r
                 self.result = {\r
                     'encoding': "X-ISO-10646-UCS-4-2143",\r
                     'confidence': 1.0\r
                 }\r
-            elif aBuf[:2] == '\xFF\xFE':\r
+            elif aBuf[:2] == codecs.BOM_LE:\r
                 # FF FE  UTF-16, little endian BOM\r
                 self.result = {'encoding': "UTF-16LE", 'confidence': 1.0}\r
-            elif aBuf[:2] == '\xFE\xFF':\r
+            elif aBuf[:2] == codecs.BOM_BE:\r
                 # FE FF  UTF-16, big endian BOM\r
                 self.result = {'encoding': "UTF-16BE", 'confidence': 1.0}\r
 \r