Integrated my history into data

2018-01-21 10:46:18 -08:00
parent bce29a723f
commit 97d35e633c
917 changed files with 50890 additions and 11 deletions
@@ -0,0 +1,91 @@
+import codecs
+
+try:
+    chr(0x10000)
+except ValueError:
+    # narrow python build
+    UCSCHAR = [
+        (0xA0, 0xD7FF),
+        (0xF900, 0xFDCF),
+        (0xFDF0, 0xFFEF),
+    ]
+
+    IPRIVATE = [
+        (0xE000, 0xF8FF),
+    ]
+else:
+    UCSCHAR = [
+        (0xA0, 0xD7FF),
+        (0xF900, 0xFDCF),
+        (0xFDF0, 0xFFEF),
+        (0x10000, 0x1FFFD),
+        (0x20000, 0x2FFFD),
+        (0x30000, 0x3FFFD),
+        (0x40000, 0x4FFFD),
+        (0x50000, 0x5FFFD),
+        (0x60000, 0x6FFFD),
+        (0x70000, 0x7FFFD),
+        (0x80000, 0x8FFFD),
+        (0x90000, 0x9FFFD),
+        (0xA0000, 0xAFFFD),
+        (0xB0000, 0xBFFFD),
+        (0xC0000, 0xCFFFD),
+        (0xD0000, 0xDFFFD),
+        (0xE1000, 0xEFFFD),
+    ]
+
+    IPRIVATE = [
+        (0xE000, 0xF8FF),
+        (0xF0000, 0xFFFFD),
+        (0x100000, 0x10FFFD),
+    ]
+
+_ESCAPE_RANGES = UCSCHAR + IPRIVATE
+
+
+def _in_escape_range(octet):
+    for start, end in _ESCAPE_RANGES:
+        if start <= octet <= end:
+            return True
+    return False
+
+
+def _starts_surrogate_pair(character):
+    char_value = ord(character)
+    return 0xD800 <= char_value <= 0xDBFF
+
+
+def _ends_surrogate_pair(character):
+    char_value = ord(character)
+    return 0xDC00 <= char_value <= 0xDFFF
+
+
+def _pct_encoded_replacements(chunk):
+    replacements = []
+    chunk_iter = iter(chunk)
+    for character in chunk_iter:
+        codepoint = ord(character)
+        if _in_escape_range(codepoint):
+            for char in chr(codepoint).encode("utf-8"):
+                replacements.append("%%%X" % char)
+        elif _starts_surrogate_pair(character):
+            next_character = next(chunk_iter)
+            for char in (character + next_character).encode("utf-8"):
+                replacements.append("%%%X" % char)
+        else:
+            replacements.append(chr(codepoint))
+    return replacements
+
+
+def _pct_escape_handler(err):
+    '''
+    Encoding error handler that does percent-escaping of Unicode, to be used
+    with codecs.register_error
+    TODO: replace use of this with urllib.parse.quote as appropriate
+    '''
+    chunk = err.object[err.start:err.end]
+    replacements = _pct_encoded_replacements(chunk)
+    return ("".join(replacements), err.end)
+
+
+codecs.register_error("oid_percent_escape", _pct_escape_handler)