patch from upstream: 'Python3: Stop breaking surrogate pairs in toDelta()'

Author: matthewhegarty

diff_match_patch/diff_match_patch.py

@@ -1136,6 +1136,8 @@ def diff_prettyHtml(self, diffs):
         """
         html = []
         for op, data in diffs:
+            if 0 == len(data):
+                continue
             text = (
                 data.replace("&", "&")
                 .replace("<", "&lt;")
@@ -1225,9 +1227,9 @@ def diff_toDelta(self, diffs):
                 data = data.encode("utf-8")
                 text.append("+" + urllib.parse.quote(data, "!~*'();/?:@&=+$,# "))
             elif op == self.DIFF_DELETE:
-                text.append("-%d" % len(data))
+                text.append("-%d" % (len(data.encode('utf-16-be')) // 2))
             elif op == self.DIFF_EQUAL:
-                text.append("=%d" % len(data))
+                text.append("=%d" % (len(data.encode('utf-16-be')) // 2))
         return "\t".join(text)
 
     def diff_fromDelta(self, text1, delta):
@@ -1245,7 +1247,8 @@ def diff_fromDelta(self, text1, delta):
           ValueError: If invalid input.
         """
         diffs = []
-        pointer = 0  # Cursor in text1
+        as_utf16 = text1.encode('utf-16-be')
+        pointer = 0  # Cursor in as_utf16
         tokens = delta.split("\t")
         for token in tokens:
             if token == "":
@@ -1264,8 +1267,8 @@ def diff_fromDelta(self, text1, delta):
                     raise ValueError("Invalid number in diff_fromDelta: " + param)
                 if n < 0:
                     raise ValueError("Negative number in diff_fromDelta: " + param)
-                text = text1[pointer : pointer + n]
-                pointer += n
+                text = as_utf16[pointer: pointer + n * 2].decode('utf-16-be')
+                pointer += n * 2
                 if token[0] == "=":
                     diffs.append((self.DIFF_EQUAL, text))
                 else:
@@ -1275,10 +1278,10 @@ def diff_fromDelta(self, text1, delta):
                 raise ValueError(
                     "Invalid diff operation in diff_fromDelta: " + token[0]
                 )
-        if pointer != len(text1):
+        if pointer != len(as_utf16):
             raise ValueError(
                 "Delta length (%d) does not equal source text length (%d)."
-                % (pointer, len(text1))
+                % (pointer, len(as_utf16))
             )
         return diffs
 

diff_match_patch/tests/diff_match_patch_test.py

@@ -833,6 +833,11 @@ def testDiffDelta(self):
         # Convert delta string into a diff.
         self.assertEqual(diffs, self.dmp.diff_fromDelta(text1, delta))
 
+        diffs = self.dmp.diff_main("\U0001F64B\U0001F64B", "\U0001F64B\U0001F64C\U0001F64B")
+        delta = self.dmp.diff_toDelta(diffs)
+        self.assertEqual("=2\t+%F0%9F%99%8C\t=2", delta)
+        self.assertEqual(diffs, self.dmp.diff_fromDelta("\U0001F64B\U0001F64B", "=2\t+%F0%9F%99%8C\t=2"))
+
         # Verify pool of unchanged characters.
         diffs = [
             (
@@ -849,6 +854,65 @@ def testDiffDelta(self):
         # Convert delta string into a diff.
         self.assertEqual(diffs, self.dmp.diff_fromDelta("", delta))
 
+        # Unicode: split surrogates
+        self.assertEqual(
+            self.dmp.diff_toDelta([
+                (self.dmp.DIFF_INSERT, '\U0001F171'),
+                (self.dmp.DIFF_EQUAL, '\U0001F170\U0001F171')
+            ]),
+            self.dmp.diff_toDelta(self.dmp.diff_main(
+                '\U0001F170\U0001F171',
+                '\U0001F171\U0001F170\U0001F171'
+            )),
+            'Inserting similar surrogate pair at beginning'
+        )
+        self.assertEqual(
+            self.dmp.diff_toDelta([
+                (self.dmp.DIFF_EQUAL, '\U0001F170'),
+                (self.dmp.DIFF_INSERT, '\U0001F172'),
+                (self.dmp.DIFF_EQUAL, '\U0001F171')
+            ]),
+            self.dmp.diff_toDelta(self.dmp.diff_main(
+                '\U0001F170\U0001F171',
+                '\U0001F170\U0001F172\U0001F171'
+            )),
+            'Inserting similar surrogate pair in the middle'
+        )
+        self.assertEqual(
+            self.dmp.diff_toDelta([
+                (self.dmp.DIFF_DELETE, '\U0001F171'),
+                (self.dmp.DIFF_EQUAL, '\U0001F170\U0001F171')
+            ]),
+            self.dmp.diff_toDelta(self.dmp.diff_main(
+                '\U0001F171\U0001F170\U0001F171',
+                '\U0001F170\U0001F171'
+            )),
+            'Deleting similar surogate pair at the beginning'
+        )
+        self.assertEqual(
+            self.dmp.diff_toDelta([
+                (self.dmp.DIFF_EQUAL, '\U0001F170'),
+                (self.dmp.DIFF_DELETE, '\U0001F172'),
+                (self.dmp.DIFF_EQUAL, '\U0001F171')
+            ]),
+            self.dmp.diff_toDelta(self.dmp.diff_main(
+                '\U0001F170\U0001F172\U0001F171',
+                '\U0001F170\U0001F171'
+            )),
+            'Deleting similar surogate pair in the middle'
+        )
+        self.assertEqual(
+            self.dmp.diff_toDelta([
+                (self.dmp.DIFF_DELETE, '\U0001F170'),
+                (self.dmp.DIFF_INSERT, '\U0001F171')
+            ]),
+            self.dmp.diff_toDelta(self.dmp.diff_main(
+                '\U0001F170',
+                '\U0001F171'
+            )),
+            'Swap surrogate pair'
+        )
+
         # 160 kb string.
         a = "abcdefghij"
         for i in range(14):