fix(diff): read raw diff records field by field

git/diff.py

@@ -733,12 +733,17 @@ def _handle_diff_line(lines_bytes: bytes, repo: "Repo", index: DiffIndex["Diff"]
         # Discard everything before the first colon, and the colon itself.
         _, _, lines = lines.partition(":")
 
-        for line in lines.split("\x00:"):
-            if not line:
+        # Each record is a metadata field starting with a colon, followed by one path,
+        # or two for copies and renames, each terminated by NUL. A path may itself
+        # start with a colon or contain a line feed, so read the fields in order
+        # instead of splitting at colons.
+        fields = iter(lines.rstrip("\n").split("\x00"))
+        for meta in fields:
+            if not meta:
                 # The line data is empty, skip.
                 continue
-            meta, _, path = line.partition("\x00")
-            path = path.rstrip("\x00")
+            if meta.startswith(":"):
+                meta = meta[1:]
             a_blob_id: Optional[str]
             b_blob_id: Optional[str]
             old_mode, new_mode, a_blob_id, b_blob_id, _change_type = meta.split(None, 4)
@@ -748,7 +753,7 @@ def _handle_diff_line(lines_bytes: bytes, repo: "Repo", index: DiffIndex["Diff"]
             change_type: Lit_change_type = cast(Lit_change_type, _change_type[0])
             score_str = "".join(_change_type[1:])
             score = int(score_str) if score_str.isdigit() else None
-            path = path.strip("\n")
+            path = next(fields, "")
             a_path = path.encode(defenc)
             b_path = path.encode(defenc)
             deleted_file = False
@@ -767,13 +772,9 @@ def _handle_diff_line(lines_bytes: bytes, repo: "Repo", index: DiffIndex["Diff"]
                 new_file = True
             elif change_type == "C":
                 copied_file = True
-                a_path_str, b_path_str = path.split("\x00", 1)
-                a_path = a_path_str.encode(defenc)
-                b_path = b_path_str.encode(defenc)
+                b_path = next(fields, "").encode(defenc)
             elif change_type == "R":
-                a_path_str, b_path_str = path.split("\x00", 1)
-                a_path = a_path_str.encode(defenc)
-                b_path = b_path_str.encode(defenc)
+                b_path = next(fields, "").encode(defenc)
                 rename_from, rename_to = a_path, b_path
             elif change_type == "T":
                 # Nothing to do.
@@ -817,17 +818,17 @@ def _index_from_raw_format(cls, repo: "Repo", proc: "Popen") -> "DiffIndex[Diff]
         # :100644 100644 687099101... 37c5e30c8... M    .gitignore
 
         index: "DiffIndex" = DiffIndex()
+        text_list: List[bytes] = []
         stderr_list: List[bytes] = []
 
         def finalize_process_with_stderr(proc: Union["Popen", "Git.AutoInterrupt"]) -> None:
             finalize_process(proc, stderr=b"".join(stderr_list))
 
+        # Output is read line by line, but paths may contain line feeds, so parse
+        # the records only once all of it has been read.
         handle_process_output(
-            proc,
-            lambda byt: cls._handle_diff_line(byt, repo, index),
-            stderr_list.append,
-            finalize_process_with_stderr,
-            decode_streams=False,
+            proc, text_list.append, stderr_list.append, finalize_process_with_stderr, decode_streams=False
         )
+        cls._handle_diff_line(b"".join(text_list), repo, index)
 
         return index

test/test_diff.py

@@ -4,6 +4,7 @@
 # 3-Clause BSD License: https://opensource.org/license/bsd-3-clause/
 
 import gc
+from io import BytesIO
 import os.path as osp
 import shutil
 import sys
@@ -12,10 +13,13 @@
 import ddt
 import pytest
 
-from git import NULL_TREE, Diff, DiffIndex, Diffable, GitCommandError, Repo, Submodule
+from gitdb.base import IStream
+
+from git import NULL_TREE, Blob, Diff, DiffIndex, Diffable, GitCommandError, Repo, Submodule
 from git.cmd import Git
 from git.diff import decode_path
 from git.exc import UnsafeOptionError
+from git.index.typ import BaseIndexEntry
 
 from test.lib import StringProcessAdapter, TestBase, fixture, with_rw_directory
 
@@ -253,6 +257,19 @@ def test_diff_file_with_colon(self):
         Diff._handle_diff_line(output, None, res)
         self.assertEqual(len(res), 3)
 
+    @with_rw_directory
+    def test_diff_raw_paths_with_colon_or_line_feed(self, rw_dir):
+        repo = Repo.init(rw_dir)
+        binsha = repo.odb.store(IStream(Blob.type, 1, BytesIO(b"x"))).binsha
+        fake_meta = ":100644 100644 %s %s M" % ("1" * 40, "2" * 40)
+        paths = [":planted", "docs/a\n" + fake_meta, "docs/b", "setup.py"]
+        repo.index.add([BaseIndexEntry((0o100644, binsha, 0, path)) for path in paths])
+        commit = repo.index.commit("paths")
+
+        diffs = commit.diff(NULL_TREE)
+        self.assertEqual([d.b_path for d in diffs], sorted(paths))
+        self.assertEqual({d.change_type for d in diffs}, {"A"})
+
     def test_empty_diff(self):
         res = []
         Diff._handle_diff_line(b"", None, res)