From 85d96eb1c4518e4d4e2a9553421b9bfb8caa58b1 Mon Sep 17 00:00:00 2001 From: Ben Date: Fri, 9 Jan 2026 20:50:53 +0000 Subject: [PATCH 1/9] Also check output for multi patches --- features/patch-after-fetch-git.feature | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/features/patch-after-fetch-git.feature b/features/patch-after-fetch-git.feature index 61ab463a..85f1e5c2 100644 --- a/features/patch-after-fetch-git.feature +++ b/features/patch-after-fetch-git.feature @@ -121,3 +121,12 @@ Feature: Patch after fetching from git repo # Test-repo A test repo for testing dfetch. """ + And the output shows + """ + Dfetch (0.11.0) + ext/test-repo-tag : Fetched v2.0 + successfully patched 1/1: b'README.md' + ext/test-repo-tag : Applied patch "001-diff.patch" + successfully patched 1/1: b'README.md' + ext/test-repo-tag : Applied patch "002-diff.patch" + """ From dbf8c7fccf7e51cce9687d6842062d0fc00df70f Mon Sep 17 00:00:00 2001 From: Ben Date: Fri, 9 Jan 2026 20:52:40 +0000 Subject: [PATCH 2/9] Move patching to dfetch.vcs.patch --- dfetch/project/subproject.py | 23 +++++++---------------- dfetch/vcs/patch.py | 10 ++++++++++ 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/dfetch/project/subproject.py b/dfetch/project/subproject.py index 8a5e2869..d676bc5e 100644 --- a/dfetch/project/subproject.py +++ b/dfetch/project/subproject.py @@ -8,7 +8,6 @@ from typing import Optional from halo import Halo -from patch_ng import fromfile from dfetch.log import get_logger from dfetch.manifest.project import ProjectEntry @@ -17,6 +16,7 @@ from dfetch.project.metadata import Metadata from dfetch.util.util import hash_directory, safe_rm from dfetch.util.versions import latest_tag_from_list +from dfetch.vcs.patch import apply_patch logger = get_logger(__name__) @@ -130,11 +130,13 @@ def update( applied_patches = [] for patch in self.__project.patch: - if os.path.exists(patch): - self.apply_patch(patch) - applied_patches.append(patch) - else: + if not os.path.exists(patch): logger.warning(f"Skipping non-existent patch {patch}") + continue + + apply_patch(patch, root=self.local_path) + self._log_project(f'Applied patch "{patch}"') + applied_patches.append(patch) self.__metadata.fetched( actually_fetched, @@ -145,17 +147,6 @@ def update( logger.debug(f"Writing repo metadata to: {self.__metadata.path}") self.__metadata.dump() - def apply_patch(self, patch: str) -> None: - """Apply the specified patch to the destination.""" - patch_set = fromfile(patch) - - if not patch_set: - raise RuntimeError(f'Invalid patch file: "{patch}"') - if patch_set.apply(0, root=self.local_path, fuzz=True): - self._log_project(f'Applied patch "{patch}"') - else: - raise RuntimeError(f'Applying patch "{patch}" failed') - def check_for_update( self, reporters: Sequence[AbstractCheckReporter], files_to_ignore: Sequence[str] ) -> None: diff --git a/dfetch/vcs/patch.py b/dfetch/vcs/patch.py index 6380ffc2..bafe8c44 100644 --- a/dfetch/vcs/patch.py +++ b/dfetch/vcs/patch.py @@ -55,6 +55,16 @@ def dump_patch(patch_set: patch_ng.PatchSet) -> str: return "\n".join(patch_lines) + "\n" if patch_lines else "" +def apply_patch(patch_path: str, root: str = ".") -> None: + """Apply the specified patch relative to the root.""" + patch_set = patch_ng.fromfile(patch_path) + + if not patch_set: + raise RuntimeError(f'Invalid patch file: "{patch_path}"') + if not patch_set.apply(strip=0, root=root, fuzz=True): + raise RuntimeError(f'Applying patch "{patch_path}" failed') + + def create_svn_patch_for_new_file(file_path: str) -> str: """Create a svn patch for a new file.""" diff = unified_diff_new_file(Path(file_path)) From 96d9ba0f7380c9761c7a20a6c82620ed97503bee Mon Sep 17 00:00:00 2001 From: Ben Date: Fri, 9 Jan 2026 20:55:06 +0000 Subject: [PATCH 3/9] Reproduce Unclear error message if patch file is utf-16 encoded --- features/patch-after-fetch-git.feature | 38 ++++++++++++++++++++++++-- features/steps/generic_steps.py | 9 +++--- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/features/patch-after-fetch-git.feature b/features/patch-after-fetch-git.feature index 85f1e5c2..0dba5a69 100644 --- a/features/patch-after-fetch-git.feature +++ b/features/patch-after-fetch-git.feature @@ -125,8 +125,42 @@ Feature: Patch after fetching from git repo """ Dfetch (0.11.0) ext/test-repo-tag : Fetched v2.0 - successfully patched 1/1: b'README.md' + successfully patched 1/1: b'README.md' ext/test-repo-tag : Applied patch "001-diff.patch" - successfully patched 1/1: b'README.md' + successfully patched 1/1: b'README.md' ext/test-repo-tag : Applied patch "002-diff.patch" """ + + Scenario: A UTF-16 encoded patch file is applied after fetching + Given the manifest 'dfetch.yaml' + """ + manifest: + version: '0.0' + + remotes: + - name: github-com-dfetch-org + url-base: https://github.com/dfetch-org/test-repo + + projects: + - name: ext/test-repo-tag + tag: v2.0 + dst: ext/test-repo-tag + patch: diff.patch + """ + And the patch file 'diff.patch' with 'UTF-16' encoding + """ + diff --git a/README.md b/README.md + index 32d9fad..62248b7 100644 + --- a/README.md + +++ b/README.md + @@ -1,2 +1,2 @@ + # Test-repo + -A test repo for testing dfetch. + +A test repo for testing patch. + """ + When I run "dfetch update" + Then the patched 'ext/test-repo-tag/README.md' is + """ + # Test-repo + A test repo for testing patch. + """ diff --git a/features/steps/generic_steps.py b/features/steps/generic_steps.py index 14763a56..1cceb228 100644 --- a/features/steps/generic_steps.py +++ b/features/steps/generic_steps.py @@ -104,13 +104,13 @@ def check_content( ) -def generate_file(path, content): +def generate_file(path, content, encoding="UTF-8"): opt_dir = path.rsplit("/", maxsplit=1) if len(opt_dir) > 1: pathlib.Path(opt_dir[0]).mkdir(parents=True, exist_ok=True) - with open(path, "w", encoding="UTF-8") as new_file: + with open(path, "w", encoding=encoding) as new_file: for line in content.splitlines(): print(line, file=new_file) @@ -202,8 +202,9 @@ def step_impl(_, old: str, new: str, path: str): @given("the patch file '{name}'") -def step_impl(context, name): - generate_file(os.path.join(os.getcwd(), name), context.text) +@given("the patch file '{name}' with '{encoding}' encoding") +def step_impl(context, name, encoding="UTF-8"): + generate_file(os.path.join(os.getcwd(), name), context.text, encoding) @given('"{path}" in {directory} is created') From d9e18b1a8f87cb161c9a67577c3ece8f3d194b70 Mon Sep 17 00:00:00 2001 From: Ben Date: Fri, 9 Jan 2026 21:28:08 +0000 Subject: [PATCH 4/9] Fixes Unclear error message if patch file is utf-16 encoded Fixes #941 --- CHANGELOG.rst | 1 + dfetch/manifest/project.py | 3 ++- dfetch/vcs/patch.py | 15 +++++++++++++++ features/patch-after-fetch-git.feature | 11 ++++++++++- 4 files changed, 28 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0d52f1e5..44608fb3 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,7 @@ Release 0.12.0 (unreleased) * Add Fuzzing (#819) * Don't allow NULL or control characters in manifest (#114) * Allow multiple patches in manifest (#897) +* Fallback and warn if patch is not UTF-8 encoded (#941) Release 0.11.0 (released 2026-01-03) ==================================== diff --git a/dfetch/manifest/project.py b/dfetch/manifest/project.py index f804b467..9a5d7e10 100644 --- a/dfetch/manifest/project.py +++ b/dfetch/manifest/project.py @@ -225,7 +225,8 @@ ##### *DFetch* promotes upstreaming changes, but also allows local changes. These changes can be managed with local patch files. *DFetch* will apply the patch files in order every time a new upstream version is fetched. The patch file can -be specified with the ``patch:`` attribute. This can be a single patch file or multiple. +be specified with the ``patch:`` attribute. This can be a single patch file or multiple. Patch files should be UTF-8 +encoded files. .. code-block:: yaml diff --git a/dfetch/vcs/patch.py b/dfetch/vcs/patch.py index bafe8c44..9e95a6e8 100644 --- a/dfetch/vcs/patch.py +++ b/dfetch/vcs/patch.py @@ -8,6 +8,10 @@ import patch_ng +from dfetch.log import get_logger + +logger = get_logger(__name__) + def _git_mode(path: Path) -> str: if path.is_symlink(): @@ -59,6 +63,17 @@ def apply_patch(patch_path: str, root: str = ".") -> None: """Apply the specified patch relative to the root.""" patch_set = patch_ng.fromfile(patch_path) + if not patch_set: + with open(patch_path, "rb") as patch_file: + patch_text = patch_ng.decode_text(patch_file.read()).encode("utf-8") + patch_set = patch_ng.fromstring(patch_text) + + if patch_set: + logger.warning( + f'After retrying found that patch-file "{patch_path}" ' + "is not UTF-8 encoded, consider saving it with UTF-8 encoding." + ) + if not patch_set: raise RuntimeError(f'Invalid patch file: "{patch_path}"') if not patch_set.apply(strip=0, root=root, fuzz=True): diff --git a/features/patch-after-fetch-git.feature b/features/patch-after-fetch-git.feature index 0dba5a69..ff895871 100644 --- a/features/patch-after-fetch-git.feature +++ b/features/patch-after-fetch-git.feature @@ -131,7 +131,7 @@ Feature: Patch after fetching from git repo ext/test-repo-tag : Applied patch "002-diff.patch" """ - Scenario: A UTF-16 encoded patch file is applied after fetching + Scenario: Fallback to other file encodings if patch file is not UTF-8 encoded Given the manifest 'dfetch.yaml' """ manifest: @@ -164,3 +164,12 @@ Feature: Patch after fetching from git repo # Test-repo A test repo for testing patch. """ + And the output shows + """ + Dfetch (0.11.0) + ext/test-repo-tag : Fetched v2.0 + error: no patch data found! + After retrying found that patch-file "diff.patch" is not UTF-8 encoded, consider saving it with UTF-8 encoding. + successfully patched 1/1: b'README.md' + ext/test-repo-tag : Applied patch "diff.patch" + """ From 65f0ccaefaff412ddd571ad64ed2a7f7c0632636 Mon Sep 17 00:00:00 2001 From: Ben Date: Fri, 9 Jan 2026 21:44:19 +0000 Subject: [PATCH 5/9] Skip patches outside manifest dir --- CHANGELOG.rst | 1 + dfetch/project/subproject.py | 32 ++++++++++++++++++-------- features/patch-after-fetch-git.feature | 24 +++++++++++++++++++ 3 files changed, 48 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 44608fb3..2b3b4fad 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,6 +8,7 @@ Release 0.12.0 (unreleased) * Don't allow NULL or control characters in manifest (#114) * Allow multiple patches in manifest (#897) * Fallback and warn if patch is not UTF-8 encoded (#941) +* Skip patches outside manifest dir (#942) Release 0.11.0 (released 2026-01-03) ==================================== diff --git a/dfetch/project/subproject.py b/dfetch/project/subproject.py index d676bc5e..263a057e 100644 --- a/dfetch/project/subproject.py +++ b/dfetch/project/subproject.py @@ -128,15 +128,7 @@ def update( actually_fetched = self._fetch_impl(to_fetch) self._log_project(f"Fetched {actually_fetched}") - applied_patches = [] - for patch in self.__project.patch: - if not os.path.exists(patch): - logger.warning(f"Skipping non-existent patch {patch}") - continue - - apply_patch(patch, root=self.local_path) - self._log_project(f'Applied patch "{patch}"') - applied_patches.append(patch) + applied_patches = self._apply_patches() self.__metadata.fetched( actually_fetched, @@ -147,6 +139,28 @@ def update( logger.debug(f"Writing repo metadata to: {self.__metadata.path}") self.__metadata.dump() + def _apply_patches(self) -> list[str]: + """Apply the patches.""" + manifest_dir = os.getcwd() + applied_patches = [] + for patch in self.__project.patch: + + real_path = os.path.realpath(patch) + if os.path.commonprefix((real_path, manifest_dir)) != manifest_dir: + self._log_project( + f'Skipping patch "{patch}" which is outside manifest dir.' + ) + continue + + if not os.path.exists(patch): + self._log_project(f"Skipping non-existent patch {patch}") + continue + + apply_patch(patch, root=self.local_path) + self._log_project(f'Applied patch "{patch}"') + applied_patches.append(patch) + return applied_patches + def check_for_update( self, reporters: Sequence[AbstractCheckReporter], files_to_ignore: Sequence[str] ) -> None: diff --git a/features/patch-after-fetch-git.feature b/features/patch-after-fetch-git.feature index ff895871..1c474019 100644 --- a/features/patch-after-fetch-git.feature +++ b/features/patch-after-fetch-git.feature @@ -173,3 +173,27 @@ Feature: Patch after fetching from git repo successfully patched 1/1: b'README.md' ext/test-repo-tag : Applied patch "diff.patch" """ + + Scenario: Patch files are outside manifest dir + Given the manifest 'dfetch.yaml' + """ + manifest: + version: '0.0' + + remotes: + - name: github-com-dfetch-org + url-base: https://github.com/dfetch-org/test-repo + + projects: + - name: ext/test-repo-tag + tag: v2.0 + dst: ext/test-repo-tag + patch: ../diff.patch + """ + When I run "dfetch update" + Then the output shows + """ + Dfetch (0.11.0) + ext/test-repo-tag : Fetched v2.0 + ext/test-repo-tag : Skipping patch "../diff.patch" which is outside manifest dir. + """ From 51e4df1dc225cb0d2e2204df842f7588704e16e2 Mon Sep 17 00:00:00 2001 From: Ben Date: Fri, 9 Jan 2026 22:37:52 +0000 Subject: [PATCH 6/9] Patch file path in metadata not platform independent Fixes #937 --- CHANGELOG.rst | 1 + dfetch/manifest/project.py | 9 +++++---- dfetch/project/subproject.py | 20 +++++++++++++------- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2b3b4fad..c75cdd62 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,6 +9,7 @@ Release 0.12.0 (unreleased) * Allow multiple patches in manifest (#897) * Fallback and warn if patch is not UTF-8 encoded (#941) * Skip patches outside manifest dir (#942) +* Make patch path in metadata platform independent (#937) Release 0.11.0 (released 2026-01-03) ==================================== diff --git a/dfetch/manifest/project.py b/dfetch/manifest/project.py index 9a5d7e10..567cfe15 100644 --- a/dfetch/manifest/project.py +++ b/dfetch/manifest/project.py @@ -226,7 +226,8 @@ *DFetch* promotes upstreaming changes, but also allows local changes. These changes can be managed with local patch files. *DFetch* will apply the patch files in order every time a new upstream version is fetched. The patch file can be specified with the ``patch:`` attribute. This can be a single patch file or multiple. Patch files should be UTF-8 -encoded files. +encoded files and using forward slashes is encouraged for cross-platform support. The path should be relative to the +directory of the manifest. .. code-block:: yaml @@ -241,11 +242,11 @@ - name: cpputest vcs: git repo-path: cpputest/cpputest - patch: local_changes.patch + patch: patches/local_changes.patch The patch should be generated using the *Dfetch* :ref:`Diff` command. -Alternately the patch can be generated manually as such. -Note that the patch should be *relative* to the projects root. +Alternately the patch can be generated manually as such and should be +a *relative* patch, relative to the fetched projects root. .. tabs:: diff --git a/dfetch/project/subproject.py b/dfetch/project/subproject.py index 263a057e..42313fda 100644 --- a/dfetch/project/subproject.py +++ b/dfetch/project/subproject.py @@ -5,6 +5,7 @@ import pathlib from abc import ABC, abstractmethod from collections.abc import Sequence +from pathlib import Path from typing import Optional from halo import Halo @@ -141,24 +142,29 @@ def update( def _apply_patches(self) -> list[str]: """Apply the patches.""" - manifest_dir = os.getcwd() + manifest_dir = Path(".").resolve() applied_patches = [] for patch in self.__project.patch: - real_path = os.path.realpath(patch) - if os.path.commonprefix((real_path, manifest_dir)) != manifest_dir: + patch_path = (manifest_dir / patch).resolve() + + try: + patch_path.relative_to(manifest_dir) + except ValueError: self._log_project( f'Skipping patch "{patch}" which is outside manifest dir.' ) continue - if not os.path.exists(patch): + if not patch_path.exists(): self._log_project(f"Skipping non-existent patch {patch}") continue - apply_patch(patch, root=self.local_path) - self._log_project(f'Applied patch "{patch}"') - applied_patches.append(patch) + normalized_patch_path = str(patch_path.relative_to(manifest_dir).as_posix()) + + apply_patch(normalized_patch_path, root=self.local_path) + self._log_project(f'Applied patch "{normalized_patch_path}"') + applied_patches.append(normalized_patch_path) return applied_patches def check_for_update( From 52b4395781c471c39b2ca2f415373d492808b1e6 Mon Sep 17 00:00:00 2001 From: Ben Date: Fri, 9 Jan 2026 23:09:04 +0000 Subject: [PATCH 7/9] Don't let subproject get knowledge about manifest --- dfetch/project/subproject.py | 12 +++++------- features/patch-after-fetch-git.feature | 2 +- features/steps/generic_steps.py | 2 ++ 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/dfetch/project/subproject.py b/dfetch/project/subproject.py index 42313fda..790e792a 100644 --- a/dfetch/project/subproject.py +++ b/dfetch/project/subproject.py @@ -142,25 +142,23 @@ def update( def _apply_patches(self) -> list[str]: """Apply the patches.""" - manifest_dir = Path(".").resolve() + cwd = Path(".").resolve() applied_patches = [] for patch in self.__project.patch: - patch_path = (manifest_dir / patch).resolve() + patch_path = (cwd / patch).resolve() try: - patch_path.relative_to(manifest_dir) + patch_path.relative_to(cwd) except ValueError: - self._log_project( - f'Skipping patch "{patch}" which is outside manifest dir.' - ) + self._log_project(f'Skipping patch "{patch}" which is outside {cwd}.') continue if not patch_path.exists(): self._log_project(f"Skipping non-existent patch {patch}") continue - normalized_patch_path = str(patch_path.relative_to(manifest_dir).as_posix()) + normalized_patch_path = str(patch_path.relative_to(cwd).as_posix()) apply_patch(normalized_patch_path, root=self.local_path) self._log_project(f'Applied patch "{normalized_patch_path}"') diff --git a/features/patch-after-fetch-git.feature b/features/patch-after-fetch-git.feature index 1c474019..81a1fbdb 100644 --- a/features/patch-after-fetch-git.feature +++ b/features/patch-after-fetch-git.feature @@ -195,5 +195,5 @@ Feature: Patch after fetching from git repo """ Dfetch (0.11.0) ext/test-repo-tag : Fetched v2.0 - ext/test-repo-tag : Skipping patch "../diff.patch" which is outside manifest dir. + ext/test-repo-tag : Skipping patch "../diff.patch" which is outside /some/path. """ diff --git a/features/steps/generic_steps.py b/features/steps/generic_steps.py index 1cceb228..24737fee 100644 --- a/features/steps/generic_steps.py +++ b/features/steps/generic_steps.py @@ -25,6 +25,7 @@ urn_uuid = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") bom_ref = re.compile(r"BomRef\.[0-9]+\.[0-9]+") svn_error = re.compile(r"svn: E\d{6}: .+") +abs_path = re.compile(r"/tmp/[\w_]+") def remote_server_path(context): @@ -181,6 +182,7 @@ def check_output(context, line_count=None): "some-remote-server", ), (svn_error, "svn: EXXXXXX: "), + (abs_path, "/some/path"), ], text=context.cmd_output, ) From 5865826655249390155ea4cd587f482c44026fba Mon Sep 17 00:00:00 2001 From: Ben Date: Fri, 9 Jan 2026 23:15:22 +0000 Subject: [PATCH 8/9] Remove redundant import --- dfetch/project/subproject.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dfetch/project/subproject.py b/dfetch/project/subproject.py index 790e792a..b38c0703 100644 --- a/dfetch/project/subproject.py +++ b/dfetch/project/subproject.py @@ -5,7 +5,6 @@ import pathlib from abc import ABC, abstractmethod from collections.abc import Sequence -from pathlib import Path from typing import Optional from halo import Halo @@ -142,7 +141,7 @@ def update( def _apply_patches(self) -> list[str]: """Apply the patches.""" - cwd = Path(".").resolve() + cwd = pathlib.Path(".").resolve() applied_patches = [] for patch in self.__project.patch: From 8031217f9ea45a79cfa36318881f3a73c1ee3f1a Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 10 Jan 2026 00:29:13 +0100 Subject: [PATCH 9/9] calculate relative patch only once --- dfetch/project/subproject.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dfetch/project/subproject.py b/dfetch/project/subproject.py index b38c0703..d31f7243 100644 --- a/dfetch/project/subproject.py +++ b/dfetch/project/subproject.py @@ -148,7 +148,7 @@ def _apply_patches(self) -> list[str]: patch_path = (cwd / patch).resolve() try: - patch_path.relative_to(cwd) + relative_patch_path = patch_path.relative_to(cwd) except ValueError: self._log_project(f'Skipping patch "{patch}" which is outside {cwd}.') continue @@ -157,7 +157,7 @@ def _apply_patches(self) -> list[str]: self._log_project(f"Skipping non-existent patch {patch}") continue - normalized_patch_path = str(patch_path.relative_to(cwd).as_posix()) + normalized_patch_path = str(relative_patch_path.as_posix()) apply_patch(normalized_patch_path, root=self.local_path) self._log_project(f'Applied patch "{normalized_patch_path}"')