diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 0d52f1e5..c75cdd62 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Release 0.12.0 (unreleased) * Add Fuzzing (#819) * Don't allow NULL or control characters in manifest (#114) * Allow multiple patches in manifest (#897) +* Fallback and warn if patch is not UTF-8 encoded (#941) +* Skip patches outside manifest dir (#942) +* Make patch path in metadata platform independent (#937) Release 0.11.0 (released 2026-01-03) ==================================== diff --git a/dfetch/manifest/project.py b/dfetch/manifest/project.py index f804b467..567cfe15 100644 --- a/dfetch/manifest/project.py +++ b/dfetch/manifest/project.py @@ -225,7 +225,9 @@ ##### *DFetch* promotes upstreaming changes, but also allows local changes. These changes can be managed with local patch files. *DFetch* will apply the patch files in order every time a new upstream version is fetched. The patch file can -be specified with the ``patch:`` attribute. This can be a single patch file or multiple. +be specified with the ``patch:`` attribute. This can be a single patch file or multiple. Patch files should be UTF-8 +encoded files and using forward slashes is encouraged for cross-platform support. The path should be relative to the +directory of the manifest. .. code-block:: yaml @@ -240,11 +242,11 @@ - name: cpputest vcs: git repo-path: cpputest/cpputest - patch: local_changes.patch + patch: patches/local_changes.patch The patch should be generated using the *Dfetch* :ref:`Diff` command. -Alternately the patch can be generated manually as such. -Note that the patch should be *relative* to the projects root. +Alternately the patch can be generated manually as such and should be +a *relative* patch, relative to the fetched projects root. .. tabs:: diff --git a/dfetch/project/subproject.py b/dfetch/project/subproject.py index 8a5e2869..d31f7243 100644 --- a/dfetch/project/subproject.py +++ b/dfetch/project/subproject.py @@ -8,7 +8,6 @@ from typing import Optional from halo import Halo -from patch_ng import fromfile from dfetch.log import get_logger from dfetch.manifest.project import ProjectEntry @@ -17,6 +16,7 @@ from dfetch.project.metadata import Metadata from dfetch.util.util import hash_directory, safe_rm from dfetch.util.versions import latest_tag_from_list +from dfetch.vcs.patch import apply_patch logger = get_logger(__name__) @@ -128,13 +128,7 @@ def update( actually_fetched = self._fetch_impl(to_fetch) self._log_project(f"Fetched {actually_fetched}") - applied_patches = [] - for patch in self.__project.patch: - if os.path.exists(patch): - self.apply_patch(patch) - applied_patches.append(patch) - else: - logger.warning(f"Skipping non-existent patch {patch}") + applied_patches = self._apply_patches() self.__metadata.fetched( actually_fetched, @@ -145,16 +139,30 @@ def update( logger.debug(f"Writing repo metadata to: {self.__metadata.path}") self.__metadata.dump() - def apply_patch(self, patch: str) -> None: - """Apply the specified patch to the destination.""" - patch_set = fromfile(patch) + def _apply_patches(self) -> list[str]: + """Apply the patches.""" + cwd = pathlib.Path(".").resolve() + applied_patches = [] + for patch in self.__project.patch: - if not patch_set: - raise RuntimeError(f'Invalid patch file: "{patch}"') - if patch_set.apply(0, root=self.local_path, fuzz=True): - self._log_project(f'Applied patch "{patch}"') - else: - raise RuntimeError(f'Applying patch "{patch}" failed') + patch_path = (cwd / patch).resolve() + + try: + relative_patch_path = patch_path.relative_to(cwd) + except ValueError: + self._log_project(f'Skipping patch "{patch}" which is outside {cwd}.') + continue + + if not patch_path.exists(): + self._log_project(f"Skipping non-existent patch {patch}") + continue + + normalized_patch_path = str(relative_patch_path.as_posix()) + + apply_patch(normalized_patch_path, root=self.local_path) + self._log_project(f'Applied patch "{normalized_patch_path}"') + applied_patches.append(normalized_patch_path) + return applied_patches def check_for_update( self, reporters: Sequence[AbstractCheckReporter], files_to_ignore: Sequence[str] diff --git a/dfetch/vcs/patch.py b/dfetch/vcs/patch.py index 6380ffc2..9e95a6e8 100644 --- a/dfetch/vcs/patch.py +++ b/dfetch/vcs/patch.py @@ -8,6 +8,10 @@ import patch_ng +from dfetch.log import get_logger + +logger = get_logger(__name__) + def _git_mode(path: Path) -> str: if path.is_symlink(): @@ -55,6 +59,27 @@ def dump_patch(patch_set: patch_ng.PatchSet) -> str: return "\n".join(patch_lines) + "\n" if patch_lines else "" +def apply_patch(patch_path: str, root: str = ".") -> None: + """Apply the specified patch relative to the root.""" + patch_set = patch_ng.fromfile(patch_path) + + if not patch_set: + with open(patch_path, "rb") as patch_file: + patch_text = patch_ng.decode_text(patch_file.read()).encode("utf-8") + patch_set = patch_ng.fromstring(patch_text) + + if patch_set: + logger.warning( + f'After retrying found that patch-file "{patch_path}" ' + "is not UTF-8 encoded, consider saving it with UTF-8 encoding." + ) + + if not patch_set: + raise RuntimeError(f'Invalid patch file: "{patch_path}"') + if not patch_set.apply(strip=0, root=root, fuzz=True): + raise RuntimeError(f'Applying patch "{patch_path}" failed') + + def create_svn_patch_for_new_file(file_path: str) -> str: """Create a svn patch for a new file.""" diff = unified_diff_new_file(Path(file_path)) diff --git a/features/patch-after-fetch-git.feature b/features/patch-after-fetch-git.feature index 61ab463a..81a1fbdb 100644 --- a/features/patch-after-fetch-git.feature +++ b/features/patch-after-fetch-git.feature @@ -121,3 +121,79 @@ Feature: Patch after fetching from git repo # Test-repo A test repo for testing dfetch. """ + And the output shows + """ + Dfetch (0.11.0) + ext/test-repo-tag : Fetched v2.0 + successfully patched 1/1: b'README.md' + ext/test-repo-tag : Applied patch "001-diff.patch" + successfully patched 1/1: b'README.md' + ext/test-repo-tag : Applied patch "002-diff.patch" + """ + + Scenario: Fallback to other file encodings if patch file is not UTF-8 encoded + Given the manifest 'dfetch.yaml' + """ + manifest: + version: '0.0' + + remotes: + - name: github-com-dfetch-org + url-base: https://github.com/dfetch-org/test-repo + + projects: + - name: ext/test-repo-tag + tag: v2.0 + dst: ext/test-repo-tag + patch: diff.patch + """ + And the patch file 'diff.patch' with 'UTF-16' encoding + """ + diff --git a/README.md b/README.md + index 32d9fad..62248b7 100644 + --- a/README.md + +++ b/README.md + @@ -1,2 +1,2 @@ + # Test-repo + -A test repo for testing dfetch. + +A test repo for testing patch. + """ + When I run "dfetch update" + Then the patched 'ext/test-repo-tag/README.md' is + """ + # Test-repo + A test repo for testing patch. + """ + And the output shows + """ + Dfetch (0.11.0) + ext/test-repo-tag : Fetched v2.0 + error: no patch data found! + After retrying found that patch-file "diff.patch" is not UTF-8 encoded, consider saving it with UTF-8 encoding. + successfully patched 1/1: b'README.md' + ext/test-repo-tag : Applied patch "diff.patch" + """ + + Scenario: Patch files are outside manifest dir + Given the manifest 'dfetch.yaml' + """ + manifest: + version: '0.0' + + remotes: + - name: github-com-dfetch-org + url-base: https://github.com/dfetch-org/test-repo + + projects: + - name: ext/test-repo-tag + tag: v2.0 + dst: ext/test-repo-tag + patch: ../diff.patch + """ + When I run "dfetch update" + Then the output shows + """ + Dfetch (0.11.0) + ext/test-repo-tag : Fetched v2.0 + ext/test-repo-tag : Skipping patch "../diff.patch" which is outside /some/path. + """ diff --git a/features/steps/generic_steps.py b/features/steps/generic_steps.py index 14763a56..24737fee 100644 --- a/features/steps/generic_steps.py +++ b/features/steps/generic_steps.py @@ -25,6 +25,7 @@ urn_uuid = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") bom_ref = re.compile(r"BomRef\.[0-9]+\.[0-9]+") svn_error = re.compile(r"svn: E\d{6}: .+") +abs_path = re.compile(r"/tmp/[\w_]+") def remote_server_path(context): @@ -104,13 +105,13 @@ def check_content( ) -def generate_file(path, content): +def generate_file(path, content, encoding="UTF-8"): opt_dir = path.rsplit("/", maxsplit=1) if len(opt_dir) > 1: pathlib.Path(opt_dir[0]).mkdir(parents=True, exist_ok=True) - with open(path, "w", encoding="UTF-8") as new_file: + with open(path, "w", encoding=encoding) as new_file: for line in content.splitlines(): print(line, file=new_file) @@ -181,6 +182,7 @@ def check_output(context, line_count=None): "some-remote-server", ), (svn_error, "svn: EXXXXXX: "), + (abs_path, "/some/path"), ], text=context.cmd_output, ) @@ -202,8 +204,9 @@ def step_impl(_, old: str, new: str, path: str): @given("the patch file '{name}'") -def step_impl(context, name): - generate_file(os.path.join(os.getcwd(), name), context.text) +@given("the patch file '{name}' with '{encoding}' encoding") +def step_impl(context, name, encoding="UTF-8"): + generate_file(os.path.join(os.getcwd(), name), context.text, encoding) @given('"{path}" in {directory} is created')