From 43991ea787dd3a928e7fbbbe95378ec145913e32 Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 3 Jan 2026 15:12:14 +0000 Subject: [PATCH 01/13] Try to reproduce #570 --- features/check-git-repo.feature | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/features/check-git-repo.feature b/features/check-git-repo.feature index c6bc68e3..ba496a6f 100644 --- a/features/check-git-repo.feature +++ b/features/check-git-repo.feature @@ -205,3 +205,22 @@ Feature: Checking dependencies from a git repository SomeProjectNonExistentBranch: wanted (i-dont-exist), but not available at the upstream. SomeProjectNonExistentRevision: wanted (0123112321234123512361236123712381239123), but not available at the upstream. """ + + Scenario: Credentials required for remote + Given the manifest 'dfetch.yaml' + """ + manifest: + version: '0.0' + + projects: + - name: private-repo + url: https://github.com/dfetch-org/test-repo-private.git + """ + When I run "dfetch check" + Then the output shows + """ + Dfetch (0.10.0) + >>>git ls-remote --heads --tags https://github.com/dfetch-org/test-repo-private.git<<< returned 128: + remote: Write access to repository not granted. + fatal: unable to access 'https://github.com/dfetch-org/test-repo-private.git/': The requested URL returned error: 403 + """ From 7c4070e5466bcb654577e23125219fe09a814f25 Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 3 Jan 2026 16:11:56 +0000 Subject: [PATCH 02/13] Avoid waiting for user input in git & svn commands Fixes #570 --- CHANGELOG.rst | 1 + dfetch/project/svn.py | 32 ++++++++++++------ dfetch/util/cmdline.py | 15 ++++----- dfetch/vcs/git.py | 57 +++++++++++++++++++++------------ features/check-git-repo.feature | 3 +- 5 files changed, 68 insertions(+), 40 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2c31e1d8..ab87fb82 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -24,6 +24,7 @@ Release 0.11.0 (unreleased) * Add more tests and documentation for patching (#888) * Restrict ``src`` to string only in schema (#888) * Don't consider ignored files for determining local changes (#350) +* Avoid waiting for user input in ``git`` & ``svn`` commands (#570) Release 0.10.0 (released 2025-03-12) ==================================== diff --git a/dfetch/project/svn.py b/dfetch/project/svn.py index b56ef289..0bb5c9d0 100644 --- a/dfetch/project/svn.py +++ b/dfetch/project/svn.py @@ -52,6 +52,7 @@ def externals() -> list[External]: logger, [ "svn", + "--non-interactive", "propget", "svn:externals", "-R", @@ -130,7 +131,7 @@ def _split_url(url: str, repo_root: str) -> tuple[str, str, str, str]: def check(self) -> bool: """Check if is SVN.""" try: - run_on_cmdline(logger, f"svn info {self.remote} --non-interactive") + run_on_cmdline(logger, ["svn", "info", self.remote, "--non-interactive"]) return True except SubprocessCommandError as exc: if exc.stdout.startswith("svn: E170013"): @@ -147,7 +148,7 @@ def check_path(path: str = ".") -> bool: """Check if is SVN.""" try: with in_directory(path): - run_on_cmdline(logger, "svn info --non-interactive") + run_on_cmdline(logger, ["svn", "info", "--non-interactive"]) return True except (SubprocessCommandError, RuntimeError): return False @@ -171,7 +172,9 @@ def _does_revision_exist(self, revision: str) -> bool: def _list_of_tags(self) -> list[str]: """Get list of all available tags.""" - result = run_on_cmdline(logger, f"svn ls --non-interactive {self.remote}/tags") + result = run_on_cmdline( + logger, ["svn", "ls", "--non-interactive", f"{self.remote}/tags"] + ) return [ str(tag).strip("/\r") for tag in result.stdout.decode().split("\n") if tag ] @@ -180,7 +183,7 @@ def _list_of_tags(self) -> list[str]: def list_tool_info() -> None: """Print out version information.""" try: - result = run_on_cmdline(logger, "svn --version") + result = run_on_cmdline(logger, ["svn", "--version", "--non-interactive"]) except RuntimeError as exc: logger.debug( f"Something went wrong trying to get the version of svn: {exc}" @@ -304,7 +307,9 @@ def _export(url: str, rev: str = "", dst: str = ".") -> None: def _files_in_path(url_path: str) -> list[str]: return [ str(line) - for line in run_on_cmdline(logger, f"svn list --non-interactive {url_path}") + for line in run_on_cmdline( + logger, ["svn", "list", "--non-interactive", url_path] + ) .stdout.decode() .splitlines() ] @@ -322,7 +327,7 @@ def _license_files(url_path: str) -> list[str]: def _get_info_from_target(target: str = "") -> dict[str, str]: try: result = run_on_cmdline( - logger, f"svn info --non-interactive {target.strip()}" + logger, ["svn", "info", "--non-interactive", target.strip()] ).stdout.decode() except SubprocessCommandError as exc: if exc.stdout.startswith("svn: E170013"): @@ -347,7 +352,7 @@ def _get_last_changed_revision(target: str) -> str: if os.path.isdir(target): last_digits = re.compile(r"(?P\d+)(?!.*\d)") version = run_on_cmdline( - logger, f"svnversion {target.strip()}" + logger, ["svnversion", target.strip()] ).stdout.decode() parsed_version = last_digits.search(version) @@ -358,7 +363,14 @@ def _get_last_changed_revision(target: str) -> str: return str( run_on_cmdline( logger, - f"svn info --non-interactive --show-item last-changed-revision {target.strip()}", + [ + "svn", + "info", + "--non-interactive", + "--show-item", + "last-changed-revision", + target.strip(), + ], ) .stdout.decode() .strip() @@ -415,7 +427,7 @@ def _untracked_files(path: str, ignore: Sequence[str]) -> list[str]: result = ( run_on_cmdline( logger, - ["svn", "status", path], + ["svn", "status", "--non-interactive", path], ) .stdout.decode() .splitlines() @@ -441,7 +453,7 @@ def ignored_files(path: str) -> Sequence[str]: result = ( run_on_cmdline( logger, - ["svn", "status", "--no-ignore", "."], + ["svn", "status", "--non-interactive", "--no-ignore", "."], ) .stdout.decode() .splitlines() diff --git a/dfetch/util/cmdline.py b/dfetch/util/cmdline.py index c302e77d..f4698c6b 100644 --- a/dfetch/util/cmdline.py +++ b/dfetch/util/cmdline.py @@ -3,7 +3,8 @@ import logging import os import subprocess # nosec -from typing import Any, Optional, Union # pylint: disable=unused-import +from collections.abc import Mapping +from typing import Any, Optional class SubprocessCommandError(Exception): @@ -36,16 +37,15 @@ def message(self) -> str: def run_on_cmdline( - logger: logging.Logger, cmd: Union[str, list[str]] + logger: logging.Logger, + cmd: list[str], + env: Optional[Mapping[str, str]] = None, ) -> "subprocess.CompletedProcess[Any]": """Run a command and log the output, and raise if something goes wrong.""" logger.debug(f"Running {cmd}") - if not isinstance(cmd, list): - cmd = cmd.split(" ") - try: - proc = subprocess.run(cmd, capture_output=True, check=True) # nosec + proc = subprocess.run(cmd, env=env, capture_output=True, check=True) # nosec except subprocess.CalledProcessError as exc: raise SubprocessCommandError( exc.cmd, @@ -54,8 +54,7 @@ def run_on_cmdline( exc.returncode, ) from exc except FileNotFoundError as exc: - cmd = cmd[0] - raise RuntimeError(f"{cmd} not available on system, please install") from exc + raise RuntimeError(f"{cmd[0]} not available on system, please install") from exc stdout, stderr = proc.stdout, proc.stderr diff --git a/dfetch/vcs/git.py b/dfetch/vcs/git.py index 94e4bfb3..9ece369d 100644 --- a/dfetch/vcs/git.py +++ b/dfetch/vcs/git.py @@ -30,7 +30,7 @@ class Submodule(NamedTuple): def get_git_version() -> tuple[str, str]: """Get the name and version of git.""" - result = run_on_cmdline(logger, "git --version") + result = run_on_cmdline(logger, ["git", "--version"]) tool, version = result.stdout.decode().strip().split("version", maxsplit=1) return (str(tool), str(version)) @@ -48,7 +48,11 @@ def is_git(self) -> bool: return True try: - run_on_cmdline(logger, f"git ls-remote --heads {self._remote}") + run_on_cmdline( + logger, + cmd=["git", "ls-remote", "--heads", self._remote], + env={"GIT_TERMINAL_PROMPT": "0"}, + ) return True except SubprocessCommandError as exc: if exc.returncode == 128 and "Could not resolve host" in exc.stdout: @@ -82,7 +86,9 @@ def get_default_branch(self) -> str: """Try to get the default branch or fallback to master.""" try: result = run_on_cmdline( - logger, f"git ls-remote --symref {self._remote} HEAD" + logger, + cmd=["git", "ls-remote", "--symref", self._remote, "HEAD"], + env={"GIT_TERMINAL_PROMPT": "0"}, ).stdout.decode() except SubprocessCommandError: logger.debug( @@ -101,7 +107,9 @@ def get_default_branch(self) -> str: @staticmethod def _ls_remote(remote: str) -> dict[str, str]: result = run_on_cmdline( - logger, f"git ls-remote --heads --tags {remote}" + logger, + cmd=["git", "ls-remote", "--heads", "--tags", remote], + env={"GIT_TERMINAL_PROMPT": "0"}, ).stdout.decode() info: dict[str, str] = {} @@ -156,12 +164,14 @@ def check_version_exists( temp_dir = tempfile.mkdtemp() exists = False with in_directory(temp_dir): - run_on_cmdline(logger, "git init") - run_on_cmdline(logger, f"git remote add origin {self._remote}") - run_on_cmdline(logger, "git checkout -b dfetch-local-branch") + run_on_cmdline(logger, ["git", "init"]) + run_on_cmdline(logger, ["git", "remote", "add", "origin", self._remote]) + run_on_cmdline(logger, ["git", "checkout", "-b", "dfetch-local-branch"]) try: run_on_cmdline( - logger, f"git fetch --dry-run --depth 1 origin {version}" + logger, + ["git", "fetch", "--dry-run", "--depth", "1", "origin", version], + env={"GIT_TERMINAL_PROMPT": "0"}, ) exists = True except SubprocessCommandError as exc: @@ -185,7 +195,11 @@ def is_git(self) -> bool: """Check if is git.""" try: with in_directory(self._path): - run_on_cmdline(logger, "git status") + run_on_cmdline( + logger, + ["git", "status"], + env={"GIT_TERMINAL_PROMPT": "0"}, + ) return True except (SubprocessCommandError, RuntimeError): return False @@ -209,12 +223,12 @@ def checkout_version( # pylint: disable=too-many-arguments ignore (Optional[Sequence[str]]): Optional sequence of glob patterns to ignore (relative to src) """ with in_directory(self._path): - run_on_cmdline(logger, "git init") - run_on_cmdline(logger, f"git remote add origin {remote}") - run_on_cmdline(logger, "git checkout -b dfetch-local-branch") + run_on_cmdline(logger, ["git", "init"]) + run_on_cmdline(logger, ["git", "remote", "add", "origin", remote]) + run_on_cmdline(logger, ["git", "checkout", "-b", "dfetch-local-branch"]) if src or ignore: - run_on_cmdline(logger, "git config core.sparsecheckout true") + run_on_cmdline(logger, ["git", "config", "core.sparsecheckout", "true"]) with open( ".git/info/sparse-checkout", "a", encoding="utf-8" ) as sparse_checkout_file: @@ -228,11 +242,17 @@ def checkout_version( # pylint: disable=too-many-arguments sparse_checkout_file.write("\n") sparse_checkout_file.write("\n".join(ignore_abs_paths)) - run_on_cmdline(logger, f"git fetch --depth 1 origin {version}") - run_on_cmdline(logger, "git reset --hard FETCH_HEAD") + run_on_cmdline( + logger, + ["git", "fetch", "--depth", "1", "origin", version], + env={"GIT_TERMINAL_PROMPT": "0"}, + ) + run_on_cmdline(logger, ["git", "reset", "--hard", "FETCH_HEAD"]) current_sha = ( - run_on_cmdline(logger, "git rev-parse HEAD").stdout.decode().strip() + run_on_cmdline(logger, ["git", "rev-parse", "HEAD"]) + .stdout.decode() + .strip() ) if src: @@ -305,10 +325,7 @@ def get_current_hash(self) -> str: def get_remote_url() -> str: """Get the url of the remote origin.""" try: - result = run_on_cmdline( - logger, - ["git", "remote", "get-url", "origin"], - ) + result = run_on_cmdline(logger, ["git", "remote", "get-url", "origin"]) decoded_result = str(result.stdout.decode()) except SubprocessCommandError: decoded_result = "" diff --git a/features/check-git-repo.feature b/features/check-git-repo.feature index ba496a6f..02ab3291 100644 --- a/features/check-git-repo.feature +++ b/features/check-git-repo.feature @@ -221,6 +221,5 @@ Feature: Checking dependencies from a git repository """ Dfetch (0.10.0) >>>git ls-remote --heads --tags https://github.com/dfetch-org/test-repo-private.git<<< returned 128: - remote: Write access to repository not granted. - fatal: unable to access 'https://github.com/dfetch-org/test-repo-private.git/': The requested URL returned error: 403 + fatal: could not read Username for 'https://github.com': terminal prompts disabled """ From 642f05cc52b435589ebeb17618f7fbd45d0c188f Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 3 Jan 2026 16:43:34 +0000 Subject: [PATCH 03/13] Extend git ssh command to run in BatchMode --- CHANGELOG.rst | 1 + dfetch/vcs/git.py | 53 +++++++++++++++++++++++++++++---- features/check-git-repo.feature | 22 ++++++++++++++ 3 files changed, 70 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ab87fb82..df37a3cf 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -25,6 +25,7 @@ Release 0.11.0 (unreleased) * Restrict ``src`` to string only in schema (#888) * Don't consider ignored files for determining local changes (#350) * Avoid waiting for user input in ``git`` & ``svn`` commands (#570) +* Extend git ssh command to run in BatchMode (#570) Release 0.10.0 (released 2025-03-12) ==================================== diff --git a/dfetch/vcs/git.py b/dfetch/vcs/git.py index 9ece369d..7ae25658 100644 --- a/dfetch/vcs/git.py +++ b/dfetch/vcs/git.py @@ -1,5 +1,6 @@ """Git specific implementation.""" +import functools import os import re import shutil @@ -35,6 +36,46 @@ def get_git_version() -> tuple[str, str]: return (str(tool), str(version)) +def _build_git_ssh_command() -> str: + """Returns a safe SSH command string for Git that enforces non-interactive mode. + + Respects existing GIT_SSH_COMMAND and git core.sshCommand. + """ + ssh_cmd = os.environ.get("GIT_SSH_COMMAND") + + if not ssh_cmd: + + try: + ssh_cmd = ( + run_on_cmdline( + logger, + ["git", "config", "--get", "core.sshCommand"], + ) + .stdout.decode("utf-8") + .strip() + ) + + except SubprocessCommandError: + ssh_cmd = None + + if not ssh_cmd: + ssh_cmd = "ssh" + + if "BatchMode=" not in ssh_cmd: + ssh_cmd += " -o BatchMode=yes" + + return ssh_cmd + + +@functools.lru_cache +def _extend_env_for_non_interactive_mode() -> dict[str, str]: + """Extend the environment vars for git running in non-interactive mode.""" + env = os.environ.copy() + env["GIT_TERMINAL_PROMPT"] = "0" + env["GIT_SSH_COMMAND"] = _build_git_ssh_command() + return env + + class GitRemote: """A remote git repo.""" @@ -51,7 +92,7 @@ def is_git(self) -> bool: run_on_cmdline( logger, cmd=["git", "ls-remote", "--heads", self._remote], - env={"GIT_TERMINAL_PROMPT": "0"}, + env=_extend_env_for_non_interactive_mode(), ) return True except SubprocessCommandError as exc: @@ -88,7 +129,7 @@ def get_default_branch(self) -> str: result = run_on_cmdline( logger, cmd=["git", "ls-remote", "--symref", self._remote, "HEAD"], - env={"GIT_TERMINAL_PROMPT": "0"}, + env=_extend_env_for_non_interactive_mode(), ).stdout.decode() except SubprocessCommandError: logger.debug( @@ -109,7 +150,7 @@ def _ls_remote(remote: str) -> dict[str, str]: result = run_on_cmdline( logger, cmd=["git", "ls-remote", "--heads", "--tags", remote], - env={"GIT_TERMINAL_PROMPT": "0"}, + env=_extend_env_for_non_interactive_mode(), ).stdout.decode() info: dict[str, str] = {} @@ -171,7 +212,7 @@ def check_version_exists( run_on_cmdline( logger, ["git", "fetch", "--dry-run", "--depth", "1", "origin", version], - env={"GIT_TERMINAL_PROMPT": "0"}, + env=_extend_env_for_non_interactive_mode(), ) exists = True except SubprocessCommandError as exc: @@ -198,7 +239,7 @@ def is_git(self) -> bool: run_on_cmdline( logger, ["git", "status"], - env={"GIT_TERMINAL_PROMPT": "0"}, + env=_extend_env_for_non_interactive_mode(), ) return True except (SubprocessCommandError, RuntimeError): @@ -245,7 +286,7 @@ def checkout_version( # pylint: disable=too-many-arguments run_on_cmdline( logger, ["git", "fetch", "--depth", "1", "origin", version], - env={"GIT_TERMINAL_PROMPT": "0"}, + env=_extend_env_for_non_interactive_mode(), ) run_on_cmdline(logger, ["git", "reset", "--hard", "FETCH_HEAD"]) diff --git a/features/check-git-repo.feature b/features/check-git-repo.feature index 02ab3291..bde18a8b 100644 --- a/features/check-git-repo.feature +++ b/features/check-git-repo.feature @@ -223,3 +223,25 @@ Feature: Checking dependencies from a git repository >>>git ls-remote --heads --tags https://github.com/dfetch-org/test-repo-private.git<<< returned 128: fatal: could not read Username for 'https://github.com': terminal prompts disabled """ + + Scenario: SSH issues + Given the manifest 'dfetch.yaml' + """ + manifest: + version: '0.0' + + projects: + - name: private-repo + url: git@github.com:dfetch-org/test-repo-private.git + """ + When I run "dfetch check" + Then the output shows + """ + Dfetch (0.10.0) + >>>git ls-remote --heads --tags git@github.com:dfetch-org/test-repo-private.git<<< returned 128: + Host key verification failed. + fatal: Could not read from remote repository. + + Please make sure you have the correct access rights + and the repository exists. + """ From ba6864d9978b3d305a0708bbee0ffe13a272842a Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 3 Jan 2026 16:51:25 +0000 Subject: [PATCH 04/13] Make windows credentials manager non-interactive --- dfetch/vcs/git.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/dfetch/vcs/git.py b/dfetch/vcs/git.py index 7ae25658..b17a0919 100644 --- a/dfetch/vcs/git.py +++ b/dfetch/vcs/git.py @@ -69,10 +69,16 @@ def _build_git_ssh_command() -> str: @functools.lru_cache def _extend_env_for_non_interactive_mode() -> dict[str, str]: - """Extend the environment vars for git running in non-interactive mode.""" + """Extend the environment vars for git running in non-interactive mode. + + See https://serverfault.com/a/1054253 for background info + """ env = os.environ.copy() env["GIT_TERMINAL_PROMPT"] = "0" env["GIT_SSH_COMMAND"] = _build_git_ssh_command() + + # https://stackoverflow.com/questions/37182847/how-do-i-disable-git-credential-manager-for-windows#answer-45513654 + env["GCM_INTERACTIVE"] = "never" return env From c0b2ac18579844d342368156e667fe62e22f45a8 Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 3 Jan 2026 16:53:10 +0000 Subject: [PATCH 05/13] Show some debug output --- dfetch/vcs/git.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dfetch/vcs/git.py b/dfetch/vcs/git.py index b17a0919..2210345f 100644 --- a/dfetch/vcs/git.py +++ b/dfetch/vcs/git.py @@ -63,6 +63,8 @@ def _build_git_ssh_command() -> str: if "BatchMode=" not in ssh_cmd: ssh_cmd += " -o BatchMode=yes" + else: + logger.debug(f'BatchMode already configured in "{ssh_cmd}"') return ssh_cmd From 61fa3c773df8a34352a3934ba60d3066058031ab Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 3 Jan 2026 17:04:37 +0000 Subject: [PATCH 06/13] Make feature tests less specific --- features/check-git-repo.feature | 10 ++-------- features/steps/generic_steps.py | 13 ++++++++++++- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/features/check-git-repo.feature b/features/check-git-repo.feature index bde18a8b..58695ef7 100644 --- a/features/check-git-repo.feature +++ b/features/check-git-repo.feature @@ -217,11 +217,10 @@ Feature: Checking dependencies from a git repository url: https://github.com/dfetch-org/test-repo-private.git """ When I run "dfetch check" - Then the output shows + Then the output starts with: """ Dfetch (0.10.0) >>>git ls-remote --heads --tags https://github.com/dfetch-org/test-repo-private.git<<< returned 128: - fatal: could not read Username for 'https://github.com': terminal prompts disabled """ Scenario: SSH issues @@ -235,13 +234,8 @@ Feature: Checking dependencies from a git repository url: git@github.com:dfetch-org/test-repo-private.git """ When I run "dfetch check" - Then the output shows + Then the output starts with: """ Dfetch (0.10.0) >>>git ls-remote --heads --tags git@github.com:dfetch-org/test-repo-private.git<<< returned 128: - Host key verification failed. - fatal: Could not read from remote repository. - - Please make sure you have the correct access rights - and the repository exists. """ diff --git a/features/steps/generic_steps.py b/features/steps/generic_steps.py index 40d940ab..5e32e170 100644 --- a/features/steps/generic_steps.py +++ b/features/steps/generic_steps.py @@ -247,8 +247,17 @@ def multisub(patterns: List[Tuple[Pattern[str], str]], text: str) -> str: return text +@then("the output starts with:") +def step_impl(context): + check_output(context, line_count=len(context.text.splitlines())) + + @then("the output shows") def step_impl(context): + check_output(context) + + +def check_output(context, line_count=None): expected_text = multisub( patterns=[ (git_hash, r"\1[commit hash]\2"), @@ -273,7 +282,9 @@ def step_impl(context): text=context.cmd_output, ) - diff = difflib.ndiff(actual_text.splitlines(), expected_text.splitlines()) + diff = difflib.ndiff( + actual_text.splitlines()[:line_count], expected_text.splitlines() + ) diffs = [x for x in diff if x[0] in ("+", "-")] if diffs: From 30035dcdf303f8586cfcfcd88c83bb77b44ea977 Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 3 Jan 2026 18:10:10 +0000 Subject: [PATCH 07/13] Fix mixup stdout/stderr --- dfetch/project/svn.py | 8 ++++---- dfetch/util/cmdline.py | 4 ++-- dfetch/vcs/git.py | 13 +++++-------- features/steps/generic_steps.py | 5 ++--- tests/test_git_vcs.py | 8 +++++++- 5 files changed, 20 insertions(+), 18 deletions(-) diff --git a/dfetch/project/svn.py b/dfetch/project/svn.py index 0bb5c9d0..09f8824e 100644 --- a/dfetch/project/svn.py +++ b/dfetch/project/svn.py @@ -134,10 +134,10 @@ def check(self) -> bool: run_on_cmdline(logger, ["svn", "info", self.remote, "--non-interactive"]) return True except SubprocessCommandError as exc: - if exc.stdout.startswith("svn: E170013"): + if exc.stderr.startswith("svn: E170013"): raise RuntimeError( f">>>{exc.cmd}<<< failed!\n" - + f"'{self.remote}' is not a valid URL or unreachable:\n{exc.stderr or exc.stdout}" + + f"'{self.remote}' is not a valid URL or unreachable:\n{exc.stdout or exc.stderr}" ) from exc return False except RuntimeError: @@ -330,10 +330,10 @@ def _get_info_from_target(target: str = "") -> dict[str, str]: logger, ["svn", "info", "--non-interactive", target.strip()] ).stdout.decode() except SubprocessCommandError as exc: - if exc.stdout.startswith("svn: E170013"): + if exc.stderr.startswith("svn: E170013"): raise RuntimeError( f">>>{exc.cmd}<<< failed!\n" - + f"'{target.strip()}' is not a valid URL or unreachable:\n{exc.stdout}" + + f"'{target.strip()}' is not a valid URL or unreachable:\n{exc.stderr or exc.stdout}" ) from exc raise diff --git a/dfetch/util/cmdline.py b/dfetch/util/cmdline.py index f4698c6b..d0de8245 100644 --- a/dfetch/util/cmdline.py +++ b/dfetch/util/cmdline.py @@ -25,8 +25,8 @@ def __init__( cmd_str: str = " ".join(cmd or []) self._message = f">>>{cmd_str}<<< returned {returncode}:{os.linesep}{stderr}" self.cmd = cmd_str - self.stderr = stdout - self.stdout = stderr + self.stdout = stdout + self.stderr = stderr self.returncode = returncode super().__init__(self._message) diff --git a/dfetch/vcs/git.py b/dfetch/vcs/git.py index 2210345f..a08aaefd 100644 --- a/dfetch/vcs/git.py +++ b/dfetch/vcs/git.py @@ -46,14 +46,11 @@ def _build_git_ssh_command() -> str: if not ssh_cmd: try: - ssh_cmd = ( - run_on_cmdline( - logger, - ["git", "config", "--get", "core.sshCommand"], - ) - .stdout.decode("utf-8") - .strip() + result = run_on_cmdline( + logger, + ["git", "config", "--get", "core.sshCommand"], ) + ssh_cmd = result.stdout.decode().strip() except SubprocessCommandError: ssh_cmd = None @@ -104,7 +101,7 @@ def is_git(self) -> bool: ) return True except SubprocessCommandError as exc: - if exc.returncode == 128 and "Could not resolve host" in exc.stdout: + if exc.returncode == 128 and "Could not resolve host" in exc.stderr: raise RuntimeError( f">>>{exc.cmd}<<< failed!\n" + f"'{self._remote}' is not a valid URL or unreachable:\n{exc.stderr or exc.stdout}" diff --git a/features/steps/generic_steps.py b/features/steps/generic_steps.py index 5e32e170..dc92b982 100644 --- a/features/steps/generic_steps.py +++ b/features/steps/generic_steps.py @@ -282,9 +282,8 @@ def check_output(context, line_count=None): text=context.cmd_output, ) - diff = difflib.ndiff( - actual_text.splitlines()[:line_count], expected_text.splitlines() - ) + actual_lines = actual_text.splitlines()[:line_count] + diff = difflib.ndiff(actual_lines, expected_text.splitlines()) diffs = [x for x in diff if x[0] in ("+", "-")] if diffs: diff --git a/tests/test_git_vcs.py b/tests/test_git_vcs.py index 3bf65b11..562a5166 100644 --- a/tests/test_git_vcs.py +++ b/tests/test_git_vcs.py @@ -3,6 +3,8 @@ # mypy: ignore-errors # flake8: noqa +import os +from subprocess import CompletedProcess from unittest.mock import patch import pytest @@ -14,12 +16,16 @@ @pytest.mark.parametrize( "name, cmd_result, expectation", [ - ("git repo", ["Yep!"], True), + ("git repo", [CompletedProcess(args=[], returncode=0, stdout="Yep!")], True), ("not a git repo", [SubprocessCommandError()], False), ("no git", [RuntimeError()], False), + ("somewhere.git", [], True), ], ) def test_remote_check(name, cmd_result, expectation): + + os.environ["GIT_SSH_COMMAND"] = "ssh" # prevents addition subprocess call + with patch("dfetch.vcs.git.run_on_cmdline") as run_on_cmdline_mock: run_on_cmdline_mock.side_effect = cmd_result From 8d6314274502b8a352218e63367ae8993537c9ca Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 3 Jan 2026 18:11:07 +0000 Subject: [PATCH 08/13] Remove non-interactive from local command --- dfetch/vcs/git.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dfetch/vcs/git.py b/dfetch/vcs/git.py index a08aaefd..06b49ece 100644 --- a/dfetch/vcs/git.py +++ b/dfetch/vcs/git.py @@ -244,7 +244,6 @@ def is_git(self) -> bool: run_on_cmdline( logger, ["git", "status"], - env=_extend_env_for_non_interactive_mode(), ) return True except (SubprocessCommandError, RuntimeError): From 862d35090c8bf4c107575a3c9a7c2be87ee63406 Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 3 Jan 2026 18:13:30 +0000 Subject: [PATCH 09/13] Add note about caching --- dfetch/vcs/git.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dfetch/vcs/git.py b/dfetch/vcs/git.py index 06b49ece..82051e57 100644 --- a/dfetch/vcs/git.py +++ b/dfetch/vcs/git.py @@ -66,6 +66,7 @@ def _build_git_ssh_command() -> str: return ssh_cmd +# As a cli tool, we can safely assume this remains stable during the runtime, caching for speed is better @functools.lru_cache def _extend_env_for_non_interactive_mode() -> dict[str, str]: """Extend the environment vars for git running in non-interactive mode. From 1fc55dcfa0203c7318cc6f613405973ad80937ed Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 3 Jan 2026 19:34:20 +0000 Subject: [PATCH 10/13] Add unit test for ssh BatchMode logic --- tests/test_git_vcs.py | 51 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/tests/test_git_vcs.py b/tests/test_git_vcs.py index 562a5166..94b6f828 100644 --- a/tests/test_git_vcs.py +++ b/tests/test_git_vcs.py @@ -5,7 +5,7 @@ import os from subprocess import CompletedProcess -from unittest.mock import patch +from unittest.mock import Mock, patch import pytest @@ -110,3 +110,52 @@ def test_ls_remote(): } assert info == expected + + +from dfetch.vcs.git import SubprocessCommandError, _build_git_ssh_command + + +@pytest.mark.parametrize( + "name, env_ssh, git_config_ssh, expected", + [ + ( + "env var present", + "ssh -i keyfile", + None, + "ssh -i keyfile -o BatchMode=yes", + ), + ( + "git config", + None, + "ssh -F configfile", + "ssh -F configfile -o BatchMode=yes", + ), + ("no env or git config", None, None, "ssh -o BatchMode=yes"), + ( + "env with bachmode", + "ssh -o BatchMode=yes", + None, + "ssh -o BatchMode=yes", + ), + ], +) +def test_build_git_ssh_command(name, env_ssh, git_config_ssh, expected): + + with patch.dict( + os.environ, {"GIT_SSH_COMMAND": env_ssh} if env_ssh else {}, clear=True + ): + mock_run_git_config = Mock() + if git_config_ssh is not None: + mock_run_git_config.return_value.stdout = git_config_ssh.encode() + else: + mock_run_git_config.side_effect = SubprocessCommandError() + + with patch("dfetch.vcs.git.run_on_cmdline", mock_run_git_config): + with patch("dfetch.vcs.git.logger") as mock_logger: + result = _build_git_ssh_command() + assert result == expected + + if "BatchMode=" in (env_ssh or git_config_ssh or ""): + mock_logger.debug.assert_called_once() + else: + mock_logger.debug.assert_not_called() From f2559fa9ccd88bdbbd71f324dd830be3a4474b9f Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 3 Jan 2026 19:36:52 +0000 Subject: [PATCH 11/13] Move check_output to other utils + add docstring --- features/steps/generic_steps.py | 78 ++++++++++++++++++--------------- 1 file changed, 42 insertions(+), 36 deletions(-) diff --git a/features/steps/generic_steps.py b/features/steps/generic_steps.py index dc92b982..14763a56 100644 --- a/features/steps/generic_steps.py +++ b/features/steps/generic_steps.py @@ -154,6 +154,48 @@ def list_dir(path): return result +def check_output(context, line_count=None): + """Check command output against expected text. + + Args: + context: Behave context with cmd_output and expected text + line_count: If set, compare only the first N lines of actual output + """ + expected_text = multisub( + patterns=[ + (git_hash, r"\1[commit hash]\2"), + (timestamp, "[timestamp]"), + (dfetch_title, ""), + (svn_error, "svn: EXXXXXX: "), + ], + text=context.text, + ) + + actual_text = multisub( + patterns=[ + (git_hash, r"\1[commit hash]\2"), + (timestamp, "[timestamp]"), + (ansi_escape, ""), + ( + re.compile(f"file:///{remote_server_path(context)}"), + "some-remote-server", + ), + (svn_error, "svn: EXXXXXX: "), + ], + text=context.cmd_output, + ) + + actual_lines = actual_text.splitlines()[:line_count] + diff = difflib.ndiff(actual_lines, expected_text.splitlines()) + + diffs = [x for x in diff if x[0] in ("+", "-")] + if diffs: + comp = "\n".join(diffs) + print(actual_text) + print(comp) + assert False, "Output not as expected!" + + @given('"{old}" is replaced with "{new}" in "{path}"') def step_impl(_, old: str, new: str, path: str): replace_in_file(path, old, new) @@ -257,42 +299,6 @@ def step_impl(context): check_output(context) -def check_output(context, line_count=None): - expected_text = multisub( - patterns=[ - (git_hash, r"\1[commit hash]\2"), - (timestamp, "[timestamp]"), - (dfetch_title, ""), - (svn_error, "svn: EXXXXXX: "), - ], - text=context.text, - ) - - actual_text = multisub( - patterns=[ - (git_hash, r"\1[commit hash]\2"), - (timestamp, "[timestamp]"), - (ansi_escape, ""), - ( - re.compile(f"file:///{remote_server_path(context)}"), - "some-remote-server", - ), - (svn_error, "svn: EXXXXXX: "), - ], - text=context.cmd_output, - ) - - actual_lines = actual_text.splitlines()[:line_count] - diff = difflib.ndiff(actual_lines, expected_text.splitlines()) - - diffs = [x for x in diff if x[0] in ("+", "-")] - if diffs: - comp = "\n".join(diffs) - print(actual_text) - print(comp) - assert False, "Output not as expected!" - - @then("the following projects are fetched") def step_impl(context): for project in context.table: From f5fcee4ada639a1037a9850e61d4d9a1a2d5a992 Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 3 Jan 2026 20:02:54 +0000 Subject: [PATCH 12/13] Fix typo's in unittest --- tests/test_git_vcs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_git_vcs.py b/tests/test_git_vcs.py index 94b6f828..8fdb4c31 100644 --- a/tests/test_git_vcs.py +++ b/tests/test_git_vcs.py @@ -24,7 +24,7 @@ ) def test_remote_check(name, cmd_result, expectation): - os.environ["GIT_SSH_COMMAND"] = "ssh" # prevents addition subprocess call + os.environ["GIT_SSH_COMMAND"] = "ssh" # prevents additional subprocess call with patch("dfetch.vcs.git.run_on_cmdline") as run_on_cmdline_mock: run_on_cmdline_mock.side_effect = cmd_result @@ -132,7 +132,7 @@ def test_ls_remote(): ), ("no env or git config", None, None, "ssh -o BatchMode=yes"), ( - "env with bachmode", + "env with batchmode", "ssh -o BatchMode=yes", None, "ssh -o BatchMode=yes", From 0a94fc2ccbfb4ffe22c294ae0e4bdb99d3e8b998 Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 3 Jan 2026 20:09:41 +0000 Subject: [PATCH 13/13] Move imports to top --- tests/test_git_vcs.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_git_vcs.py b/tests/test_git_vcs.py index 8fdb4c31..0089dcd4 100644 --- a/tests/test_git_vcs.py +++ b/tests/test_git_vcs.py @@ -10,7 +10,11 @@ import pytest from dfetch.util.cmdline import SubprocessCommandError -from dfetch.vcs.git import GitLocalRepo, GitRemote +from dfetch.vcs.git import ( + GitLocalRepo, + GitRemote, + _build_git_ssh_command, +) @pytest.mark.parametrize( @@ -112,9 +116,6 @@ def test_ls_remote(): assert info == expected -from dfetch.vcs.git import SubprocessCommandError, _build_git_ssh_command - - @pytest.mark.parametrize( "name, env_ssh, git_config_ssh, expected", [