Fetching any submodule in a subproject with submodules

Fixes #1013

Author: spoorcc

CHANGELOG.rst

@@ -1,3 +1,8 @@
+Release 0.13.0 (unreleased)
+====================================
+
+* Fetch git submodules in git subproject at pinned revision (#1015)
+
 Release 0.12.1 (released 2026-02-24)
 ====================================
 

dfetch/project/gitsubproject.py

@@ -8,7 +8,7 @@
 from dfetch.manifest.project import ProjectEntry
 from dfetch.manifest.version import Version
 from dfetch.project.subproject import SubProject
-from dfetch.util.util import safe_rmtree
+from dfetch.util.util import safe_rm, safe_rmtree
 from dfetch.vcs.git import GitLocalRepo, GitRemote, get_git_version
 
 logger = get_logger(__name__)
@@ -69,15 +69,22 @@ def _fetch_impl(self, version: Version) -> Version:
         ]
 
         local_repo = GitLocalRepo(self.local_path)
-        fetched_sha = local_repo.checkout_version(
+        fetched_sha, submodules = local_repo.checkout_version(
             remote=self.remote,
             version=rev_or_branch_or_tag,
             src=self.source,
-            must_keeps=license_globs,
+            must_keeps=license_globs + [".gitmodules"],
             ignore=self.ignore,
         )
 
+        for submodule in submodules:
+            self._log_project(
+                f'Found & fetched submodule "./{submodule.path}" '
+                f" ({submodule.url} @ {Version(tag=submodule.tag, branch=submodule.branch, revision=submodule.sha)})",
+            )
+
         safe_rmtree(os.path.join(self.local_path, local_repo.METADATA_DIR))
+        safe_rm(os.path.join(self.local_path, local_repo.GIT_MODULES_FILE))
 
         return self._determine_fetched_version(version, fetched_sha)
 

dfetch/util/util.py

@@ -44,10 +44,11 @@ def find_matching_files(directory: str, patterns: Sequence[str]) -> Iterator[Pat
 
 def safe_rm(path: str | Path) -> None:
     """Delete an file or directory safely."""
-    if os.path.isdir(path):
-        safe_rmtree(str(path))
-    else:
-        os.remove(path)
+    if os.path.exists(path):
+        if os.path.isdir(path):
+            safe_rmtree(str(path))
+        else:
+            os.remove(path)
 
 
 def safe_rmtree(path: str) -> None:

dfetch/vcs/git.py

@@ -233,6 +233,7 @@ class GitLocalRepo:
     """A git repository."""
 
     METADATA_DIR = ".git"
+    GIT_MODULES_FILE = ".gitmodules"
 
     def __init__(self, path: str | Path = ".") -> None:
         """Create a local git repo."""
@@ -258,7 +259,7 @@ def checkout_version(  # pylint: disable=too-many-arguments
         src: str | None = None,
         must_keeps: list[str] | None = None,
         ignore: Sequence[str] | None = None,
-    ) -> str:
+    ) -> tuple[str, list[Submodule]]:
         """Checkout a specific version from a given remote.
 
         Args:
@@ -295,6 +296,14 @@ def checkout_version(  # pylint: disable=too-many-arguments
             )
             run_on_cmdline(logger, ["git", "reset", "--hard", "FETCH_HEAD"])
 
+            run_on_cmdline(
+                logger,
+                ["git", "submodule", "update", "--init", "--recursive"],
+                env=_extend_env_for_non_interactive_mode(),
+            )
+
+            submodules = self.submodules()
+
             current_sha = (
                 run_on_cmdline(logger, ["git", "rev-parse", "HEAD"])
                 .stdout.decode()
@@ -304,7 +313,7 @@ def checkout_version(  # pylint: disable=too-many-arguments
             if src:
                 self.move_src_folder_up(remote, src)
 
-            return str(current_sha)
+            return str(current_sha), submodules
 
     def move_src_folder_up(self, remote: str, src: str) -> None:
         """Move the files from the src folder into the root of the project.

features/fetch-git-repo-with-submodule.feature

@@ -0,0 +1,34 @@
+Feature: Fetch projects with nested VCS dependencies
+
+    Some projects include nested version control dependencies
+    such as Git submodules or other externals
+    These dependencies must be fetched at the exact revision
+    pinned by the parent repository to ensure reproducibility
+
+    Scenario: A project with a git submodule is fetched at the pinned revision
+        Given the manifest 'dfetch.yaml' in MyProject
+            """
+            manifest:
+                version: 0.0
+                projects:
+                    - name: TF-PSA-Crypto
+                      url: https://github.com/Mbed-TLS/TF-PSA-Crypto.git
+                      tag: v1.0.0
+                      dst: ext/TF-PSA-Crypto
+                      ignore:
+                      - tests
+                      - scripts
+                      - programs
+                      - drivers
+                      - doxygen
+                      - docs
+            """
+        When I run "dfetch update"
+        Then the output shows
+            """
+            Dfetch (0.12.1)
+              TF-PSA-Crypto:
+              > Found & fetched submodule "./framework"  (https://github.com/Mbed-TLS/mbedtls-framework @ mbedtls-4.0.0_tf-psa-crypto-1.0.0)
+              > Fetched v1.0.0
+            """
+        Then 'MyProject/ext/TF-PSA-Crypto/framework/LICENSE' exists

features/steps/generic_steps.py

@@ -330,6 +330,11 @@ def step_impl(context, name):
         check_file(name, context.text)
 
 
+@then("'{name}' exists")
+def step_impl(_, name):
+    check_file_exists(name)
+
+
 def multisub(patterns: List[Tuple[Pattern[str], str]], text: str) -> str:
     """Apply a list of tuples that each contain a regex + replace string."""
     for pattern, replace in patterns: