Fetching any submodule in a subproject with submodules

Fixes #1013

Author: spoorcc

CHANGELOG.rst

@@ -1,3 +1,9 @@
+Release 0.13.0 (unreleased)
+====================================
+
+* Fetch git submodules in git subproject at pinned revision (#1015)
+* Add nested projects in subprojects to project report (#1015)
+
 Release 0.12.1 (released 2026-02-24)
 ====================================
 

dfetch/project/gitsubproject.py

@@ -7,8 +7,8 @@
 from dfetch.log import get_logger
 from dfetch.manifest.project import ProjectEntry
 from dfetch.manifest.version import Version
-from dfetch.project.subproject import SubProject
-from dfetch.util.util import safe_rmtree
+from dfetch.project.subproject import NestedProject, SubProject
+from dfetch.util.util import safe_rm, safe_rmtree
 from dfetch.vcs.git import GitLocalRepo, GitRemote, get_git_version
 
 logger = get_logger(__name__)
@@ -57,7 +57,7 @@ def list_tool_info() -> None:
             )
             SubProject._log_tool("git", "<not found in PATH>")
 
-    def _fetch_impl(self, version: Version) -> Version:
+    def _fetch_impl(self, version: Version) -> tuple[Version, list[NestedProject]]:
         """Get the revision of the remote and place it at the local path."""
         rev_or_branch_or_tag = self._determine_what_to_fetch(version)
 
@@ -69,17 +69,35 @@ def _fetch_impl(self, version: Version) -> Version:
         ]
 
         local_repo = GitLocalRepo(self.local_path)
-        fetched_sha = local_repo.checkout_version(
+        fetched_sha, submodules = local_repo.checkout_version(
             remote=self.remote,
             version=rev_or_branch_or_tag,
             src=self.source,
-            must_keeps=license_globs,
+            must_keeps=license_globs + [".gitmodules"],
             ignore=self.ignore,
         )
 
+        nested = []
+        for submodule in submodules:
+            self._log_project(
+                f'Found & fetched submodule "./{submodule.path}" '
+                f" ({submodule.url} @ {Version(tag=submodule.tag, branch=submodule.branch, revision=submodule.sha)})",
+            )
+            nested.append(
+                NestedProject(
+                    remote_url=submodule.url,
+                    destination=submodule.path,
+                    branch=submodule.branch,
+                    tag=submodule.tag,
+                    revision=submodule.sha,
+                    kind="git-submodule",
+                )
+            )
+
         safe_rmtree(os.path.join(self.local_path, local_repo.METADATA_DIR))
+        safe_rm(os.path.join(self.local_path, local_repo.GIT_MODULES_FILE))
 
-        return self._determine_fetched_version(version, fetched_sha)
+        return self._determine_fetched_version(version, fetched_sha), nested
 
     def _determine_what_to_fetch(self, version: Version) -> str:
         """Based on asked version, target to fetch."""

dfetch/project/metadata.py

@@ -16,6 +16,17 @@
 """
 
 
+class Nested(TypedDict):  # pylint: disable=too-many-ancestors
+    """Argument types for Metadata class construction."""
+
+    branch: str
+    tag: str
+    revision: str
+    remote_url: str
+    destination: str
+    kind: str
+
+
 class Options(TypedDict):  # pylint: disable=too-many-ancestors
     """Argument types for Metadata class construction."""
 
@@ -27,6 +38,7 @@ class Options(TypedDict):  # pylint: disable=too-many-ancestors
     destination: str
     hash: str
     patch: str | list[str]
+    nested: list["Nested"]
 
 
 class Metadata:
@@ -54,6 +66,8 @@ def __init__(self, kwargs: Options) -> None:
         # Historically only a single patch was allowed
         self._patch: list[str] = always_str_list(kwargs.get("patch", []))
 
+        self._nested: list[Nested] = kwargs.get("nested", [])
+
     @classmethod
     def from_project_entry(cls, project: ProjectEntry) -> "Metadata":
         """Create a metadata object from a project entry."""
@@ -66,6 +80,7 @@ def from_project_entry(cls, project: ProjectEntry) -> "Metadata":
             "last_fetch": datetime.datetime(2000, 1, 1, 0, 0, 0),
             "hash": "",
             "patch": project.patch,
+            "nested": [],
         }
         return cls(data)
 
@@ -77,13 +92,18 @@ def from_file(cls, path: str) -> "Metadata":
             return cls(data)
 
     def fetched(
-        self, version: Version, hash_: str = "", patch_: list[str] | None = None
+        self,
+        version: Version,
+        hash_: str = "",
+        patch_: list[str] | None = None,
+        nested: list[Nested] | None = None,
     ) -> None:
         """Update metadata."""
         self._last_fetch = datetime.datetime.now()
         self._version = version
         self._hash = hash_
         self._patch = patch_ or []
+        self._nested = nested or []
 
     @property
     def version(self) -> Version:
@@ -129,6 +149,11 @@ def patch(self) -> list[str]:
         """The list of applied patches as stored in the metadata."""
         return self._patch
 
+    @property
+    def nested(self) -> list[Nested]:
+        """The list of nested projects as stored in the metadata."""
+        return self._nested
+
     @property
     def path(self) -> str:
         """Path to metadata file."""
@@ -152,12 +177,13 @@ def __eq__(self, other: object) -> bool:
                 other._version.revision == self._version.revision,
                 other.hash == self.hash,
                 other.patch == self.patch,
+                other.nested == self.nested,
             ]
         )
 
     def dump(self) -> None:
         """Dump metadata file to correct path."""
-        metadata = {
+        metadata: dict[str, dict[str, str | list[str] | list[Nested]]] = {
             "dfetch": {
                 "remote_url": self.remote_url,
                 "branch": self._version.branch,
@@ -169,6 +195,9 @@ def dump(self) -> None:
             }
         }
 
+        if self.nested:
+            metadata["dfetch"]["nested"] = self.nested
+
         with open(self.path, "w+", encoding="utf-8") as metadata_file:
             metadata_file.write(DONT_EDIT_WARNING)
             yaml.dump(metadata, metadata_file)

dfetch/project/subproject.py

@@ -5,19 +5,42 @@
 import pathlib
 from abc import ABC, abstractmethod
 from collections.abc import Sequence
+from typing import NamedTuple
 
 from dfetch.log import get_logger
 from dfetch.manifest.project import ProjectEntry
 from dfetch.manifest.version import Version
 from dfetch.project.abstract_check_reporter import AbstractCheckReporter
-from dfetch.project.metadata import Metadata
+from dfetch.project.metadata import Metadata, Nested
 from dfetch.util.util import hash_directory, safe_rm
 from dfetch.util.versions import latest_tag_from_list
 from dfetch.vcs.patch import Patch
 
 logger = get_logger(__name__)
 
 
+class NestedProject(NamedTuple):
+    """Information about a nested project."""
+
+    destination: str
+    remote_url: str
+    branch: str
+    tag: str
+    revision: str
+    kind: str
+
+    def to_nested(self) -> Nested:
+        """Convert this nested project to a Nested object."""
+        return Nested(
+            destination=self.destination,
+            remote_url=self.remote_url,
+            branch=self.branch,
+            tag=self.tag,
+            revision=self.revision,
+            kind=self.kind,
+        )
+
+
 class SubProject(ABC):
     """Abstract SubProject object.
 
@@ -125,7 +148,7 @@ def update(
             f"Fetching {to_fetch}",
             enabled=self._show_animations,
         ):
-            actually_fetched = self._fetch_impl(to_fetch)
+            actually_fetched, nested = self._fetch_impl(to_fetch)
         self._log_project(f"Fetched {actually_fetched}")
 
         applied_patches = self._apply_patches(patch_count)
@@ -134,6 +157,7 @@ def update(
             actually_fetched,
             hash_=hash_directory(self.local_path, skiplist=[self.__metadata.FILENAME]),
             patch_=applied_patches,
+            nested=[nested.to_nested() for nested in nested],
         )
 
         logger.debug(f"Writing repo metadata to: {self.__metadata.path}")
@@ -381,7 +405,7 @@ def _are_there_local_changes(self, files_to_ignore: Sequence[str]) -> bool:
         )
 
     @abstractmethod
-    def _fetch_impl(self, version: Version) -> Version:
+    def _fetch_impl(self, version: Version) -> tuple[Version, list[NestedProject]]:
         """Fetch the given version of the subproject, should be implemented by the child class."""
 
     @abstractmethod

dfetch/project/svnsubproject.py

@@ -7,7 +7,7 @@
 from dfetch.log import get_logger
 from dfetch.manifest.project import ProjectEntry
 from dfetch.manifest.version import Version
-from dfetch.project.subproject import SubProject
+from dfetch.project.subproject import NestedProject, SubProject
 from dfetch.util.util import (
     find_matching_files,
     find_non_matching_files,
@@ -106,7 +106,7 @@ def _remove_ignored_files(self) -> None:
             if not (file_or_dir.is_file() and self.is_license_file(file_or_dir.name)):
                 safe_rm(file_or_dir)
 
-    def _fetch_impl(self, version: Version) -> Version:
+    def _fetch_impl(self, version: Version) -> tuple[Version, list[NestedProject]]:
         """Get the revision of the remote and place it at the local path."""
         branch, branch_path, revision = self._determine_what_to_fetch(version)
         rev_arg = f"--revision {revision}" if revision else ""
@@ -147,7 +147,7 @@ def _fetch_impl(self, version: Version) -> Version:
         if self.ignore:
             self._remove_ignored_files()
 
-        return Version(tag=version.tag, branch=branch, revision=revision)
+        return Version(tag=version.tag, branch=branch, revision=revision), []
 
     @staticmethod
     def _parse_file_pattern(complete_path: str) -> tuple[str, str]:

dfetch/reporting/stdout_reporter.py

@@ -40,6 +40,18 @@ def add_project(
                 "    licenses", ",".join(license.name for license in licenses)
             )
 
+            if metadata.nested:
+                logger.info("")
+                logger.info("      nested:")
+            for nested in metadata.nested:
+                logger.print_info_field("    - path", nested.get("destination", ""))
+                logger.print_info_field("      url", nested.get("remote_url", ""))
+                logger.print_info_field("      branch", nested.get("branch", ""))
+                logger.print_info_field("      tag", nested.get("tag", ""))
+                logger.print_info_field("      revision", nested.get("revision", ""))
+                logger.print_info_field("      kind", nested.get("kind", ""))
+                logger.info("")
+
         except FileNotFoundError:
             logger.print_info_field("    last fetch", "never")
 

dfetch/util/util.py

@@ -44,10 +44,11 @@ def find_matching_files(directory: str, patterns: Sequence[str]) -> Iterator[Pat
 
 def safe_rm(path: str | Path) -> None:
     """Delete an file or directory safely."""
-    if os.path.isdir(path):
-        safe_rmtree(str(path))
-    else:
-        os.remove(path)
+    if os.path.exists(path):
+        if os.path.isdir(path):
+            safe_rmtree(str(path))
+        else:
+            os.remove(path)
 
 
 def safe_rmtree(path: str) -> None:

dfetch/vcs/git.py

@@ -233,6 +233,7 @@ class GitLocalRepo:
     """A git repository."""
 
     METADATA_DIR = ".git"
+    GIT_MODULES_FILE = ".gitmodules"
 
     def __init__(self, path: str | Path = ".") -> None:
         """Create a local git repo."""
@@ -258,7 +259,7 @@ def checkout_version(  # pylint: disable=too-many-arguments
         src: str | None = None,
         must_keeps: list[str] | None = None,
         ignore: Sequence[str] | None = None,
-    ) -> str:
+    ) -> tuple[str, list[Submodule]]:
         """Checkout a specific version from a given remote.
 
         Args:
@@ -295,6 +296,14 @@ def checkout_version(  # pylint: disable=too-many-arguments
             )
             run_on_cmdline(logger, ["git", "reset", "--hard", "FETCH_HEAD"])
 
+            run_on_cmdline(
+                logger,
+                ["git", "submodule", "update", "--init", "--recursive"],
+                env=_extend_env_for_non_interactive_mode(),
+            )
+
+            submodules = self.submodules()
+
             current_sha = (
                 run_on_cmdline(logger, ["git", "rev-parse", "HEAD"])
                 .stdout.decode()
@@ -304,7 +313,7 @@ def checkout_version(  # pylint: disable=too-many-arguments
             if src:
                 self.move_src_folder_up(remote, src)
 
-            return str(current_sha)
+            return str(current_sha), submodules
 
     def move_src_folder_up(self, remote: str, src: str) -> None:
         """Move the files from the src folder into the root of the project.