Enhance temporary package directory handling and debugging in BuildManager and SubfolderBuildConfig

This commit adds detailed debug logging to the BuildManager and SubfolderBuildConfig classes, improving visibility into the temporary package directory creation and file copying processes. It introduces checks for Python files in the temporary directory, including recursive verification, and implements exclude pattern handling during directory copying. Additionally, a new test is added to ensure exclude patterns do not incorrectly match pytest temporary directories, addressing a regression issue. These changes enhance the robustness and traceability of the build process.

Author: alelom

src/python_package_folder/manager.py

@@ -1153,12 +1153,24 @@ def build():
                         f"Temporary package directory does not exist: {temp_dir}. "
                         "This should have been created during prepare_build()."
                     )
+                # Debug: List all files in temp directory
+                all_files = list(temp_dir.rglob("*"))
+                all_files = [f for f in all_files if f.is_file()]
+                print(
+                    f"DEBUG: Temp package directory {temp_dir} contains {len(all_files)} files: "
+                    f"{[str(f.relative_to(temp_dir)) for f in all_files[:10]]}",
+                    file=sys.stderr,
+                )
                 # Verify it contains Python files
                 py_files = list(temp_dir.glob("*.py"))
                 if not py_files:
-                    raise RuntimeError(
-                        f"Temporary package directory exists but contains no Python files: {temp_dir}"
-                    )
+                    # Also check recursively
+                    py_files = list(temp_dir.rglob("*.py"))
+                    if not py_files:
+                        raise RuntimeError(
+                            f"Temporary package directory exists but contains no Python files: {temp_dir}. "
+                            f"Total files found: {len(all_files)}"
+                        )
                 # Verify __init__.py exists
                 if not (temp_dir / "__init__.py").exists():
                     raise RuntimeError(

src/python_package_folder/subfolder_build.py

@@ -134,6 +134,7 @@ def _create_temp_package_directory(self) -> None:
         This way, hatchling will install it with the correct name without needing force-include.
         """
         if not self.package_name:
+            print("DEBUG: No package_name provided, skipping temp package directory creation", file=sys.stderr)
             return
 
         # Convert package name (with hyphens) to import name (with underscores)
@@ -144,13 +145,21 @@ def _create_temp_package_directory(self) -> None:
         # This way, hatchling will install it with the correct name
         import_name_dir = self.project_root / import_name
 
+        print(
+            f"DEBUG: Creating temporary package directory: {import_name_dir} "
+            f"(from src_dir: {self.src_dir}, import name: {import_name})",
+            file=sys.stderr,
+        )
+        
         # Check if the directory already exists and is the correct one
         if import_name_dir.exists() and import_name_dir == self._temp_package_dir:
             # Directory already exists and is the correct one, no need to recreate
+            print(f"DEBUG: Temporary package directory already exists: {import_name_dir}", file=sys.stderr)
             return
 
         # Remove if it already exists (from a previous build)
         if import_name_dir.exists():
+            print(f"DEBUG: Removing existing temporary package directory: {import_name_dir}", file=sys.stderr)
             shutil.rmtree(import_name_dir)
 
         # Copy the entire source directory contents directly to the import name directory
@@ -171,9 +180,38 @@ def _create_temp_package_directory(self) -> None:
             self._temp_package_dir = None
             return
 
+        # Get exclude patterns from parent pyproject.toml
+        exclude_patterns = []
+        original_pyproject = self.project_root / "pyproject.toml"
+        if original_pyproject.exists():
+            exclude_patterns = read_exclude_patterns(original_pyproject)
+            print(
+                f"DEBUG: Using exclude patterns for temp directory copy: {exclude_patterns}",
+                file=sys.stderr,
+            )
+        
+        # Check if src_dir has any files before copying
+        src_files = list(self.src_dir.rglob("*"))
+        src_files = [f for f in src_files if f.is_file()]
+        print(
+            f"DEBUG: Source directory {self.src_dir} contains {len(src_files)} files before copy",
+            file=sys.stderr,
+        )
+        
+        # Use a copy method that respects exclude patterns and handles missing directories
         try:
-            shutil.copytree(self.src_dir, import_name_dir)
+            print(f"DEBUG: Starting copy from {self.src_dir} to {import_name_dir}", file=sys.stderr)
+            self._copytree_excluding_patterns(self.src_dir, import_name_dir, exclude_patterns)
             self._temp_package_dir = import_name_dir
+            
+            # Verify files were copied
+            copied_files = list(import_name_dir.rglob("*"))
+            copied_files = [f for f in copied_files if f.is_file()]
+            print(
+                f"DEBUG: After copy, temp directory {import_name_dir} contains {len(copied_files)} files",
+                file=sys.stderr,
+            )
+            
             print(
                 f"Created temporary package directory: {import_name_dir} "
                 f"(import name: {import_name})"
@@ -183,8 +221,141 @@ def _create_temp_package_directory(self) -> None:
                 f"Warning: Could not create temporary package directory: {e}",
                 file=sys.stderr,
             )
+            import traceback
+            print(f"DEBUG: Traceback: {traceback.format_exc()}", file=sys.stderr)
             # Fall back to using src_dir directly
             self._temp_package_dir = None
+    
+    def _copytree_excluding_patterns(self, src: Path, dst: Path, exclude_patterns: list[str]) -> None:
+        """
+        Copy a directory tree, excluding certain patterns and handling missing directories gracefully.
+        
+        This is similar to BuildManager._copytree_excluding but works without needing
+        the BuildManager instance. It respects exclude patterns and skips missing directories
+        (e.g., broken symlinks or already-excluded directories).
+        
+        Args:
+            src: Source directory
+            dst: Destination directory
+            exclude_patterns: List of patterns to exclude (e.g., ['_SS', '__SS', '.*_test.*'])
+        """
+        default_patterns = [
+            "_SS",
+            "__SS",
+            "_sandbox",
+            "__sandbox",
+            "_skip",
+            "__skip",
+            "_test",
+            "__test__",
+        ]
+        all_exclude_patterns = default_patterns + exclude_patterns
+        
+        def should_exclude(path: Path) -> bool:
+            """Check if a path should be excluded."""
+            import re
+            # Only check parts of the path relative to src_dir, not the entire absolute path
+            # This prevents matching test directory names or other parts outside the source
+            try:
+                rel_path = path.relative_to(src)
+                # Check each component of the relative path
+                for part in rel_path.parts:
+                    # Check if any part matches an exclusion pattern
+                    for pattern in all_exclude_patterns:
+                        # Determine if pattern is a regex (contains regex special characters)
+                        is_regex = any(c in pattern for c in ['.', '*', '+', '?', '^', '$', '[', ']', '(', ')', '{', '}', '|', '\\'])
+                        
+                        if is_regex:
+                            # Use regex matching for patterns like '.*_test.*'
+                            try:
+                                if re.search(pattern, part):
+                                    print(f"DEBUG: Excluding {path} (part '{part}' matches regex pattern '{pattern}')", file=sys.stderr)
+                                    return True
+                            except re.error:
+                                # Invalid regex, fall back to simple string matching
+                                if part == pattern or part.startswith(pattern):
+                                    print(f"DEBUG: Excluding {path} (part '{part}' matches pattern '{pattern}')", file=sys.stderr)
+                                    return True
+                        else:
+                            # Simple string matching for patterns like '_SS'
+                            if part == pattern or part.startswith(pattern):
+                                print(f"DEBUG: Excluding {path} (part '{part}' matches pattern '{pattern}')", file=sys.stderr)
+                                return True
+            except ValueError:
+                # Path is not relative to src, check the name only
+                for pattern in all_exclude_patterns:
+                    is_regex = any(c in pattern for c in ['.', '*', '+', '?', '^', '$', '[', ']', '(', ')', '{', '}', '|', '\\'])
+                    if is_regex:
+                        try:
+                            if re.search(pattern, path.name):
+                                return True
+                        except re.error:
+                            if path.name == pattern or path.name.startswith(pattern):
+                                return True
+                    else:
+                        if path.name == pattern or path.name.startswith(pattern):
+                            return True
+            return False
+        
+        # Create destination directory
+        dst.mkdir(parents=True, exist_ok=True)
+        
+        # Copy files and subdirectories, excluding patterns
+        copied_count = 0
+        excluded_count = 0
+        skipped_count = 0
+        try:
+            items = list(src.iterdir())
+            print(f"DEBUG: Copying from {src} to {dst}, found {len(items)} items", file=sys.stderr)
+            for item in items:
+                if should_exclude(item):
+                    print(f"DEBUG: Excluding {item} from temp package directory copy", file=sys.stderr)
+                    excluded_count += 1
+                    continue
+                
+                src_item = src / item.name
+                dst_item = dst / item.name
+                
+                # Skip if source doesn't exist (broken symlink, already deleted, etc.)
+                if not src_item.exists():
+                    print(
+                        f"DEBUG: Skipping non-existent item: {src_item}",
+                        file=sys.stderr,
+                    )
+                    skipped_count += 1
+                    continue
+                
+                if src_item.is_file():
+                    try:
+                        shutil.copy2(src_item, dst_item)
+                        copied_count += 1
+                        print(f"DEBUG: Copied file {src_item} -> {dst_item}", file=sys.stderr)
+                    except (OSError, IOError) as e:
+                        print(
+                            f"DEBUG: Could not copy file {src_item}: {e}, skipping",
+                            file=sys.stderr,
+                        )
+                        skipped_count += 1
+                        continue
+                elif src_item.is_dir():
+                    try:
+                        self._copytree_excluding_patterns(src_item, dst_item, exclude_patterns)
+                        copied_count += 1
+                        print(f"DEBUG: Copied directory {src_item} -> {dst_item}", file=sys.stderr)
+                    except (OSError, IOError) as e:
+                        print(
+                            f"DEBUG: Could not copy directory {src_item}: {e}, skipping",
+                            file=sys.stderr,
+                        )
+                        skipped_count += 1
+                        continue
+            print(
+                f"DEBUG: Copy summary: {copied_count} items copied, {excluded_count} excluded, {skipped_count} skipped",
+                file=sys.stderr,
+            )
+        except (OSError, IOError) as e:
+            # If we can't even iterate the source directory, that's a problem
+            raise RuntimeError(f"Cannot iterate source directory {src}: {e}") from e
 
     def _get_package_structure(self) -> tuple[str, list[str]]:
         """
@@ -198,6 +369,13 @@ def _get_package_structure(self) -> tuple[str, list[str]]:
         # Use temporary package directory if it exists, otherwise use src_dir
         package_dir = self._temp_package_dir if self._temp_package_dir and self._temp_package_dir.exists() else self.src_dir
 
+        print(
+            f"DEBUG: _get_package_structure: temp_package_dir={self._temp_package_dir}, "
+            f"exists={self._temp_package_dir.exists() if self._temp_package_dir else False}, "
+            f"using package_dir={package_dir}",
+            file=sys.stderr,
+        )
+        
         # Check if package_dir itself is a package (has __init__.py)
         has_init = (package_dir / "__init__.py").exists()
 
@@ -211,6 +389,16 @@ def _get_package_structure(self) -> tuple[str, list[str]]:
             packages_path = str(rel_path).replace("\\", "/")
         except ValueError:
             packages_path = None
+            print(
+                f"DEBUG: Could not calculate relative path from {self.project_root} to {package_dir}",
+                file=sys.stderr,
+            )
+
+        print(
+            f"DEBUG: _get_package_structure returning: packages_path={packages_path}, "
+            f"has_init={has_init}, has_py_files={has_py_files}",
+            file=sys.stderr,
+        )
 
         # If package_dir has Python files but no __init__.py, we need to make it a package
         # or include it as a module directory
@@ -518,8 +706,28 @@ def create_temp_pyproject(self) -> Path | None:
         # This will copy the __init__.py we just created (if any)
         self._create_temp_package_directory()
 
+        # Log the result of temp directory creation
+        if self._temp_package_dir and self._temp_package_dir.exists():
+            py_files = list(self._temp_package_dir.glob("*.py"))
+            print(
+                f"DEBUG: Temp package directory created successfully: {self._temp_package_dir}, "
+                f"contains {len(py_files)} Python files",
+                file=sys.stderr,
+            )
+        else:
+            print(
+                f"WARNING: Temp package directory was NOT created. "
+                f"Will fall back to using src_dir: {self.src_dir}",
+                file=sys.stderr,
+            )
+        
         # Determine which directory to use (temp package dir or src_dir)
         package_dir = self._temp_package_dir if self._temp_package_dir and self._temp_package_dir.exists() else self.src_dir
+        print(
+            f"DEBUG: Using package_dir for build: {package_dir} "
+            f"(temp_dir={self._temp_package_dir}, src_dir={self.src_dir})",
+            file=sys.stderr,
+        )
 
         # Read the original pyproject.toml
         original_pyproject = self.project_root / "pyproject.toml"

tests/test_subfolder_build.py

@@ -1130,6 +1130,85 @@ def test_temp_package_directory_handles_existing_directory(
 
         config.restore()
 
+    def test_temp_package_directory_respects_exclude_patterns_without_matching_test_dirs(
+        self, test_project_with_pyproject: Path
+    ) -> None:
+        """
+        Test that exclude patterns don't incorrectly match pytest temp directory names.
+        
+        This test ensures that exclude patterns like '.*test_.*' only match files/directories
+        within the source directory, not the pytest temp directory name (e.g., 
+        'test_real_world_ml_drawing_assistant_data_scenario').
+        
+        This is a regression test for the bug where all files were excluded because the
+        exclude pattern matching checked the entire absolute path instead of just the
+        relative path within src_dir.
+        """
+        project_root = test_project_with_pyproject
+        subfolder = project_root / "subfolder"
+        
+        # Create files that should NOT be excluded
+        (subfolder / "__init__.py").write_text("# Package init")
+        (subfolder / "module.py").write_text("def func(): pass")
+        (subfolder / "utils.py").write_text("def util(): pass")
+        
+        # Create a file that SHOULD be excluded (matches pattern)
+        (subfolder / "test_helper.py").write_text("def test(): pass")
+        (subfolder / "_SS").mkdir()
+        (subfolder / "_SS" / "excluded.py").write_text("# Should be excluded")
+        
+        # Update pyproject.toml with exclude patterns that could match test directory names
+        pyproject_path = project_root / "pyproject.toml"
+        pyproject_content = pyproject_path.read_text()
+        # Add exclude patterns including ones that could match pytest temp dirs
+        if "[tool.python-package-folder]" not in pyproject_content:
+            pyproject_content += "\n[tool.python-package-folder]\n"
+        pyproject_content += 'exclude-patterns = ["_SS", "__SS", ".*_test.*", ".*test_.*", "sandbox"]\n'
+        pyproject_path.write_text(pyproject_content)
+        
+        config = SubfolderBuildConfig(
+            project_root=project_root,
+            src_dir=subfolder,
+            version="1.0.0",
+            package_name="my-package",
+        )
+        
+        # Create temp pyproject (which creates temp package directory with exclude patterns)
+        config.create_temp_pyproject()
+        
+        temp_package_dir = config._temp_package_dir
+        assert temp_package_dir is not None
+        assert temp_package_dir.exists()
+        
+        # Files that should NOT be excluded should be copied
+        assert (temp_package_dir / "__init__.py").exists(), (
+            "__init__.py should be copied (doesn't match exclude patterns)"
+        )
+        assert (temp_package_dir / "module.py").exists(), (
+            "module.py should be copied (doesn't match exclude patterns)"
+        )
+        assert (temp_package_dir / "utils.py").exists(), (
+            "utils.py should be copied (doesn't match exclude patterns)"
+        )
+        
+        # Files that SHOULD be excluded should NOT be copied
+        assert not (temp_package_dir / "test_helper.py").exists(), (
+            "test_helper.py should be excluded (matches '.*test_.*' pattern)"
+        )
+        assert not (temp_package_dir / "_SS").exists(), (
+            "_SS directory should be excluded (matches '_SS' pattern)"
+        )
+        
+        # Verify at least some files were copied (this would fail if all files were excluded)
+        all_files = list(temp_package_dir.rglob("*"))
+        all_files = [f for f in all_files if f.is_file()]
+        assert len(all_files) >= 3, (
+            f"Expected at least 3 files to be copied, but only found {len(all_files)}. "
+            f"This suggests exclude patterns are incorrectly matching test directory names."
+        )
+        
+        config.restore()
+
 
 class TestWheelPackaging:
     """Tests to verify that wheels are correctly packaged with the right directory structure."""