Fix multi-file DICOM pyramids leading to failed items for lower-res files

src/aignostics/CLAUDE.md

@@ -246,7 +246,7 @@ aignostics application run submit --application-id heta --files slide.svs
 aignostics dataset download --collection-id TCGA-LUAD --output-dir ./data
 
 # Get WSI info
-aignostics wsi info slide.svs
+aignostics wsi inspect slide.svs
 ```
 
 ## GUI Launch

src/aignostics/application/CLAUDE.md

@@ -163,6 +163,82 @@ APPLICATION_RUN_UPLOAD_CHUNK_SIZE = 1024 * 1024  # 1MB
 APPLICATION_RUN_DOWNLOAD_SLEEP_SECONDS = 5  # Wait between status checks
 ```
 
+### DICOM Pyramid Handling
+
+**Multi-File DICOM Pyramid Filtering (`_service.py`):**
+
+The service automatically handles multi-file DICOM pyramids (whole slide images stored across multiple DICOM instances) by selecting only the highest resolution file from each pyramid. This prevents redundant processing since OpenSlide can automatically find related pyramid files in the same directory.
+```python
+@staticmethod
+def _filter_dicom_pyramid_files(source_directory: Path) -> set[Path]:
+    """Filter DICOM files to keep only one representative per pyramid.
+
+    For multi-file DICOM pyramids (WSI images split across multiple instances),
+    keeps only the highest resolution file. Excludes segmentations, annotations,
+    thumbnails, and other non-image DICOM files.
+
+    Filtering Strategy:
+    1. SOPClassUID filtering - Only process VL Whole Slide Microscopy Image Storage
+       - Include: 1.2.840.10008.5.1.4.1.1.77.1.6 (VL WSI)
+       - Exclude: 1.2.840.10008.5.1.4.1.1.66.4 (Segmentation Storage)
+       - Exclude: Other non-WSI DICOM types
+
+    2. ImageType filtering - Exclude auxiliary images
+       - THUMBNAIL, LABEL, OVERVIEW, MACRO, ANNOTATION, LOCALIZER
+
+    3. PyramidUID grouping - Group multi-file pyramids
+       - Files with same PyramidUID are part of one logical WSI
+       - Files without PyramidUID are treated as standalone WSIs
+
+    4. Resolution selection - Keep highest resolution per pyramid
+       - Based on TotalPixelMatrixRows × TotalPixelMatrixColumns
+       - Excludes all lower resolution levels
+
+    Used automatically in: generate_metadata_from_source_directory()
+    """
+```
+
+**Key Behaviors:**
+
+- **SOPClassUID validation**: Only processes VL Whole Slide Microscopy Image Storage files (1.2.840.10008.5.1.4.1.1.77.1.6)
+- **Non-WSI exclusion**: Automatically excludes segmentations (1.2.840.10008.5.1.4.1.1.66.4), annotations, and other DICOM object types
+- **ImageType filtering**: Excludes THUMBNAIL, LABEL, OVERVIEW, MACRO, ANNOTATION, and LOCALIZER image types
+- **PyramidUID grouping**: Groups files by PyramidUID (DICOM tag identifying multi-resolution pyramids)
+- **Resolution selection**: For each pyramid, keeps only the file with largest TotalPixelMatrixRows × TotalPixelMatrixColumns
+- **Standalone handling**: Files without PyramidUID are treated as standalone WSI images and preserved
+- **Graceful degradation**: Files with missing attributes are logged and treated as standalone (not excluded)
+- **Debug logging**: Excluded files are logged at DEBUG level with pyramid/exclusion details
+- **Pre-processing filter**: Filtering occurs before metadata generation, checksum calculation, or upload
+- **Format-specific**: Only affects DICOM files (`.dcm` extension); other formats (SVS, TIFF) unaffected
+
+**DICOM WSI Structure:**
+
+In the DICOM Whole Slide Imaging standard:
+- **PyramidUID**: Uniquely identifies a single multi-resolution pyramid that may span multiple files
+- **SeriesInstanceUID**: Groups related images (may include multiple pyramids, thumbnails, labels)
+- **TotalPixelMatrixRows/Columns**: Represents full image dimensions at the highest resolution level
+
+**Example Scenario:**
+```
+Input Directory:
+├── pyramid_level_0.dcm    (10000×10000 px, PyramidUID: ABC123) ← KEPT
+├── pyramid_level_1.dcm    (5000×5000 px,   PyramidUID: ABC123) ← EXCLUDED
+├── pyramid_level_2.dcm    (2500×2500 px,   PyramidUID: ABC123) ← EXCLUDED
+├── thumbnail.dcm          (256×256 px,     PyramidUID: ABC123, ImageType: THUMBNAIL) ← EXCLUDED
+├── segmentation.dcm       (10000×10000 px, SOPClassUID: Segmentation) ← EXCLUDED
+└── standalone.dcm         (8000×8000 px,   No PyramidUID) ← KEPT
+
+Result: Only pyramid_level_0.dcm and standalone.dcm are processed
+```
+
+**Error Handling:**
+
+- Files with missing SOPClassUID are logged as warnings and excluded (malformed DICOM)
+- Files with PyramidUID but missing TotalPixelMatrix* attributes are treated as standalone
+- Files that cannot be read by pydicom are logged at DEBUG level and skipped
+- AttributeError and general exceptions are caught to prevent processing pipeline failure
+
+
 ### Progress State Management
 
 **Actual DownloadProgress Model (`_models.py`):**

src/aignostics/application/_service.py

@@ -3,20 +3,20 @@
 import base64
 import re
 import time
+from collections import defaultdict
 from collections.abc import Callable, Generator
 from http import HTTPStatus
 from importlib.util import find_spec
 from pathlib import Path
 from typing import Any
 
 import google_crc32c
+import pydicom
 import requests
 from loguru import logger
 
 from aignostics.bucket import Service as BucketService
-from aignostics.constants import (
-    TEST_APP_APPLICATION_ID,
-)
+from aignostics.constants import TEST_APP_APPLICATION_ID
 from aignostics.platform import (
     LIST_APPLICATION_RUNS_MAX_PAGE_SIZE,
     ApiException,
@@ -32,9 +32,7 @@
     RunOutput,
     RunState,
 )
-from aignostics.platform import (
-    Service as PlatformService,
-)
+from aignostics.platform import Service as PlatformService
 from aignostics.utils import BaseService, Health, sanitize_path_component
 from aignostics.wsi import Service as WSIService
 
@@ -312,6 +310,91 @@
             for key_value in key_value_pairs:
                 Service._process_key_value_pair(entry, key_value, external_id)
 
+    @staticmethod
+    def _filter_dicom_pyramid_files(source_directory: Path) -> set[Path]:  # noqa: C901
+        """Filter DICOM files to keep only one representative per pyramid.
+
+        For multi-file DICOM pyramids (WSI images split across multiple DICOM instances),
+        keeps only the highest resolution file. Excludes segmentations, annotations,
+        thumbnails, labels, and other non-image DICOM files. OpenSlide will automatically
+        find related pyramid files in the same directory when needed.
+
+        Filtering Strategy:
+        - Only processes VL Whole Slide Microscopy Image Storage
+          (SOPClassUID 1.2.840.10008.5.1.4.1.1.77.1.6, see here:
+          https://dicom.nema.org/medical/dicom/current/output/chtml/part04/sect_b.5.html)
+        - Excludes thumbnails, labels, overviews by ImageType DICOM attribute
+        - Groups files by PyramidUID (unique identifier for multi-resolution pyramids)
+        - Selects highest resolution based on TotalPixelMatrixRows x TotalPixelMatrixColumns
+        - Preserves standalone WSI files without PyramidUID
+
+        Args:
+            source_directory: The directory to scan.
+
+        Returns:
+            set[Path]: Set of DICOM files to exclude from processing.
+        """
+        dicom_files = list(source_directory.glob("**/*.dcm"))
+        pyramid_groups: dict[str, list[tuple[Path, int, int]]] = defaultdict(list)
+        files_to_exclude = set()
+
+        # Group by PyramidUID with dimensions
+        for dcm_file in dicom_files:
+            try:
+                ds = pydicom.dcmread(dcm_file, stop_before_pixels=True)
+
+                # Exclude non-WSI image files by SOPClassUID
+                # Only process VL Whole Slide Microscopy Image Storage (1.2.840.10008.5.1.4.1.1.77.1.6)
+                if ds.SOPClassUID != "1.2.840.10008.5.1.4.1.1.77.1.6":
+                    logger.debug(f"Excluding {dcm_file.name} - not a WSI image (SOPClassUID: {ds.SOPClassUID})")
+                    files_to_exclude.add(dcm_file)
+                    continue
+
+                # Exclude thumbnails, labels, and overview images by ImageType
+                if hasattr(ds, "ImageType"):
+                    image_type = [t.upper() for t in ds.ImageType]
+                    exclude_types = {"THUMBNAIL", "LABEL", "OVERVIEW", "MACRO", "ANNOTATION", "LOCALIZER"}
+                    if any(excluded in image_type for excluded in exclude_types):
+                        logger.debug(f"Excluding {dcm_file.name} - ImageType: {image_type}")
+                        files_to_exclude.add(dcm_file)
+                        continue
+
+                # Now process valid WSI images with PyramidUID
+                if not hasattr(ds, "PyramidUID"):
+                    logger.debug(f"DICOM {dcm_file.name} has no PyramidUID - treating as standalone")
+                    continue
+
+                pyramid_uid = ds.PyramidUID
+
+                # These represent the full image dimensions across all frames
+                rows = int(ds.TotalPixelMatrixRows)
+                cols = int(ds.TotalPixelMatrixColumns)
+
+                pyramid_groups[pyramid_uid].append((dcm_file, rows, cols))
+            except AttributeError as e:
+                logger.debug(f"DICOM {dcm_file} missing required attributes: {e}")
+            except Exception as e:
+                logger.debug(f"Could not read DICOM {dcm_file}: {e}")
+
+        # For each pyramid with multiple files, keep only the highest resolution one
+        for pyramid_uid, files_with_dims in pyramid_groups.items():
+            if len(files_with_dims) > 1:
+                # Find the file with the largest dimensions (rows * cols = total pixels)
+                highest_res_file = max(files_with_dims, key=lambda x: x[1] * x[2])
+                file_to_keep, rows, cols = highest_res_file
+
+                # Exclude all others
+                for file_path, _, _ in files_with_dims:
+                    if file_path != file_to_keep:
+                        files_to_exclude.add(file_path)
+
+                logger.debug(
+                    f"DICOM pyramid {pyramid_uid}: keeping {file_to_keep.name} "
+                    f"({rows}x{cols}), excluding {len(files_with_dims) - 1} related files"
+                )
+
+        return files_to_exclude
+
     @staticmethod
     def generate_metadata_from_source_directory(  # noqa: PLR0913, PLR0917
         source_directory: Path,
@@ -366,10 +449,17 @@
 
         metadata = []
 
+        # Pre-filter: exclude redundant DICOM files from multi-file pyramids
+        dicom_files_to_exclude = Service._filter_dicom_pyramid_files(source_directory)
+
         try:
             extensions = get_supported_extensions_for_application(application_id)
             for extension in extensions:
                 for file_path in source_directory.glob(f"**/*{extension}"):
+                    # Skip excluded DICOM files
+                    if file_path in dicom_files_to_exclude:
+                        continue
+
                     # Generate CRC32C checksum with google_crc32c and encode as base64
                     hash_sum = google_crc32c.Checksum()  # type: ignore[no-untyped-call]
                     with file_path.open("rb") as f: