some more overview filtering

Author: peter-urban

python/themachinethatgoesping/pingprocessing/overview/__init__.py

@@ -12,6 +12,7 @@
 from .overlap_filter import (
     filter_by_spatial_overlap,
     filter_by_temporal_overlap,
+    filter_by_speed,
     subset_overview,
 )
 

python/themachinethatgoesping/pingprocessing/overview/overlap_filter.py

@@ -85,15 +85,38 @@ def subset_overview(overview: PingOverview, indices) -> PingOverview:
     for key, values in overview.variables.items():
         subset.variables[key] = [values[i] for i in indices]
 
-    # Rebuild file-path lookup with only the paths actually referenced
-    # by the kept pings, and remap file_path_index to new indices.
-    old_indices = subset.variables.get("file_path_index", [])
-    used_old = sorted(set(old_indices))
-    old_to_new = {old: new for new, old in enumerate(used_old)}
+    # Rebuild primary file-path lookup with only referenced paths.
+    old_primary = subset.variables.get("primary_file_path_index", [])
+    used_primary = sorted(set(old_primary))
+    primary_old_to_new = {old: new for new, old in enumerate(used_primary)}
 
-    subset._file_paths = [overview._file_paths[i] for i in used_old]
-    subset._file_path_map = {fp: i for i, fp in enumerate(subset._file_paths)}
-    subset.variables["file_path_index"] = [old_to_new[i] for i in old_indices]
+    subset._primary_file_paths = [
+        overview._primary_file_paths[i] for i in used_primary
+    ]
+    subset._primary_file_path_map = {
+        fp: i for i, fp in enumerate(subset._primary_file_paths)
+    }
+    subset.variables["primary_file_path_index"] = [
+        primary_old_to_new[i] for i in old_primary
+    ]
+
+    # Rebuild all-file-paths lookup with only referenced paths.
+    used_all = set()
+    for idx_list in subset.variables.get("file_path_indices", []):
+        used_all.update(idx_list)
+    used_all = sorted(used_all)
+    all_old_to_new = {old: new for new, old in enumerate(used_all)}
+
+    subset._all_file_paths = [
+        overview._all_file_paths[i] for i in used_all
+    ]
+    subset._all_file_path_map = {
+        fp: i for i, fp in enumerate(subset._all_file_paths)
+    }
+    subset.variables["file_path_indices"] = [
+        [all_old_to_new[i] for i in idx_list]
+        for idx_list in subset.variables.get("file_path_indices", [])
+    ]
 
     subset.original_indices = indices
     return subset
@@ -305,3 +328,37 @@ def filter_by_temporal_overlap(
         subset_overview(ov, np.where(mask)[0])
         for ov, mask in zip(overviews, masks)
     ]
+
+
+def filter_by_speed(
+    overview: PingOverview,
+    min_knots: float = None,
+    max_knots: float = None,
+) -> PingOverview:
+    """
+    Filter a PingOverview by inter-ping speed.
+
+    Computes the speed between consecutive pings and keeps only pings
+    where the speed is within [*min_knots*, *max_knots*].
+
+    Parameters
+    ----------
+    overview : PingOverview
+        The overview to filter.
+    min_knots : float, optional
+        Minimum speed in knots (inclusive). Pings below this are removed.
+    max_knots : float, optional
+        Maximum speed in knots (inclusive). Pings above this are removed.
+
+    Returns
+    -------
+    PingOverview
+        Filtered overview with ``original_indices`` attribute.
+    """
+    speed = overview.get_speed_in_knots()
+    mask = np.ones(len(speed), dtype=bool)
+    if min_knots is not None:
+        mask &= speed >= min_knots
+    if max_knots is not None:
+        mask &= speed <= max_knots
+    return subset_overview(overview, np.where(mask)[0])

python/themachinethatgoesping/pingprocessing/overview/pingoverview.py

@@ -59,8 +59,10 @@ def __init__(self, ping_list: List = None, progress: bool = False) -> None:
         """
         self.variables = defaultdict(list)
         self.stats = defaultdict(dict)
-        self._file_paths = []       # unique file paths (small list)
-        self._file_path_map = {}    # path → index (for fast lookup during add)
+        self._primary_file_paths = []       # unique primary file paths
+        self._primary_file_path_map = {}    # primary path → index
+        self._all_file_paths = []            # unique file paths (primary + secondary)
+        self._all_file_path_map = {}         # path → index
 
         if ping_list is not None:
             self.add_ping_list(ping_list, progress)
@@ -170,11 +172,24 @@ def add_ping(self, ping) -> None:
         self.variables["latitude"].append(geolocation.latitude)
         self.variables["longitude"].append(geolocation.longitude)
 
-        file_path = ping.file_data.get_primary_file_path()
-        if file_path not in self._file_path_map:
-            self._file_path_map[file_path] = len(self._file_paths)
-            self._file_paths.append(file_path)
-        self.variables["file_path_index"].append(self._file_path_map[file_path])
+        # Primary file path (one per ping)
+        primary_path = ping.file_data.get_primary_file_path()
+        if primary_path not in self._primary_file_path_map:
+            self._primary_file_path_map[primary_path] = len(self._primary_file_paths)
+            self._primary_file_paths.append(primary_path)
+        self.variables["primary_file_path_index"].append(
+            self._primary_file_path_map[primary_path]
+        )
+
+        # All file paths (primary + secondary, e.g. water column + bottom)
+        all_paths = ping.file_data.get_file_paths()
+        path_indices = []
+        for fp in all_paths:
+            if fp not in self._all_file_path_map:
+                self._all_file_path_map[fp] = len(self._all_file_paths)
+                self._all_file_paths.append(fp)
+            path_indices.append(self._all_file_path_map[fp])
+        self.variables["file_path_indices"].append(path_indices)
 
         stats = defaultdict(dict)
 
@@ -281,20 +296,31 @@ def get_median(self, key: str) -> float:
 
         return self.stats[key]["median"]
 
+    def get_primary_file_paths(self) -> List[str]:
+        """
+        Return the list of unique primary file paths.
+
+        Returns
+        -------
+        List[str]
+            Unique primary file paths referenced by pings in this overview.
+        """
+        return self._primary_file_paths
+
     def get_file_paths(self) -> List[str]:
         """
-        Return the list of unique file paths.
+        Return all unique file paths (primary and secondary).
 
         Returns
         -------
         List[str]
-            Unique file paths referenced by pings in this overview.
+            All unique file paths referenced by pings in this overview.
         """
-        return self._file_paths
+        return self._all_file_paths
 
-    def get_file_path(self, ping_index: int) -> str:
+    def get_primary_file_path(self, ping_index: int) -> str:
         """
-        Return the file path for a specific ping by its index.
+        Return the primary file path for a specific ping by its index.
 
         Parameters
         ----------
@@ -304,22 +330,58 @@ def get_file_path(self, ping_index: int) -> str:
         Returns
         -------
         str
-            The file path of the ping.
+            The primary file path of the ping.
+        """
+        return self._primary_file_paths[
+            self.variables["primary_file_path_index"][ping_index]
+        ]
+
+    def get_file_paths_for_ping(self, ping_index: int) -> List[str]:
+        """
+        Return all file paths for a specific ping.
+
+        Parameters
+        ----------
+        ping_index : int
+            Index of the ping in this overview.
+
+        Returns
+        -------
+        List[str]
+            All file paths associated with the ping.
+        """
+        return [
+            self._all_file_paths[i]
+            for i in self.variables["file_path_indices"][ping_index]
+        ]
+
+    def get_pings_per_primary_file_path(self) -> dict:
         """
-        return self._file_paths[self.variables["file_path_index"][ping_index]]
+        Return a mapping from primary file path to list of ping indices.
+
+        Returns
+        -------
+        dict
+            Dictionary mapping primary_file_path → list of ping indices.
+        """
+        result = defaultdict(list)
+        for i, idx in enumerate(self.variables["primary_file_path_index"]):
+            result[self._primary_file_paths[idx]].append(i)
+        return dict(result)
 
     def get_pings_per_file_path(self) -> dict:
         """
-        Return a mapping from file path to list of ping indices.
+        Return a mapping from file path (any) to list of ping indices.
 
         Returns
         -------
         dict
             Dictionary mapping file_path → list of ping indices.
         """
         result = defaultdict(list)
-        for i, idx in enumerate(self.variables["file_path_index"]):
-            result[self._file_paths[idx]].append(i)
+        for i, idx_list in enumerate(self.variables["file_path_indices"]):
+            for idx in idx_list:
+                result[self._all_file_paths[idx]].append(i)
         return dict(result)
 
     def _get_minmax_per_file(self, key: str) -> dict:
@@ -338,7 +400,7 @@ def _get_minmax_per_file(self, key: str) -> dict:
         """
         vals = self.variables[key]
         result = {}
-        for fp, indices in self.get_pings_per_file_path().items():
+        for fp, indices in self.get_pings_per_primary_file_path().items():
             file_vals = [vals[i] for i in indices]
             result[fp] = (min(file_vals), max(file_vals))
         return result