fix(spike_sorting,data_loaders): IOStallWatchdog blind-read robustness + _build_spikedata length epsilon

.github/workflows/tests.yml

@@ -25,6 +25,6 @@ jobs:
           pip install scikit-learn networkx pandas matplotlib tqdm
 
       - name: Run tests
-        run: pytest -q
+        run: pytest -v --tb=short -p no:cacheprovider
 
 

docs/source/guides/batch_jobs.rst

@@ -219,8 +219,18 @@ Build reusable base images for CPU and GPU workloads:
 
 .. code-block:: bash
 
-   docker build -f docker/analysis-base/Dockerfile.cpu -t spikelab/analysis-base:cpu .
-   docker build -f docker/analysis-base/Dockerfile.gpu -t spikelab/analysis-base:gpu .
+   bash scripts/build_base_image.sh cpu spikelab/analysis-base:cpu
+   bash scripts/build_base_image.sh gpu spikelab/analysis-base:gpu
+
+The base image bakes in the SpikeLab source via ``COPY src ./src`` and
+``pip install -e .``. It is a frozen snapshot — published SpikeLab releases do
+not update an existing image automatically. Rebuild whenever the library
+source has changed and you need that change reflected on the cluster.
+
+When iterating on a feature branch, build under a developer-scoped tag (e.g.,
+``ghcr.io/<org>/spikelab-analysis-base:${USER}-$(git rev-parse --short HEAD)``)
+and pass it explicitly via ``--image`` so concurrent developers do not clobber
+each other's shared ``:cpu`` / ``:gpu`` tags.
 
 Temporary images
 ^^^^^^^^^^^^^^^^
@@ -232,6 +242,10 @@ Build and push a temporary image for a single run:
    bash scripts/build_temp_image.sh gpu ghcr.io/<org>/spikelab-analysis-temp:<tag>
    bash scripts/push_temp_image.sh ghcr.io/<org>/spikelab-analysis-temp:<tag>
 
+This layers analysis-time files on top of an existing ``analysis-base`` image
+without rebuilding it. Use this when only the analysis script changed; if
+``src/spikelab/`` itself changed, rebuild the base image first (see above).
+
 Reference this tag in the ``ContainerSpec`` when creating your ``JobSpec``.
 
 

scripts/build_base_image.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ $# -lt 2 ]]; then
+  echo "Usage: $0 <cpu|gpu> <image-tag>"
+  echo "Example: $0 cpu ghcr.io/acme/spikelab-analysis-base:dev-abc1234"
+  exit 1
+fi
+
+profile="$1"
+image_tag="$2"
+
+case "$profile" in
+  cpu) dockerfile="docker/analysis-base/Dockerfile.cpu" ;;
+  gpu) dockerfile="docker/analysis-base/Dockerfile.gpu" ;;
+  *)
+    echo "Error: profile must be 'cpu' or 'gpu', got '$profile'"
+    exit 1
+    ;;
+esac
+
+docker build \
+  -f "${dockerfile}" \
+  -t "${image_tag}" \
+  .
+
+echo "BUILT_IMAGE=${image_tag}"

src/spikelab/batch_jobs/INSTRUCTIONS.md

@@ -85,12 +85,38 @@ These scripts are in the SpikeLab repository under `scripts/` and `docker/`. The
    - `python scripts/generate_job_config.py --image <image-tag> --profile <cpu|gpu> --output configs/batch-temp-job.yaml`
 5. Confirm image is pullable from target cluster/namespace before deploy.
 
+### When SpikeLab source has changed (developer iteration)
+
+The `build_temp_image.sh` workflow above layers analysis code on top of an existing `analysis-base` image. It does **not** capture changes to `src/spikelab/` itself. If the user has modified the SpikeLab library (e.g., they are on a feature branch with new methods that the submitted script depends on), the `analysis-base` image must be rebuilt first — otherwise the running container exposes a stale API and the job will fail with `AttributeError` or run against outdated behavior.
+
+In that case, rebuild and push a **developer-scoped base image** before submitting, and pass it explicitly via `--image`:
+
+```bash
+# From SpikeLab repo root. Use ${USER:-${USERNAME}} for Linux/Mac/Windows compatibility.
+USER_TAG="ghcr.io/<org>/spikelab-analysis-base:${USER:-${USERNAME}}-$(git rev-parse --short HEAD)"
+
+bash scripts/build_base_image.sh cpu "${USER_TAG}"   # or 'gpu'
+bash scripts/push_temp_image.sh "${USER_TAG}"
+
+# Submit using the freshly built image
+spikelab-batch-jobs deploy-job \
+  --profile <profile> \
+  --job-config <path> \
+  --image "${USER_TAG}"
+```
+
+Notes:
+- The Dockerfile uses `COPY src ./src`, so **uncommitted edits in `src/spikelab/` are also baked into the image**. This is useful for fast iteration but can be surprising — confirm `git status` reflects the state you intend to ship.
+- Use a developer-scoped tag (username + short SHA) rather than the shared `:cpu`/`:gpu` tags so concurrent developers do not clobber each other's images.
+- The shared `ghcr.io/braingeneers/spikelab-analysis-base:cpu` / `:gpu` tags are static snapshots — they do **not** track new SpikeLab releases automatically. Always rebuild when the library source has changed locally.
+
 ## Fixed Workflow
 
 1. **Preflight checks**
    - Run `kubectl version --client`.
    - Run `kubectl config current-context`.
    - Validate registry/image tag exists and is pushed.
+   - If `git status` shows changes to `src/spikelab/`, the cluster-side image is stale relative to local code. Rebuild and push a developer-scoped base image before submitting (see "When SpikeLab source has changed" under Container Prep) and pass the resulting tag via `--image`.
    - Optionally verify S3 access if asked by the user.
 2. **Validate inputs**
    - Ensure `--job-config` is present.

src/spikelab/batch_jobs/backend_k8s.py

@@ -82,17 +82,33 @@ def apply_manifest(self, manifest_path_or_str: str) -> str:
         return payload["metadata"]["name"]
 
     def delete_job(self, name: str) -> None:
-        """Delete a job and its pods."""
+        """Delete a job and its pods. Idempotent: missing jobs are a no-op.
+
+        Matches the ``kubectl --ignore-not-found=true`` semantic on
+        the fallback path so the two delete paths behave the same
+        way for the missing-job case. Previously the Python
+        kubernetes-client path propagated ``ApiException(404)``
+        verbatim while the kubectl path exited cleanly.
+        """
         if self._batch_api is None:
             self._run_kubectl(
                 ["delete", "job", name, "-n", self.namespace, "--ignore-not-found=true"]
             )
             return
-        self._batch_api.delete_namespaced_job(
-            name=name,
-            namespace=self.namespace,
-            body=client.V1DeleteOptions(propagation_policy="Background"),
-        )
+        try:
+            self._batch_api.delete_namespaced_job(
+                name=name,
+                namespace=self.namespace,
+                body=client.V1DeleteOptions(propagation_policy="Background"),
+            )
+        except client.exceptions.ApiException as exc:
+            if exc.status == 404:
+                # Missing job — idempotent no-op, matches kubectl
+                # ``--ignore-not-found`` behaviour. Any other API
+                # error (403 Forbidden, 500 Server Error, etc.)
+                # still propagates.
+                return
+            raise
 
     def job_status(self, name: str) -> str:
         """Return one of Pending/Running/Complete/Failed/Unknown."""

src/spikelab/data_loaders/data_exporters.py

@@ -270,13 +270,6 @@ def export_spikedata_to_nwb(
           when prefer_pynwb=False.
     """
     ensure_h5py()
-    if sd.start_time != 0:
-        warnings.warn(
-            f"Exporting event-centered SpikeData (start_time={sd.start_time}) "
-            "to NWB. The NWB format does not store start_time, so spike times "
-            "are written as-is. On reload, start_time will default to 0.",
-            UserWarning,
-        )
     counts = [len(t) for t in sd.train]
     flat_ms = np.concatenate(sd.train) if sum(counts) else np.array([], float)
     flat_s = times_from_ms(flat_ms, "s", fs_Hz=None)