diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 2d0e9261..f461f088 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,10 +1,13 @@ -FROM python:3.9-slim@sha256:5f0192a4f58a6ce99f732fe05e3b3d00f12ae62e183886bca3ebe3d202686c7f +ARG PY_VER=3.11 +ARG DISTRO=bullseye +FROM mcr.microsoft.com/devcontainers/python:${PY_VER}-${DISTRO} -ENV PATH /usr/local/bin:$PATH -ENV PYTHON_VERSION 3.9.17 +# Avoid warnings by switching to noninteractive +ENV DEBIAN_FRONTEND=noninteractive + +USER root RUN \ - adduser --system --disabled-password --shell /bin/bash vscode && \ # install docker apt-get update && \ apt-get install ca-certificates curl gnupg lsb-release -y && \ @@ -45,4 +48,5 @@ ENV EPHYS_ROOT_DATA_DIR /workspaces/element-array-ephys/example_data ENV DATABASE_PREFIX neuro_ USER vscode -CMD bash -c "sudo rm /var/run/docker.pid; sudo dockerd" \ No newline at end of file + +CMD bash -c "sudo rm /var/run/docker.pid; sudo dockerd" diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index bf939e88..1c062f18 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -22,8 +22,8 @@ }, "vscode": { "extensions": [ - "ms-python.python@2023.8.0", - "ms-toolsai.jupyter@2023.3.1201040234" + "ms-python.python@2025.6.1", + "ms-toolsai.jupyter@2025.4.1" ] } } diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index d31fbace..74368549 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,5 +1,5 @@ blank_issues_enabled: false contact_links: - name: DataJoint Contribution Guideline - url: https://docs.datajoint.org/python/community/02-Contribute.html + url: https://docs.datajoint.com/about/contribute/ about: Please make sure to review the DataJoint Contribution Guidelines \ No newline at end of file diff --git a/.github/workflows/devcontainer-build-publish-caller.yml b/.github/workflows/devcontainer-build-publish-caller.yml new file mode 100644 index 00000000..2745efea --- /dev/null +++ b/.github/workflows/devcontainer-build-publish-caller.yml @@ -0,0 +1,10 @@ +name: devcontainer_build_publish +on: + workflow_dispatch: + +jobs: + call_devcontainer_build_publish: + uses: datajoint/.github/.github/workflows/devcontainer-build-publish.yaml@main + secrets: + DOCKERHUB_USERNAME: ${{secrets.DOCKERHUB_USERNAME}} + DOCKERHUB_TOKEN: ${{secrets.DOCKERHUB_TOKEN_FOR_ELEMENTS}} diff --git a/.github/workflows/mkdocs-release-caller.yml b/.github/workflows/mkdocs-release-caller.yml new file mode 100644 index 00000000..e25b0184 --- /dev/null +++ b/.github/workflows/mkdocs-release-caller.yml @@ -0,0 +1,9 @@ +name: mkdocs-release +on: + workflow_dispatch: + +jobs: + mkdocs_release: + uses: datajoint/.github/.github/workflows/mkdocs_release.yaml@main + permissions: + contents: write diff --git a/.github/workflows/semantic-release-caller.yml b/.github/workflows/semantic-release-caller.yml new file mode 100644 index 00000000..bfd7643d --- /dev/null +++ b/.github/workflows/semantic-release-caller.yml @@ -0,0 +1,10 @@ +name: semantic-release +on: + workflow_dispatch: + +jobs: + call_semantic_release: + uses: datajoint/.github/.github/workflows/semantic-release.yaml@main + secrets: + APP_ID: ${{ secrets.ELEMENT_APP_ID }} + GET_TOKEN_KEY: ${{ secrets.ELEMENT_GET_TOKEN_KEY }} diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index e5e6a07a..36199ef3 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,22 +1,19 @@ name: Test on: push: + branches: + - main pull_request: + branches: + - main workflow_dispatch: jobs: - devcontainer-build: - uses: datajoint/.github/.github/workflows/devcontainer-build.yaml@main tests: runs-on: ubuntu-latest strategy: matrix: py_ver: ["3.9", "3.10"] - mysql_ver: ["8.0", "5.7"] - include: - - py_ver: "3.8" - mysql_ver: "5.7" - - py_ver: "3.7" - mysql_ver: "5.7" + mysql_ver: ["8.0"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{matrix.py_ver}} @@ -31,4 +28,4 @@ jobs: run: | python_version=${{matrix.py_ver}} black element_array_ephys --check --verbose --target-version py${python_version//.} - + black notebooks --check --verbose --target-version py${python_version//.} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0d513df7..6d28ef11 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ exclude: (^.github/|^docs/|^images/) repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer @@ -16,7 +16,7 @@ repos: # black - repo: https://github.com/psf/black - rev: 22.12.0 + rev: 24.2.0 hooks: - id: black - id: black-jupyter @@ -25,7 +25,7 @@ repos: # isort - repo: https://github.com/pycqa/isort - rev: 5.11.2 + rev: 5.13.2 hooks: - id: isort args: ["--profile", "black"] @@ -33,7 +33,7 @@ repos: # flake8 - repo: https://github.com/pycqa/flake8 - rev: 4.0.1 + rev: 7.0.0 hooks: - id: flake8 args: # arguments to configure flake8 diff --git a/CHANGELOG.md b/CHANGELOG.md index d2e48eaa..fff5cd7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,161 +3,207 @@ Observes [Semantic Versioning](https://semver.org/spec/v2.0.0.html) standard and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) convention. -## [0.3.5] - 2024-08-16 +## [0.4.5] - 2025-09-17 -+ Fix - Improve `spikeglx` loader in extracting neuropixels probe type from the meta file -+ Update - Explicit call to `probe.create_neuropixels_probe_types()` to create entries in ProbeType ++ Fix - Rename `env.yml` to `conda_env.yml` for consistency with other projects ++ Update - Passing tests by blackifying previous updates +## [0.4.4] - 2025-06-06 -## [0.3.4] - 2024-03-22 ++ Update - Update params in test/tutorial ++ Fix - Moved KiloSort's template_features to additional_files + +## [0.4.3] - 2025-06-03 + ++ Update - Use `Spikeinterface` official released versions (instead of install directly from source) + +## [0.4.2] - 2025-03-25 + ++ Fix - Add key_source to `ProbeLevelReport` to filter for 'good' quality units + +## [0.4.1] - 2025-02-13 + ++ Fix - Auto generate ProbeInsertion allows duplicate probes (e.g. probe reuse) + +## [0.4.0] - 2025-01-28 + ++ Update - No longer support multiple variation of ephys module, keep only `ephys_no_curation` module, renamed to `ephys` ++ Update - Remove other ephys modules (e.g. `ephys_acute`, `ephys_chronic`) (moved to different branches) ++ Feat - Add support for `SpikeInterface` ++ Update - Remove support for `ecephys_spike_sorting` (moved to a different branch) ++ Update - Simplify the "activate" mechanism + +## [0.3.8] - 2025-01-16 + +* Fix - Revert GHA Semantic Release caller and update changelog -+ Add - pytest -+ Update - Ephys schema changed from `ephys_acute` to `ephys_no_curation` in `tutorial.ipynb` +## [0.3.7] - 2024-11-01 +* Fix - robust IMAX value detection from IMEC file (metadata 3.0) + +## [0.3.6] - 2024-10-01 + +* Fix - Minor bugfix in reading npx probe model of older versions and in reaeding probe model +* Add - New GitHub Action callers for devcontainer, mkdocs, and semantic release + +## [0.3.5] - 2024-08-16 + +* Fix - Improve `spikeglx` loader in extracting neuropixels probe type from the meta file +* Update - Explicit call to `probe.create_neuropixels_probe_types()` to create entries in `ProbeType` + +## [0.3.4] - 2024-03-22 + +* Add - pytest +* Fix - Fix regex patterns and add minimum version for scikit-image ## [0.3.3] - 2024-01-24 -+ Update - remove PyPi release from `release.yml` since it will fail after the new `setup.py` + +* Update - remove PyPi release from `release.yml` since it will fail after the new `setup.py` ## [0.3.2] - 2024-01-12 -+ Fix - `probe_geometry` bugfix for incorrect handling of probes with staggered electrode positions + +* Fix - `probe_geometry` bugfix for incorrect handling of probes with staggered electrode positions ## [0.3.1] - 2023-11-28 -+ Update - Flowchart borders for consistency with other DataJoint Elements -+ Fix - `dj.config()` setup moved to `tutorial_pipeline.py` instead of `__init__.py` -+ Update - Elements installed directly from GitHub instead of PyPI -+ Update - Structure of the tutorial notebook + +* Update - Flowchart borders for consistency with other DataJoint Elements +* Fix - `dj.config()` setup moved to `tutorial_pipeline.py` instead of `__init__.py` +* Update - Elements installed directly from GitHub instead of PyPI +* Update - Structure of the tutorial notebook ## [0.3.0] - 2023-10-25 -+ Add - DevContainer for codespaces -+ Add - `tutorial_pipeline.py` -+ Add - 60 min tutorial using Jupyter Notebooks, short demo notebook -+ Update - General improvements to increase consistency with other DataJoint Elements +* Add - DevContainer for codespaces +* Add - `tutorial_pipeline.py` +* Add - 60 min tutorial using Jupyter Notebooks, short demo notebook +* Update - General improvements to increase consistency with other DataJoint Elements ## [0.2.11] - 2023-06-29 -+ Update - Improve kilosort triggering routine - better logging, remove temporary files, robust resumable processing -+ Add - Null value for `package_version` to patch bug -+ Update - GitHub Actions workflows -+ Update - README instructions +* Update - Improve kilosort triggering routine - better logging, remove temporary files, robust resumable processing +* Add - Null value for `package_version` to patch bug +* Update - GitHub Actions workflows +* Update - README instructions ## [0.2.10] - 2023-05-26 -+ Add - Kilosort, NWB, and DANDI citations -+ Fix - CSS to improve readability of tables in dark mode -+ Update - mkdocs.yaml +* Add - Kilosort, NWB, and DANDI citations +* Fix - CSS to improve readability of tables in dark mode +* Update - mkdocs.yaml ## [0.2.9] - 2023-05-11 -+ Fix - `.ipynb` dark mode output for all notebooks. +* Fix - `.ipynb` dark mode output for all notebooks. ## [0.2.8] - 2023-04-28 -+ Fix - `.ipynb` output in tutorials is not visible in dark mode. +* Fix - `.ipynb` output in tutorials is not visible in dark mode. ## [0.2.7] - 2023-04-19 -+ Bugfix - A name remapping dictionary was added to ensure consistency between the column names of the `metrics.csv` file and the attribute names of the `QualityMetrics` table +* Bugfix - A name remapping dictionary was added to ensure consistency between the column names of the `metrics.csv` file and the attribute names of the `QualityMetrics` table ## [0.2.6] - 2023-04-17 -+ Fix - Update Pandas DataFrame column name to insert `pt_ratio` in `QualityMetrics.Waveform` table +* Fix - Update Pandas DataFrame column name to insert `pt_ratio` in `QualityMetrics.Waveform` table ## [0.2.5] - 2023-04-12 -+ Add - docstrings for quality metric tables -+ Fix - docstring errors -+ Update - `concepts.md` -+ Update - schema diagrams with quality metrics tables +* Add - docstrings for quality metric tables +* Fix - docstring errors +* Update - `concepts.md` +* Update - schema diagrams with quality metrics tables ## [0.2.4] - 2023-03-10 -+ Update - Requirements with `ipywidgets` and `scikit-image` for plotting widget +* Update - Requirements with `ipywidgets` and `scikit-image` for plotting widget ## [0.2.3] - 2023-02-14 -+ Add - extras_require install options for nwb and development requirement sets -+ Add - mkdocs notebook rendering -+ Add - markdown linting and spellcheck config files, with implementation edits -+ Update - license for 2023 -+ Update - blackify previous updates +* Add - extras_require install options for nwb and development requirement sets +* Add - mkdocs notebook rendering +* Add - markdown linting and spellcheck config files, with implementation edits +* Update - license for 2023 +* Update - blackify previous updates ## [0.2.2] - 2022-01-11 -+ Bugfix - Revert import order in `__init__.py` to avoid circular import error. -+ Update - `.pre-commit-config.yaml` to disable automatic positioning of import +* Bugfix - Revert import order in `__init__.py` to avoid circular import error. +* Update - `.pre-commit-config.yaml` to disable automatic positioning of import statement at the top. -+ Bugfix - Update docstrings to render API for documentation website. +* Bugfix - Update docstrings to render API for documentation website. ## [0.2.1] - 2022-01-06 -+ Add - `build_electrode_layouts` function in `probe.py` to compute the electrode layout +* Add - `build_electrode_layouts` function in `probe.py` to compute the electrode layout for all types of probes. -+ Update - parameterize run_CatGT step from parameters retrieved from +* Update - parameterize run_CatGT step from parameters retrieved from `ClusteringParamSet` table -+ Update - clustering step, update duration for "median_subtraction" step -+ Bugfix - handles single probe recording in "Neuropix-PXI" format -+ Update - safeguard in creating/inserting probe types upon probe activation -+ Add - quality control metric dashboard -+ Update & fix docstrings -+ Update - `ephys_report.UnitLevelReport` to add `ephys.ClusterQualityLabel` as a +* Update - clustering step, update duration for "median_subtraction" step +* Bugfix - handles single probe recording in "Neuropix-PXI" format +* Update - safeguard in creating/inserting probe types upon probe activation +* Add - quality control metric dashboard +* Update & fix docstrings +* Update - `ephys_report.UnitLevelReport` to add `ephys.ClusterQualityLabel` as a foreign key reference -+ Add - `.pre-commit-config.yaml` +* Add - `.pre-commit-config.yaml` ## [0.2.0] - 2022-10-28 -+ Add - New schema `ephys_report` to compute and store figures from results -+ Add - Widget to display figures -+ Add - Add `ephys_no_curation` and routines to trigger spike-sorting analysis +* Add - New schema `ephys_report` to compute and store figures from results +* Add - Widget to display figures +* Add - Add `ephys_no_curation` and routines to trigger spike-sorting analysis using Kilosort (2.0, 2.5) -+ Add - mkdocs for Element Documentation -+ Add - New `QualityMetrics` table to store clusters' and waveforms' metrics after the +* Add - mkdocs for Element Documentation +* Add - New `QualityMetrics` table to store clusters' and waveforms' metrics after the spike sorting analysis. ## [0.1.4] - 2022-07-11 -+ Bugfix - Handle case where `spike_depths` data is present. +* Bugfix - Handle case where `spike_depths` data is present. ## [0.1.3] - 2022-06-16 -+ Update - Allow for the `precluster_output_dir` attribute to be nullable when no +* Update - Allow for the `precluster_output_dir` attribute to be nullable when no pre-clustering is performed. ## [0.1.2] - 2022-06-09 -+ Bugfix - Handle case where `pc_features.npy` does not exist. +* Bugfix - Handle case where `pc_features.npy` does not exist. ## [0.1.1] - 2022-06-01 -+ Add - Secondary attributes to `PreClusterParamSteps` table +* Add - Secondary attributes to `PreClusterParamSteps` table ## [0.1.0] - 2022-05-26 -+ Update - Rename module for acute probe insertions from `ephys.py` to `ephys_acute.py`. -+ Add - Module for pre-clustering steps (`ephys_precluster.py`), which is built off of +* Update - Rename module for acute probe insertions from `ephys.py` to `ephys_acute.py`. +* Add - Module for pre-clustering steps (`ephys_precluster.py`), which is built off of `ephys_acute.py`. -+ Add - Module for chronic probe insertions (`ephys_chronic.py`). -+ Bugfix - Missing `fileTimeSecs` key in SpikeGLX meta file. -+ Update - Move common functions to `element-interface` package. -+ Add - NWB export function +* Add - Module for chronic probe insertions (`ephys_chronic.py`). +* Bugfix - Missing `fileTimeSecs` key in SpikeGLX meta file. +* Update - Move common functions to `element-interface` package. +* Add - NWB export function ## [0.1.0b4] - 2021-11-29 -+ Add - Processing with Kilosort and pyKilosort for Open Ephys and SpikeGLX +* Add - Processing with Kilosort and pyKilosort for Open Ephys and SpikeGLX ## [0.1.0b0] - 2021-05-07 -+ Update - First beta release +* Update - First beta release ## [0.1.0a5] - 2021-05-05 -+ Add - GitHub Action release process -+ Add - `probe` and `ephys` elements -+ Add - Readers for: `SpikeGLX`, `Open Ephys`, `Kilosort` -+ Add - Probe table supporting: Neuropixels probes 1.0 - 3A, 1.0 - 3B, 2.0 - SS, +* Add - GitHub Action release process +* Add - `probe` and `ephys` elements +* Add - Readers for: `SpikeGLX`, `Open Ephys`, `Kilosort` +* Add - Probe table supporting: Neuropixels probes 1.0 - 3A, 1.0 - 3B, 2.0 - SS, 2.0 - MS -[0.3.0]: https://github.com/datajoint/element-array-ephys/releases/tag/0.3.0 +[0.3.6]: https://github.com/datajoint/element-array-ephys/releases/tag/0.3.6 +[0.3.5]: https://github.com/datajoint/element-array-ephys/releases/tag/0.3.5 +[0.3.1]: https://github.com/datajoint/element-array-ephys/releases/tag/0.3.1 [0.2.11]: https://github.com/datajoint/element-array-ephys/releases/tag/0.2.11 [0.2.10]: https://github.com/datajoint/element-array-ephys/releases/tag/0.2.10 [0.2.9]: https://github.com/datajoint/element-array-ephys/releases/tag/0.2.9 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2bd0f498..61a342d6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,5 +1,5 @@ # Contribution Guidelines This project follows the -[DataJoint Contribution Guidelines](https://datajoint.com/docs/about/contribute/). +[DataJoint Contribution Guidelines](https://docs.datajoint.com/about/contribute/). Please reference the link for more full details. diff --git a/README.md b/README.md index 85277136..aad51cad 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ environment and notebooks to learn the pipeline. + [Interactive tutorial on GitHub Codespaces](https://github.com/datajoint/element-array-ephys#interactive-tutorial) -+ [Documentation](https://datajoint.com/docs/elements/element-array-ephys) ++ [Documentation](https://docs.datajoint.com/elements/element-array-ephys/) ## Support diff --git a/conda_env.yml b/conda_env.yml new file mode 100644 index 00000000..e9b3ce13 --- /dev/null +++ b/conda_env.yml @@ -0,0 +1,7 @@ +channels: + - conda-forge + - defaults +dependencies: + - pip + - python>=3.7,<3.11 +name: element_array_ephys diff --git a/docs/docker-compose.yaml b/docs/docker-compose.yaml index 5ba221df..bc2c2b8b 100644 --- a/docs/docker-compose.yaml +++ b/docs/docker-compose.yaml @@ -30,12 +30,6 @@ services: export ELEMENT_UNDERSCORE=$$(echo $${PACKAGE} | sed 's/element_//g') export ELEMENT_HYPHEN=$$(echo $${ELEMENT_UNDERSCORE} | sed 's/_/-/g') export PATCH_VERSION=$$(cat /main/$${PACKAGE}/version.py | grep -oE '\d+\.\d+\.[a-z0-9]+') - git clone https://github.com/datajoint/workflow-$${ELEMENT_HYPHEN}.git /main/delete || true - if [ -d /main/delete/ ]; then - mv /main/delete/workflow_$${ELEMENT_UNDERSCORE} /main/ - mv /main/delete/notebooks/*ipynb /main/docs/src/tutorials/ - rm -fR /main/delete - fi if echo "$${MODE}" | grep -i live &>/dev/null; then mkdocs serve --config-file ./docs/mkdocs.yaml -a 0.0.0.0:80 2>&1 | tee docs/temp_mkdocs.log elif echo "$${MODE}" | grep -iE "qa|push" &>/dev/null; then diff --git a/docs/mkdocs.yaml b/docs/mkdocs.yaml index 5fdbffd2..2fec112c 100644 --- a/docs/mkdocs.yaml +++ b/docs/mkdocs.yaml @@ -1,7 +1,7 @@ --- # ---------------------- PROJECT SPECIFIC --------------------------- site_name: DataJoint Documentation -site_url: http://localhost/docs/elements/element-array-ephys +site_url: https://docs.datajoint.com/elements/element-array-ephys/ repo_url: https://github.com/datajoint/element-array-ephys repo_name: datajoint/element-array-ephys nav: @@ -9,18 +9,7 @@ nav: - Concepts: concepts.md - Tutorials: - Overview: tutorials/index.md - - Data Download: tutorials/00-data-download-optional.ipynb - - Configure: tutorials/01-configure.ipynb - - Workflow Structure: tutorials/02-workflow-structure-optional.ipynb - - Process: tutorials/03-process.ipynb - - Automate: tutorials/04-automate-optional.ipynb - - Explore: tutorials/05-explore.ipynb - - Drop: tutorials/06-drop-optional.ipynb - - Downstream Analysis: tutorials/07-downstream-analysis.ipynb - - Visualizations: tutorials/10-data_visualization.ipynb - - Electrode Localization: tutorials/08-electrode-localization.ipynb - - NWB Export: tutorials/09-NWB-export.ipynb - - Quality Metrics: tutorials/quality_metrics.ipynb + - Tutorial: tutorials/tutorial.ipynb - Citation: citation.md - API: api/ # defer to gen-files + literate-nav - Changelog: changelog.md diff --git a/docs/src/.overrides/partials/nav.html b/docs/src/.overrides/partials/nav.html index cd6326b6..faac796a 100644 --- a/docs/src/.overrides/partials/nav.html +++ b/docs/src/.overrides/partials/nav.html @@ -7,15 +7,14 @@ {% endif %}