From 9495ea86fa75390083ebf05f9af722a74abf995a Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Thu, 18 Dec 2025 10:10:47 +0100 Subject: [PATCH 1/2] Unify scripts to check probes + autogen CN library --- .../check_for_changes_in_NP_jsons.yml | 13 ++- ...probes.yml => check_for_new_NP_probes.yml} | 0 .../generate_cambridge_neurotech_library.yml | 79 +++++++++++++++++++ .../check_for_json_changes_in_NP_probes.py | 26 ------ scripts/check_for_json_changes_in_probes.py | 44 +++++++++++ 5 files changed, 128 insertions(+), 34 deletions(-) rename .github/workflows/{check_for_new_probes.yml => check_for_new_NP_probes.yml} (100%) create mode 100644 .github/workflows/generate_cambridge_neurotech_library.yml delete mode 100644 scripts/check_for_json_changes_in_NP_probes.py create mode 100644 scripts/check_for_json_changes_in_probes.py diff --git a/.github/workflows/check_for_changes_in_NP_jsons.yml b/.github/workflows/check_for_changes_in_NP_jsons.yml index 3606d9c..0c538ad 100644 --- a/.github/workflows/check_for_changes_in_NP_jsons.yml +++ b/.github/workflows/check_for_changes_in_NP_jsons.yml @@ -38,19 +38,16 @@ jobs: pip install ./probeinterface - - name: Generate full NP library + - name: Generate full NP library and check for changes run: | cd scripts/ + # generate full library python ../probeinterface/resources/generate_neuropixels_library.py - - # Check for any new probes - - name: Run local script - run: | - cd scripts/ - python check_for_json_changes_in_NP_probes.py + # check for json changes (except probeinterface version) + python check_for_json_changes_in_probes.py ../imec/ ./neuropixels_library_generated/ + # clean up rm -r neuropixels_library_generated/ cd .. - rm -r ./probeinterface - name: Commit changes if any diff --git a/.github/workflows/check_for_new_probes.yml b/.github/workflows/check_for_new_NP_probes.yml similarity index 100% rename from .github/workflows/check_for_new_probes.yml rename to .github/workflows/check_for_new_NP_probes.yml diff --git a/.github/workflows/generate_cambridge_neurotech_library.yml b/.github/workflows/generate_cambridge_neurotech_library.yml new file mode 100644 index 0000000..09e31ce --- /dev/null +++ b/.github/workflows/generate_cambridge_neurotech_library.yml @@ -0,0 +1,79 @@ +name: Generate Cambridge Neurotech library + +on: + schedule: + - cron: '0 1 * * 1' # Every Monday at 01:00 UTC + workflow_dispatch: + +# Grant permissions for the job to write to contents (push) and create PRs. +permissions: + contents: write + pull-requests: write + +jobs: + build-and-pr: + runs-on: ubuntu-latest + steps: + + - name: Check out local repository + uses: actions/checkout@v4 + + # Set up a Python environment + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + # Clone dev version of probeinterface + - name: Clone external repositories + run: | + # probeinterface + git clone https://github.com/samuelgarcia/probeinterface ./probeinterface + # for now, we checkout the two-sided branch + cd probeinterface + git checkout two_side_probes + cd .. + + # cambridge neurotech probe maps + git clone https://github.com/cambridge-neurotech/probe_maps.git ./probe_maps + + - name: Install probeinterface and matplotlib + run: pip install ./probeinterface matplotlib + + - name: Generate full cambridge library and check for changes + run: | + cd scripts/ + python ../probeinterface/resources/generate_cambridgeneurotech_library.py ../probe_maps/ ./cambridge_library_generated/ + + # check for json changes (except probeinterface version) + python check_for_json_changes_in_probes.py ../cambridgeneurotech/ ./cambridge_library_generated/ + # clean up + rm -r cambridge_library_generated/ + cd .. + rm -r ./probeinterface + + - name: Commit changes if any + id: commit + run: | + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + + git add cambridgeneurotech/* + + # Only commit if there are changes + if git diff --staged --quiet; then + echo "No changes to commit" + echo "changes=false" >> $GITHUB_OUTPUT + else + git commit -m "Update json files for NP probes" + echo "changes=true" >> $GITHUB_OUTPUT + fi + + - name: Create pull request to add probes + if: steps.commit.outputs.changes == 'true' + uses: peter-evans/create-pull-request@v7 + with: + title: "Update Cambridge Neurotech json files" + body: "This PR updates the Neuropixel probes in probeinterace library, based on new data from the ProbeTable repository or because of a new ProbeInterface release." + branch-suffix: short-commit-hash + base: "main" diff --git a/scripts/check_for_json_changes_in_NP_probes.py b/scripts/check_for_json_changes_in_NP_probes.py deleted file mode 100644 index 87e2ae9..0000000 --- a/scripts/check_for_json_changes_in_NP_probes.py +++ /dev/null @@ -1,26 +0,0 @@ -from pathlib import Path -import shutil - -old_dir = Path('../imec') -new_dir = Path('./neuropixels_library_generated') - -for temp_probe_directory in new_dir.iterdir(): - - probe_name = str(temp_probe_directory.name) - - temp_probe_json_path = temp_probe_directory / (probe_name + '.json') - old_probe_json_path = old_dir / probe_name / (probe_name + '.json') - - if old_probe_json_path.is_file(): - with open(temp_probe_json_path, 'r') as f1, open(old_probe_json_path, 'r') as f2: - # Read in json files - lines1 = f1.readlines() - lines2 = f2.readlines() - - # We don't want to update the probes just because of a probeinterface version update. - # The probeinterface version is stored on the 3rd line of the json file, so we only - # compare the json files from line 3 and down. - if lines1[3:] == lines2[3:]: - continue - else: - shutil.copy(f"{temp_probe_json_path}", f"../imec/{probe_name}") \ No newline at end of file diff --git a/scripts/check_for_json_changes_in_probes.py b/scripts/check_for_json_changes_in_probes.py new file mode 100644 index 0000000..e0eb4b0 --- /dev/null +++ b/scripts/check_for_json_changes_in_probes.py @@ -0,0 +1,44 @@ +from argparse import ArgumentParser +from pathlib import Path +import shutil + + +parser = ArgumentParser(description="Check for JSON changes in probes from manufacturer") + +parser.add_argument( + "old_dir", + type=str, + help="Path to the old probes directory", +) + +parser.add_argument( + "new_dir", + type=str, + help="Path to the new probes directory", +) + + +if __name__ == "__main__": + args = parser.parse_args() + old_dir = Path(args.old_dir) + new_dir = Path(args.new_dir) + for temp_probe_directory in new_dir.iterdir(): + probe_name = str(temp_probe_directory.name) + + temp_probe_json_path = temp_probe_directory / (probe_name + '.json') + old_probe_json_path = old_dir / probe_name / (probe_name + '.json') + + if old_probe_json_path.is_file(): + with open(temp_probe_json_path, 'r') as f1, open(old_probe_json_path, 'r') as f2: + # Read in json files + lines1 = f1.readlines() + lines2 = f2.readlines() + + # We don't want to update the probes just because of a probeinterface version update. + # The probeinterface version is stored on the 3rd line of the json file, so we only + # compare the json files from line 3 and down. + if lines1[3:] == lines2[3:]: + continue + else: + shutil.copy(f"{temp_probe_json_path}", f"../imec/{probe_name}") + \ No newline at end of file From ccda32f0c209205fe63e45eecc917bae0e6e15ee Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Thu, 18 Dec 2025 10:43:19 +0100 Subject: [PATCH 2/2] USe probeinterface main --- .github/workflows/generate_cambridge_neurotech_library.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/generate_cambridge_neurotech_library.yml b/.github/workflows/generate_cambridge_neurotech_library.yml index 09e31ce..2089d57 100644 --- a/.github/workflows/generate_cambridge_neurotech_library.yml +++ b/.github/workflows/generate_cambridge_neurotech_library.yml @@ -28,11 +28,7 @@ jobs: - name: Clone external repositories run: | # probeinterface - git clone https://github.com/samuelgarcia/probeinterface ./probeinterface - # for now, we checkout the two-sided branch - cd probeinterface - git checkout two_side_probes - cd .. + git clone https://github.com/spikeinterface/probeinterface ./probeinterface # cambridge neurotech probe maps git clone https://github.com/cambridge-neurotech/probe_maps.git ./probe_maps