From 00925900480ab16f620a9d188ee349a23156b3ae Mon Sep 17 00:00:00 2001 From: christinehc Date: Wed, 5 Nov 2025 10:11:40 -0800 Subject: [PATCH 01/10] feat: include bmd files in workflow --- .github/workflows/database-update.yml | 269 ++++++++++++++------------ 1 file changed, 140 insertions(+), 129 deletions(-) diff --git a/.github/workflows/database-update.yml b/.github/workflows/database-update.yml index 56c95f1..ae5a194 100644 --- a/.github/workflows/database-update.yml +++ b/.github/workflows/database-update.yml @@ -5,142 +5,153 @@ on: branches: - '**' paths: - - ./data/ + - data/ - # Allows you to run this workflow manually from the Actions tab + # Allows running workflow manually from Actions tab workflow_dispatch: jobs: # Placeholder -# build-bmd: -# #if: github.actor != 'github-actions[bot]' -# runs-on: ubuntu-20.04 -# environment: build -# steps: -# - name: Checkout Repo -# uses: actions/checkout@v3 -# - name: Download artifacts from previous drug gen -# uses: actions/download-artifact@v4 -# - name: Pull sample image -# run: docker pull sgosline/srp-bmd -# - name: Run bmd python command -# run: | -# docker run -v $PWD:/tmp sgosline/srp-bmd -# - name: List files cwd -# run: ls -la - build-samples: - #if: github.actor != 'github-actions[bot]' - runs-on: ubuntu-20.04 - steps: - - name: Checkout Repo - uses: actions/checkout@v3 - - name: Download artifacts from previous drug gen - uses: actions/download-artifact@v4 - - name: Pull sample image - run: docker pull sgosline/srp-samplechem - - name: Run sample-chem python command - run: | - docker run -v $PWD:/tmp sgosline/srp-samplechem - - name: List files cwd - run: ls -la - - name: move files - run: | - mkdir samp-files - mv *csv samp-files - - name: upload artifacts - uses: actions/upload-artifact@v4 - with: - name: samp-files - path: samp-files - build-expo: - #if: github.actor != 'github-actions[bot]' - runs-on: ubuntu-20.04 - steps: - - name: Checkout Repo - uses: actions/checkout@v3 - - name: Download artifacts from previous drug gen - uses: actions/download-artifact@v4 - - name: Pull sample image - run: docker pull sgosline/srp-exposome - - name: Run sample-chem python command - run: | - docker run -v $PWD:/tmp sgosline/srp-exposome - - name: List files cwd - run: ls -la - - name: Copy files to artifact - run: | - mkdir expo-files - mv *.csv expo-files - - name: upload artifact - uses: actions/upload-artifact@v4 - with: - name: expo-files - path: expo-files + build-bmd: + #if: github.actor != 'github-actions[bot]' + runs-on: ubuntu-20.04 + # environment: build + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + - name: Download artifacts from previous drug gen + uses: actions/download-artifact@v4 + - name: Pull sample image + run: docker pull sgosline/srp-bmd + - name: Run bmd python command + run: | + docker run -v $PWD:/tmp sgosline/srp-bmd --morpho /app/zfBmd/test_files/test_morphology.csv --lpr /app/zfBmd/test_files/test_behavioral.csv --output /tmp + - name: List files cwd + run: ls -la + - name: Move files + run: | + mkdir bmd-files + mv *.csv bmd-files + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: bmd-files + path: bmd-files + + build-samples: + #if: github.actor != 'github-actions[bot]' + runs-on: ubuntu-20.04 + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + - name: Download artifacts from previous drug gen + uses: actions/download-artifact@v4 + - name: Pull sample image + run: docker pull sgosline/srp-samplechem + - name: Run sample-chem python command + run: | + docker run -v $PWD:/tmp sgosline/srp-samplechem + - name: List files cwd + run: ls -la + - name: Move files + run: | + mkdir samp-files + mv *csv samp-files + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: samp-files + path: samp-files - build-expr: - #if: github.actor != 'github-actions[bot]' - runs-on: ubuntu-20.04 - needs: build-samples - steps: - - name: Checkout Repo - uses: actions/checkout@v3 - - name: Download artifacts from sample generation - uses: actions/download-artifact@v4 - with: - name: samp-files - path: samp-files - - name: Pull sample image - run: docker pull sgosline/srp-zfexp - - name: move chemical list to tmp - run: mv samp-files/chemicals.csv . - - name: Run zfexp python command - run: | - docker run -v $PWD:/tmp sgosline/srp-zfexp - - name: List files cwd - run: ls -la - - name: Copy files to artifact - run: | - mkdir expr-files - mv *csv expr-files - - name: upload artifact - uses: actions/upload-artifact@v4 - with: - name: expr-files - path: expr-files + build-expo: + #if: github.actor != 'github-actions[bot]' + runs-on: ubuntu-20.04 + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + - name: Download artifacts from previous drug gen + uses: actions/download-artifact@v4 + - name: Pull sample image + run: docker pull sgosline/srp-exposome + - name: Run sample-chem python command + run: | + docker run -v $PWD:/tmp sgosline/srp-exposome + - name: List files cwd + run: ls -la + - name: Copy files to artifact + run: | + mkdir expo-files + mv *.csv expo-files + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: expo-files + path: expo-files - artifact-test: - needs: [build-samples,build-expo,build-expr] - runs-on: ubuntu-20.04 - steps: - - name: Checkout Repo - uses: actions/checkout@v2 - - name: Download artifacts from expression - uses: actions/download-artifact@v4 - with: - name: expr-files - - name: Downlod artifacts from exposome - uses: actions/download-artifact@v4 - with: - name: expo-files - - name: Downlod artifacts from samples - uses: actions/download-artifact@v4 - with: - name: samp-files - - name: List files cwd - run: ls -la * - - name: Moves files to single directory - run: | - mkdir srpAnalytics - mv *csv srpAnalytics - #gzip -cvf srpAnalytics/* srpAnalytics.gz - - name: Pushes to figshare - uses: figshare/github-upload-action@v1.1 - with: - FIGSHARE_TOKEN: ${{ secrets.FIGSHARE_TOKEN }} - FIGSHARE_ENDPOINT: 'https://api.figshare.com/v2' - FIGSHARE_ARTICLE_ID: 26240471 -# FIGSHARE_PROJECT_ID: 177459 - DATA_DIR: 'srpAnalytics' + build-expr: + #if: github.actor != 'github-actions[bot]' + runs-on: ubuntu-20.04 + needs: build-samples + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + - name: Download artifacts from sample generation + uses: actions/download-artifact@v4 + with: + name: samp-files + path: samp-files + - name: Pull sample image + run: docker pull sgosline/srp-zfexp + - name: move chemical list to tmp + run: mv samp-files/chemicals.csv . + - name: Run zfexp python command + run: | + docker run -v $PWD:/tmp sgosline/srp-zfexp + - name: List files cwd + run: ls -la + - name: Copy files to artifact + run: | + mkdir expr-files + mv *.csv expr-files + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: expr-files + path: expr-files + + artifact-test: + needs: [build-samples, build-expo, build-expr] + runs-on: ubuntu-20.04 + steps: + - name: Checkout Repo + uses: actions/checkout@v2 + - name: Download artifacts from expression + uses: actions/download-artifact@v4 + with: + name: expr-files + - name: Download artifacts from exposome + uses: actions/download-artifact@v4 + with: + name: expo-files + - name: Download artifacts from samples + uses: actions/download-artifact@v4 + with: + name: samp-files + - name: List files cwd + run: ls -la * + - name: Moves files to single directory + run: | + mkdir srpAnalytics + mv *.csv srpAnalytics + # gzip -cvf srpAnalytics/* srpAnalytics.gz + - name: Pushes to figshare + uses: figshare/github-upload-action@v1.1 + with: + FIGSHARE_TOKEN: ${{ secrets.FIGSHARE_TOKEN }} + FIGSHARE_ENDPOINT: 'https://api.figshare.com/v2' + FIGSHARE_ARTICLE_ID: 26240471 + # FIGSHARE_PROJECT_ID: 177459 + DATA_DIR: 'srpAnalytics' #zipfiles push to figshare From cda24c02653cf2340bd06662d0aa3eebb7b08107 Mon Sep 17 00:00:00 2001 From: christinehc Date: Wed, 5 Nov 2025 11:17:14 -0800 Subject: [PATCH 02/10] build: use ubuntu-latest for actions --- .github/workflows/database-update.yml | 33 +++++++++++++-------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/.github/workflows/database-update.yml b/.github/workflows/database-update.yml index ae5a194..496ffff 100644 --- a/.github/workflows/database-update.yml +++ b/.github/workflows/database-update.yml @@ -3,18 +3,18 @@ name: Build-database on: push: branches: - - '**' + - "**" paths: - data/ - + # Allows running workflow manually from Actions tab workflow_dispatch: jobs: -# Placeholder + # Placeholder build-bmd: #if: github.actor != 'github-actions[bot]' - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest # environment: build steps: - name: Checkout Repo @@ -37,10 +37,10 @@ jobs: with: name: bmd-files path: bmd-files - + build-samples: #if: github.actor != 'github-actions[bot]' - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - name: Checkout Repo uses: actions/checkout@v3 @@ -50,7 +50,7 @@ jobs: run: docker pull sgosline/srp-samplechem - name: Run sample-chem python command run: | - docker run -v $PWD:/tmp sgosline/srp-samplechem + docker run -v $PWD:/tmp sgosline/srp-samplechem - name: List files cwd run: ls -la - name: Move files @@ -65,7 +65,7 @@ jobs: build-expo: #if: github.actor != 'github-actions[bot]' - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - name: Checkout Repo uses: actions/checkout@v3 @@ -77,7 +77,7 @@ jobs: run: | docker run -v $PWD:/tmp sgosline/srp-exposome - name: List files cwd - run: ls -la + run: ls -la - name: Copy files to artifact run: | mkdir expo-files @@ -86,11 +86,11 @@ jobs: uses: actions/upload-artifact@v4 with: name: expo-files - path: expo-files + path: expo-files build-expr: #if: github.actor != 'github-actions[bot]' - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest needs: build-samples steps: - name: Checkout Repo @@ -108,7 +108,7 @@ jobs: run: | docker run -v $PWD:/tmp sgosline/srp-zfexp - name: List files cwd - run: ls -la + run: ls -la - name: Copy files to artifact run: | mkdir expr-files @@ -121,7 +121,7 @@ jobs: artifact-test: needs: [build-samples, build-expo, build-expr] - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - name: Checkout Repo uses: actions/checkout@v2 @@ -148,10 +148,9 @@ jobs: uses: figshare/github-upload-action@v1.1 with: FIGSHARE_TOKEN: ${{ secrets.FIGSHARE_TOKEN }} - FIGSHARE_ENDPOINT: 'https://api.figshare.com/v2' + FIGSHARE_ENDPOINT: "https://api.figshare.com/v2" FIGSHARE_ARTICLE_ID: 26240471 - # FIGSHARE_PROJECT_ID: 177459 - DATA_DIR: 'srpAnalytics' - + # FIGSHARE_PROJECT_ID: 177459 + DATA_DIR: "srpAnalytics" #zipfiles push to figshare From c0a5138bc54131ca74ada594c96ef2d935d57dd9 Mon Sep 17 00:00:00 2001 From: christinehc Date: Wed, 5 Nov 2025 11:20:17 -0800 Subject: [PATCH 03/10] build: fix container name --- .github/workflows/database-update.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/database-update.yml b/.github/workflows/database-update.yml index 496ffff..41c2805 100644 --- a/.github/workflows/database-update.yml +++ b/.github/workflows/database-update.yml @@ -25,7 +25,7 @@ jobs: run: docker pull sgosline/srp-bmd - name: Run bmd python command run: | - docker run -v $PWD:/tmp sgosline/srp-bmd --morpho /app/zfBmd/test_files/test_morphology.csv --lpr /app/zfBmd/test_files/test_behavioral.csv --output /tmp + docker run -v $PWD:/tmp sgosline/srp-zfbmd --morpho /app/zfBmd/test_files/test_morphology.csv --lpr /app/zfBmd/test_files/test_behavioral.csv --output /tmp - name: List files cwd run: ls -la - name: Move files From d249d78c696760fe60de86044fd13403ee189df1 Mon Sep 17 00:00:00 2001 From: christinehc Date: Wed, 5 Nov 2025 11:22:50 -0800 Subject: [PATCH 04/10] build: fix container name in docker pull --- .github/workflows/database-update.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/database-update.yml b/.github/workflows/database-update.yml index 41c2805..84e714e 100644 --- a/.github/workflows/database-update.yml +++ b/.github/workflows/database-update.yml @@ -22,7 +22,7 @@ jobs: - name: Download artifacts from previous drug gen uses: actions/download-artifact@v4 - name: Pull sample image - run: docker pull sgosline/srp-bmd + run: docker pull sgosline/srp-zfbmd - name: Run bmd python command run: | docker run -v $PWD:/tmp sgosline/srp-zfbmd --morpho /app/zfBmd/test_files/test_morphology.csv --lpr /app/zfBmd/test_files/test_behavioral.csv --output /tmp From 3359cddcff0fcb21258defc0660619567419fd48 Mon Sep 17 00:00:00 2001 From: christinehc Date: Wed, 5 Nov 2025 13:24:01 -0800 Subject: [PATCH 05/10] build: include bmd in artifacts upload. force continue upload --- .github/workflows/database-update.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/database-update.yml b/.github/workflows/database-update.yml index 84e714e..2bdbad0 100644 --- a/.github/workflows/database-update.yml +++ b/.github/workflows/database-update.yml @@ -120,11 +120,15 @@ jobs: path: expr-files artifact-test: - needs: [build-samples, build-expo, build-expr] + needs: [build-bmd, build-samples, build-expo, build-expr] runs-on: ubuntu-latest steps: - name: Checkout Repo uses: actions/checkout@v2 + - name: Download artifacts from bmd + uses: actions/download-artifact@v4 + with: + name: bmd-files - name: Download artifacts from expression uses: actions/download-artifact@v4 with: @@ -146,6 +150,7 @@ jobs: # gzip -cvf srpAnalytics/* srpAnalytics.gz - name: Pushes to figshare uses: figshare/github-upload-action@v1.1 + continue-on-error: true with: FIGSHARE_TOKEN: ${{ secrets.FIGSHARE_TOKEN }} FIGSHARE_ENDPOINT: "https://api.figshare.com/v2" From f42d65b9878fee65e984db1013fed0b871f6a45a Mon Sep 17 00:00:00 2001 From: christinehc Date: Wed, 5 Nov 2025 14:27:20 -0800 Subject: [PATCH 06/10] build: create new figshare articles for each upload --- .github/workflows/database-update.yml | 58 +++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 3 deletions(-) diff --git a/.github/workflows/database-update.yml b/.github/workflows/database-update.yml index 2bdbad0..1d6f0ce 100644 --- a/.github/workflows/database-update.yml +++ b/.github/workflows/database-update.yml @@ -119,8 +119,59 @@ jobs: name: expr-files path: expr-files + create-figshare-article: + runs-on: ubuntu-latest + outputs: + article_id: ${{ steps.create_article.outputs.article_id }} + steps: + - name: Create new private Figshare article + id: create_article + run: | + # Generate timestamp for unique article title + TIMESTAMP=$(date +"%Y-%m-%d_%H-%M-%S") + COMMIT_SHA="${{ github.sha }}" + BRANCH_NAME="${{ github.ref_name }}" + + # Create article metadata + ARTICLE_DATA=$(cat <> $GITHUB_OUTPUT + artifact-test: - needs: [build-bmd, build-samples, build-expo, build-expr] + needs: + [ + build-bmd, + build-samples, + build-expo, + build-expr, + create-figshare-article, + ] runs-on: ubuntu-latest steps: - name: Checkout Repo @@ -154,8 +205,9 @@ jobs: with: FIGSHARE_TOKEN: ${{ secrets.FIGSHARE_TOKEN }} FIGSHARE_ENDPOINT: "https://api.figshare.com/v2" - FIGSHARE_ARTICLE_ID: 26240471 - # FIGSHARE_PROJECT_ID: 177459 + FIGSHARE_ARTICLE_ID: ${{ needs.create-figshare-article.outputs.article_id }} + # FIGSHARE_ARTICLE_ID: 26240471 + # FIGSHARE_PROJECT_ID: 177459 DATA_DIR: "srpAnalytics" #zipfiles push to figshare From f24b584a21e285559b6cde692bdd048ed2e31b8a Mon Sep 17 00:00:00 2001 From: christinehc Date: Wed, 5 Nov 2025 14:34:20 -0800 Subject: [PATCH 07/10] build: use correct category IDs for metadata --- .github/workflows/database-update.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/database-update.yml b/.github/workflows/database-update.yml index 1d6f0ce..459fba8 100644 --- a/.github/workflows/database-update.yml +++ b/.github/workflows/database-update.yml @@ -138,7 +138,7 @@ jobs: "title": "SRP Analytics Database - ${TIMESTAMP}", "description": "Testing automated database build from commit ${COMMIT_SHA} on branch ${BRANCH_NAME}. Generated on ${TIMESTAMP}.", "tags": ["srp-analytics", "database", "automated", "github-actions", "private"], - "categories": [1], + "categories": [26965, 26968], "authors": [{"name": "SRP Analytics Pipeline"}], "defined_type": "dataset" } From d21adaf606665b6b74f56466f60d401bc5e9b931 Mon Sep 17 00:00:00 2001 From: christinehc Date: Wed, 5 Nov 2025 14:42:35 -0800 Subject: [PATCH 08/10] build: use secret for project ID. specify dataset. --- .github/workflows/database-update.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/database-update.yml b/.github/workflows/database-update.yml index 459fba8..20e62eb 100644 --- a/.github/workflows/database-update.yml +++ b/.github/workflows/database-update.yml @@ -131,13 +131,14 @@ jobs: TIMESTAMP=$(date +"%Y-%m-%d_%H-%M-%S") COMMIT_SHA="${{ github.sha }}" BRANCH_NAME="${{ github.ref_name }}" + FIGSHARE_PROJECT_ID="${{ secrets.FIGSHARE_PROJECT_ID }}" - # Create article metadata + # Define article metadata ARTICLE_DATA=$(cat < Date: Wed, 5 Nov 2025 14:45:07 -0800 Subject: [PATCH 09/10] build: remove tags from metadata --- .github/workflows/database-update.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/database-update.yml b/.github/workflows/database-update.yml index 20e62eb..a6fafea 100644 --- a/.github/workflows/database-update.yml +++ b/.github/workflows/database-update.yml @@ -134,11 +134,10 @@ jobs: FIGSHARE_PROJECT_ID="${{ secrets.FIGSHARE_PROJECT_ID }}" # Define article metadata - ARTICLE_DATA=$(cat <> $GITHUB_OUTPUT + # metadata field: "tags": ["srp-analytics", "database"], + artifact-test: needs: [ @@ -207,8 +208,6 @@ jobs: FIGSHARE_TOKEN: ${{ secrets.FIGSHARE_TOKEN }} FIGSHARE_ENDPOINT: "https://api.figshare.com/v2" FIGSHARE_ARTICLE_ID: ${{ needs.create-figshare-article.outputs.article_id }} - # FIGSHARE_ARTICLE_ID: 26240471 - # FIGSHARE_PROJECT_ID: 177459 DATA_DIR: "srpAnalytics" #zipfiles push to figshare From 743ee3c5548ff58f9c2e8db98caa222afcd8e4af Mon Sep 17 00:00:00 2001 From: christinehc Date: Wed, 5 Nov 2025 14:48:43 -0800 Subject: [PATCH 10/10] build: fix variable name --- .github/workflows/database-update.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/database-update.yml b/.github/workflows/database-update.yml index a6fafea..6edc3e1 100644 --- a/.github/workflows/database-update.yml +++ b/.github/workflows/database-update.yml @@ -131,7 +131,7 @@ jobs: TIMESTAMP=$(date +"%Y-%m-%d_%H-%M-%S") COMMIT_SHA="${{ github.sha }}" BRANCH_NAME="${{ github.ref_name }}" - FIGSHARE_PROJECT_ID="${{ secrets.FIGSHARE_PROJECT_ID }}" + PROJECT_ID="${{ secrets.FIGSHARE_PROJECT_ID }}" # Define article metadata METADATA=$(cat <