Skip to content
DEV

Feat: Adds interactive-session resume #58

Sign in to view logs

Feat: Adds interactive-session resume

Feat: Adds interactive-session resume #58

Workflow file for this run

.github/workflows/ci_dev.yml at 84cc896
name: DEV
on:
pull_request:
types: [review_requested, ready_for_review]
branches:
- main
- dev
paths-ignore:
- 'docs/**'
workflow_dispatch:
permissions:
contents: read
jobs:
job_list_dev:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.9"]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: pip
cache-dependency-path: setup.py
- name: Install dependencies
run: |
pip install -e .
- name: Run tests
env:
CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN_DEV }}
CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID_DEV }}
CLOUDOS_URL: "https://dev.sdlc.lifebit.ai"
run: |
cloudos job list --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID
job_list_filtering_dev:
needs: job_run_and_status_dev
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.9"]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: pip
cache-dependency-path: setup.py
- name: Install dependencies
run: |
pip install -e .
- name: Test job list filtering
env:
CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN_DEV }}
CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID_DEV }}
CLOUDOS_URL: "https://dev.sdlc.lifebit.ai"
run: |
JOB_ID="${{ needs.job_run_and_status_dev.outputs.job_id }}"
# Test filtering by status, project and workflow name
cloudos job list --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --filter-status completed --filter-project cloudos-cli-tests --filter-workflow GH-rnatoy --last --last-n-jobs 10
# Test filtering job id
cloudos job list --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --filter-job-id $JOB_ID
# Test filtering job name
cloudos job list --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --filter-job-name "cloudos-cli-CI-test" --last-n-jobs 10
# Test filtering by only mine
cloudos job list --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --filter-only-mine --last-n-jobs 10
# Test filtering by queue
#cloudos job list --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --filter-queue "cost_saving_standard_nextflow" --last-n-jobs 10
job_details_dev:
needs: job_run_and_status_dev
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.9"]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: pip
cache-dependency-path: setup.py
- name: Install dependencies
run: |
pip install -e .
- name: Run tests
env:
CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN_DEV }}
CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID_DEV }}
CLOUDOS_URL: "https://dev.sdlc.lifebit.ai"
run: |
JOB_ID="${{ needs.job_run_and_status_dev.outputs.job_id }}"
cloudos job details --cloudos-url $CLOUDOS_URL --workspace-id $CLOUDOS_WORKSPACE_ID --apikey $CLOUDOS_TOKEN --job-id $JOB_ID
job_workdir_dev:
needs: job_run_and_status_dev
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.9"]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: pip
cache-dependency-path: setup.py
- name: Install dependencies
run: |
pip install -e .
- name: Run tests
env:
CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN_DEV }}
CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID_DEV }}
CLOUDOS_URL: "https://dev.sdlc.lifebit.ai"
run: |
JOB_ID="${{ needs.job_run_and_status_dev.outputs.job_id }}"
cloudos job workdir --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --job-id $JOB_ID
job_related_analyses_dev:
needs: job_run_and_status_dev
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.9"]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: pip
cache-dependency-path: setup.py
- name: Install dependencies
run: |
pip install -e .
- name: Run tests
env:
CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN_DEV }}
CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID_DEV }}
CLOUDOS_URL: "https://dev.sdlc.lifebit.ai"
run: |
JOB_ID="${{ needs.job_run_and_status_dev.outputs.job_id }}"
cloudos job related --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --job-id $JOB_ID
import_gitlab_dev:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: [ "3.9" ]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: pip
cache-dependency-path: setup.py
- name: Install dependencies
run: |
pip install -e .
- name: Run tests
env:
CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN_DEV }}
CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID_DEV }}
CLOUDOS_URL: "https://dev.sdlc.lifebit.ai"
run: |
cloudos workflow import --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --workflow-name imported_from_gitlab --workflow-url https://gitlab.com/lifebit-ai/spammer-nf --repository-platform gitlab
import_github_dev:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: [ "3.9" ]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: pip
cache-dependency-path: setup.py
- name: Install dependencies
run: |
pip install -e .
- name: Run tests
env:
CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN_DEV }}
CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID_DEV }}
CLOUDOS_URL: "https://dev.sdlc.lifebit.ai"
run: |
cloudos workflow import --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --workflow-name imported_from_github --workflow-url https://github.com/lifebit-ai/spammer-nf --repository-platform github
job_run_and_status_dev:
runs-on: ubuntu-latest
outputs:
job_id: ${{ steps.get-job-id.outputs.job_id }}
strategy:
matrix:
python-version: ["3.9"]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: pip
cache-dependency-path: setup.py
- name: Install dependencies
run: |
pip install -e .
- name: Run tests
id: get-job-id
env:
CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN_DEV }}
CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID_DEV }}
CLOUDOS_URL: "https://dev.sdlc.lifebit.ai"
PROJECT_NAME: "cloudos-cli-tests"
WORKFLOW: "GH-rnatoy"
JOB_CONFIG: "cloudos_cli/examples/rnatoy.config"
JOB_NAME_BASE: "cloudos-cli-CI-test"
COMMIT_HASH: ${{ github.event.pull_request.head.sha || github.sha }}
PR_NUMBER: ${{ github.event.pull_request.number || 'manual' }}
INSTANCE_TYPE: "m4.xlarge"
run: |
JOB_NAME="$JOB_NAME_BASE""|GitHubCommit:""${COMMIT_HASH:0:6}""|PR-NUMBER:""$PR_NUMBER"
cloudos job run --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --project-name "$PROJECT_NAME" --workflow-name "$WORKFLOW" --job-config $JOB_CONFIG --job-name "$JOB_NAME" --wait-completion --resumable --instance-type $INSTANCE_TYPE 2>&1 | tee out.txt
JOB_ID=$(grep -e "Your assigned job id is:" out.txt | rev | cut -f1 -d " " | rev)
echo "job_id=$JOB_ID" >> $GITHUB_OUTPUT
cloudos job status --cloudos-url $CLOUDOS_URL --workspace-id $CLOUDOS_WORKSPACE_ID --apikey $CLOUDOS_TOKEN --job-id $JOB_ID
job_resume_dev:
needs: job_run_and_status_dev
outputs:
job_id: ${{ steps.get-resumed-job-id.outputs.job_id }}
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.9"]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: pip
cache-dependency-path: setup.py
- name: Install dependencies
run: |
pip install -e .
- name: Resume completed job
id: get-resumed-job-id
env:
CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN_DEV }}
CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID_DEV }}
CLOUDOS_URL: "https://dev.sdlc.lifebit.ai"
PROJECT_NAME: "cloudos-cli-tests"
INSTANCE_TYPE: "m4.xlarge"
JOB_NAME_BASE: "cloudos-cli-CI-test-resume"
run: |
JOB_ID_TO_RESUME="${{ needs.job_run_and_status_dev.outputs.job_id }}"
echo "Resuming job $JOB_ID_TO_RESUME..."
cloudos job resume --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --project-name "$PROJECT_NAME" --job-name "$JOB_NAME_BASE" --job-id $JOB_ID_TO_RESUME --wait-completion --instance-type $INSTANCE_TYPE 2>&1 | tee out.txt
JOB_ID=$(grep -e "Job successfully resumed. New job ID:" out.txt | rev | cut -f1 -d " " | rev)
echo "job_id=$JOB_ID" >> $GITHUB_OUTPUT
echo "Job resumed successfully!"
logs_results_aws_dev:
needs: job_run_and_status_dev
runs-on: ubuntu-latest
strategy:
matrix:
python-version: [ "3.9" ]
feature: ["logs", "results"]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: pip
cache-dependency-path: setup.py
- name: Install dependencies
run: |
pip install -e .
- name: Run tests
env:
CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN_DEV }}
CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID_DEV }}
CLOUDOS_URL: "https://dev.sdlc.lifebit.ai"
run: |
cloudos job ${{ matrix.feature }} --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --job-id ${{ needs.job_run_and_status_dev.outputs.job_id }}
workflow_list_dev:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.9"]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: pip
cache-dependency-path: setup.py
- name: Install dependencies
run: |
pip install -e .
- name: Run tests
env:
CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN_DEV }}
CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID_DEV }}
CLOUDOS_URL: "https://dev.sdlc.lifebit.ai"
run: |
# stdout
echo 'q'|cloudos workflow list --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID
# csv
cloudos workflow list --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --output-format csv
# json
cloudos workflow list --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --output-format json
project_list_dev:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.9"]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: pip
cache-dependency-path: setup.py
- name: Install dependencies
run: |
pip install -e .
- name: Run tests
env:
CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN_DEV }}
CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID_DEV }}
CLOUDOS_URL: "https://dev.sdlc.lifebit.ai"
run: |
# stdout
echo 'q'|cloudos project list --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID
# csv
cloudos project list --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --output-format csv
# json
cloudos project list --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --output-format json
queue_list_dev:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.9"]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: pip
cache-dependency-path: setup.py
- name: Install dependencies
run: |
pip install -e .
- name: Run tests
env:
CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN_DEV }}
CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID_DEV }}
CLOUDOS_URL: "https://dev.sdlc.lifebit.ai"
run: |
# stdout
echo 'q'|cloudos queue list --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID
# csv
cloudos queue list --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --output-format csv
# json
cloudos queue list --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --output-format json
dataset_list_dev:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.9"]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: pip
cache-dependency-path: setup.py
- name: Install dependencies
run: |
pip install -e .
- name: Run tests
env:
CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN_DEV }}
CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID_DEV }}
PROJECT_NAME: "cloudos-cli-tests"
CLOUDOS_URL: "https://dev.sdlc.lifebit.ai"
run: |
cloudos datasets ls --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --project-name "$PROJECT_NAME"
dataset_move_dev:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.9"]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: pip
cache-dependency-path: setup.py
- name: Install dependencies
run: |
pip install -e .
- name: Run tests
env:
CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN_DEV }}
CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID_DEV }}
PROJECT_NAME: "cloudos-cli-tests"
CLOUDOS_URL: "https://dev.sdlc.lifebit.ai"
run: |
cloudos datasets mv --cloudos-url "$CLOUDOS_URL" --apikey "$CLOUDOS_TOKEN" --workspace-id "$CLOUDOS_WORKSPACE_ID" --project-name DB-new Data/test_mv/test_file.txt --destination-project-name "$PROJECT_NAME" Data/mv_test
cloudos datasets mv --cloudos-url "$CLOUDOS_URL" --apikey "$CLOUDOS_TOKEN" --workspace-id "$CLOUDOS_WORKSPACE_ID" --project-name "$PROJECT_NAME" Data/mv_test/test_file.txt --destination-project-name DB-new Data/test_mv
dataset_rename_dev:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.9"]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: pip
cache-dependency-path: setup.py
- name: Install dependencies
run: |
pip install -e .
- name: Run tests
env:
CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN_DEV }}
CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID_DEV }}
PROJECT_NAME: "cloudos-cli-tests"
CLOUDOS_URL: "https://dev.sdlc.lifebit.ai"
run: |
cloudos datasets rename --cloudos-url "$CLOUDOS_URL" --apikey "$CLOUDOS_TOKEN" --workspace-id "$CLOUDOS_WORKSPACE_ID" --project-name "$PROJECT_NAME" Data/test_rename new_renamed_folder
cloudos datasets rename --cloudos-url "$CLOUDOS_URL" --apikey "$CLOUDOS_TOKEN" --workspace-id "$CLOUDOS_WORKSPACE_ID" --project-name "$PROJECT_NAME" Data/new_renamed_folder test_rename
bash_job_dev:
runs-on: ubuntu-latest
outputs:
job_id: ${{ steps.get-bash-job-id.outputs.job_id }}
strategy:
matrix:
python-version: ["3.9"]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: pip
cache-dependency-path: setup.py
- name: Install dependencies
run: |
pip install -e .
- name: Run tests
id: get-bash-job-id
env:
CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN_DEV }}
CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID_DEV }}
CLOUDOS_URL: "https://dev.sdlc.lifebit.ai"
PROJECT_NAME: "cloudos-cli-tests"
WORKFLOW: "plink2"
JOB_NAME_BASE: "cloudos_cli_CI_test_bash_job"
COMMIT_HASH: ${{ github.event.pull_request.head.sha || github.sha }}
PR_NUMBER: ${{ github.event.pull_request.number || 'manual' }}
INSTANCE_TYPE: "m4.xlarge"
run: |
cloudos bash job \
--instance-type m4.xlarge \
--memory 2 \
--parameter -test=value \
--parameter --gaq=test \
--job-name "$JOB_NAME_BASE" \
--command "echo 'test' > new_file.txt" \
--workspace-id "$CLOUDOS_WORKSPACE_ID" \
--project-name "$PROJECT_NAME" \
--workflow-name plink2 \
--apikey "$CLOUDOS_TOKEN" \
--cloudos-url "$CLOUDOS_URL" \
--last \
--wait-completion 2>&1 | tee bash_job_out.txt
JOB_ID=$(grep -e "Your assigned job id is:" bash_job_out.txt | rev | cut -f1 -d " " | rev)
echo "job_id=$JOB_ID" >> $GITHUB_OUTPUT
bash_job_clone_dev:
needs: bash_job_dev
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.9"]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: pip
cache-dependency-path: setup.py
- name: Install dependencies
run: |
pip install -e .
- name: Run tests
id: run-bash-job
env:
CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN_DEV }}
CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID_DEV }}
CLOUDOS_URL: "https://dev.sdlc.lifebit.ai"
PROJECT_NAME: "cloudos-cli-tests"
JOB_NAME_BASE: "cloudos_cli_CI_test_bash_job_cloned"
INSTANCE_TYPE: "m4.xlarge"
run: |
JOB_ID="${{ needs.bash_job_dev.outputs.job_id }}"
cloudos job clone \
--job-id "$JOB_ID" \
--job-name "$JOB_NAME_BASE" \
--instance-type "$INSTANCE_TYPE" \
--workspace-id "$CLOUDOS_WORKSPACE_ID" \
--project-name "$PROJECT_NAME" \
--apikey "$CLOUDOS_TOKEN" \
--cloudos-url "$CLOUDOS_URL"
bash_array_job_run_and_multiple_projects_dev:
runs-on: ubuntu-latest
outputs:
job_id: ${{ steps.get-bash-array-job-id.outputs.job_id }}
strategy:
matrix:
python-version: ["3.9"]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: pip
cache-dependency-path: setup.py
- name: Install dependencies
run: |
pip install -e .
- name: Run tests
id: get-bash-array-job-id
env:
CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN_DEV }}
CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID_DEV }}
CLOUDOS_URL: "https://dev.sdlc.lifebit.ai"
PROJECT_NAME: "cloudos-cli-tests"
WORKFLOW: "plink2"
JOB_NAME_BASE: "cloudos_cli_CI_test_bash_array_job"
COMMIT_HASH: ${{ github.event.pull_request.head.sha || github.sha }}
PR_NUMBER: ${{ github.event.pull_request.number || 'manual' }}
run: |
cloudos bash array-job \
--instance-type m4.xlarge \
--memory 2 \
--parameter -test=value \
--parameter --gaq=test \
--array-parameter --array_var=title \
--job-name "$JOB_NAME_BASE" \
--command "echo 'test' > new_file.txt" \
--workspace-id "$CLOUDOS_WORKSPACE_ID" \
--project-name "$PROJECT_NAME" \
--workflow-name plink2 \
--array-file Data/bash_array/sampleArray.csv \
--separator "," \
--apikey "$CLOUDOS_TOKEN" \
--cloudos-url "$CLOUDOS_URL" \
--parameter --file=DB-new/Data/input.csv \
--last \
--wait-completion 2>&1 | tee bash_array_job_out.txt
JOB_ID=$(grep -e "Your assigned job id is:" bash_array_job_out.txt | rev | cut -f1 -d " " | rev)
echo "job_id=$JOB_ID" >> $GITHUB_OUTPUT
bash_array_clone_dev:
needs: bash_array_job_run_and_multiple_projects_dev
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.9"]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: pip
cache-dependency-path: setup.py
- name: Install dependencies
run: |
pip install -e .
- name: Run tests
id: run-bash-job
env:
CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN_DEV }}
CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID_DEV }}
CLOUDOS_URL: "https://dev.sdlc.lifebit.ai"
PROJECT_NAME: "cloudos-cli-tests"
JOB_NAME_BASE: "cloudos_cli_CI_test_bash_array_job_cloned"
INSTANCE_TYPE: "m4.xlarge"
run: |
JOB_ID="${{ needs.bash_array_job_run_and_multiple_projects_dev.outputs.job_id }}"
cloudos job clone \
--job-id "$JOB_ID" \
--job-name "$JOB_NAME_BASE" \
--instance-type "$INSTANCE_TYPE" \
--workspace-id "$CLOUDOS_WORKSPACE_ID" \
--project-name "$PROJECT_NAME" \
--apikey "$CLOUDOS_TOKEN" \
--cloudos-url "$CLOUDOS_URL"
dataset_copy_and_remove_dev:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.9"]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: pip
cache-dependency-path: setup.py
- name: Install dependencies
run: |
pip install -e .
- name: Run tests
env:
CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN_DEV }}
CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID_DEV }}
PROJECT_NAME: "cloudos-cli-tests"
CLOUDOS_URL: "https://dev.sdlc.lifebit.ai"
run: |
cloudos datasets cp --cloudos-url "$CLOUDOS_URL" --apikey "$CLOUDOS_TOKEN" --workspace-id "$CLOUDOS_WORKSPACE_ID" --project-name "$PROJECT_NAME" Data/mv_test/diffexpr-results.csv Data/test_copy
cloudos datasets rm --cloudos-url "$CLOUDOS_URL" --apikey "$CLOUDOS_TOKEN" --workspace-id "$CLOUDOS_WORKSPACE_ID" --project-name "$PROJECT_NAME" Data/test_copy/diffexpr-results.csv --force
dataset_mkdir_and_remove_dev:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.9"]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: pip
cache-dependency-path: setup.py
- name: Install dependencies
run: |
pip install -e .
- name: Run tests
env:
CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN_DEV }}
CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID_DEV }}
PROJECT_NAME: "cloudos-cli-tests"
CLOUDOS_URL: "https://dev.sdlc.lifebit.ai"
run: |
cloudos datasets mkdir --cloudos-url "$CLOUDOS_URL" --apikey "$CLOUDOS_TOKEN" --workspace-id "$CLOUDOS_WORKSPACE_ID" --project-name "$PROJECT_NAME" Data/rm_test
cloudos datasets rm --cloudos-url "$CLOUDOS_URL" --apikey "$CLOUDOS_TOKEN" --workspace-id "$CLOUDOS_WORKSPACE_ID" --project-name "$PROJECT_NAME" Data/rm_test
archive_unarchive_job_dev:
needs: job_run_and_status_dev
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.9"]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: pip
cache-dependency-path: setup.py
- name: Install dependencies
run: |
pip install -e .
- name: Run tests
env:
CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN_DEV }}
CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID_DEV }}
PROJECT_NAME: "cloudos-cli-tests"
CLOUDOS_URL: "https://dev.sdlc.lifebit.ai"
run: |
JOB_ID="${{ needs.job_run_and_status_dev.outputs.job_id }}"
cloudos job archive --cloudos-url "$CLOUDOS_URL" --apikey "$CLOUDOS_TOKEN" --workspace-id "$CLOUDOS_WORKSPACE_ID" --job-ids $JOB_ID
cloudos job unarchive --cloudos-url "$CLOUDOS_URL" --apikey "$CLOUDOS_TOKEN" --workspace-id "$CLOUDOS_WORKSPACE_ID" --job-ids $JOB_ID
configure_profile_dev:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.9"]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: pip
cache-dependency-path: setup.py
- name: Install dependencies
run: |
pip install -e .
- name: Test configure profile workflow
env:
CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN_DEV }}
CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID_DEV }}
CLOUDOS_URL: "https://dev.sdlc.lifebit.ai"
PROJECT_NAME: "cloudos-cli-tests"
run: |
# Step 1: Create a test profile by providing inputs via stdin
echo "Creating profile 'ci-test-profile'..."
echo -e "$CLOUDOS_TOKEN\n$CLOUDOS_URL\n$CLOUDOS_WORKSPACE_ID\n\n$PROJECT_NAME\n\n\n\n\n" | cloudos configure --profile ci-test-profile
echo "Profile created successfully!"
# Step 2: List all profiles to verify creation
echo "Listing all profiles..."
cloudos configure list-profiles | tee profiles_output.txt
# Verify that our test profile exists in the list
if grep -q "ci-test-profile" profiles_output.txt; then
echo "✅ Profile 'ci-test-profile' found in the list"
else
echo "❌ Profile 'ci-test-profile' not found in the list"
exit 1
fi
# Step 3: Remove the test profile
echo "Removing profile 'ci-test-profile'..."
cloudos configure remove-profile --profile ci-test-profile
echo "Profile removed successfully!"
# Step 4: List profiles again to verify removal
echo "Listing profiles after removal..."
cloudos configure list-profiles | tee profiles_after_removal.txt
# Verify that our test profile no longer exists
if grep -q "ci-test-profile" profiles_after_removal.txt; then
echo "❌ Profile 'ci-test-profile' still exists after removal"
exit 1
else
echo "✅ Profile 'ci-test-profile' successfully removed"
fi
echo "Configure profile workflow test completed successfully!"
job_run_and_abort_dev:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.9"]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: pip
cache-dependency-path: setup.py
- name: Install dependencies
run: |
pip install -e .
- name: Run job, wait for running status, then abort
env:
CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN_DEV }}
CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID_DEV }}
CLOUDOS_URL: "https://dev.sdlc.lifebit.ai"
PROJECT_NAME: "cloudos-cli-tests"
WORKFLOW: "GH-rnatoy"
JOB_CONFIG: "cloudos_cli/examples/rnatoy.config"
JOB_NAME_BASE: "cloudos-cli-CI-test-abort"
COMMIT_HASH: ${{ github.event.pull_request.head.sha || github.sha }}
PR_NUMBER: ${{ github.event.pull_request.number || 'manual' }}
INSTANCE_TYPE: "m4.xlarge"
run: |
# Step 1: Run job (without --wait-completion so it doesn't wait until completion)
JOB_NAME="$JOB_NAME_BASE""|GitHubCommit:""${COMMIT_HASH:0:6}""|PR-NUMBER:""$PR_NUMBER"
cloudos job run --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --project-name "$PROJECT_NAME" --workflow-name "$WORKFLOW" --job-config $JOB_CONFIG --job-name "$JOB_NAME" --instance-type $INSTANCE_TYPE 2>&1 | tee out.txt
JOB_ID=$(grep -e "Your assigned job id is:" out.txt | rev | cut -f1 -d " " | rev)
echo "Job ID: $JOB_ID"
# Step 2: Poll status until it reaches "running"
MAX_ATTEMPTS=60
SLEEP_TIME=10
attempt=0
while [ $attempt -lt $MAX_ATTEMPTS ]; do
echo "Checking job status (attempt $((attempt+1))/$MAX_ATTEMPTS)..."
STATUS=$(cloudos job status --cloudos-url $CLOUDOS_URL --workspace-id $CLOUDOS_WORKSPACE_ID --apikey $CLOUDOS_TOKEN --job-id $JOB_ID 2>&1 | grep -i "Your current job status is" | head -1 || echo "unknown")
echo "Current status: $STATUS"
if echo "$STATUS" | grep -qi "running"; then
echo "Job is now running!"
break
elif echo "$STATUS" | grep -qi "completed\|failed\|aborted"; then
echo "Job finished before we could abort it (status: $STATUS)"
exit 1
fi
attempt=$((attempt+1))
sleep $SLEEP_TIME
done
if [ $attempt -eq $MAX_ATTEMPTS ]; then
echo "Job did not reach running status within timeout"
exit 1
fi
# Step 3: Abort the running job
echo "Aborting job $JOB_ID..."
cloudos job abort --cloudos-url "$CLOUDOS_URL" --apikey "$CLOUDOS_TOKEN" --workspace-id "$CLOUDOS_WORKSPACE_ID" --job-ids $JOB_ID
echo "Job abort command executed successfully!"
delete_workdir_dev:
needs: [job_run_and_status_dev, job_workdir_dev, job_resume_dev]
runs-on: ubuntu-latest
strategy:
matrix:
python-version: [ "3.9" ]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: pip
cache-dependency-path: setup.py
- name: Install dependencies
run: |
pip install -e .
- name: Run tests
env:
CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN_DEV }}
CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID_DEV }}
CLOUDOS_URL: "https://dev.sdlc.lifebit.ai"
run: |
cloudos job workdir --delete --yes --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --job-id ${{ needs.job_resume_dev.outputs.job_id }}
delete_results_dev:
needs: [job_run_and_status_dev, job_workdir_dev, job_resume_dev]
runs-on: ubuntu-latest
strategy:
matrix:
python-version: [ "3.9" ]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: pip
cache-dependency-path: setup.py
- name: Install dependencies
run: |
pip install -e .
- name: Run tests
env:
CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN_DEV }}
CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID_DEV }}
CLOUDOS_URL: "https://dev.sdlc.lifebit.ai"
run: |
cloudos job results --delete --yes --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --job-id ${{ needs.job_resume_dev.outputs.job_id }}
params_file_dev:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: [ "3.9" ]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: pip
cache-dependency-path: setup.py
- name: Install dependencies
run: |
pip install -e .
- name: Run tests
env:
CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN_DEV }}
CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID_DEV }}
CLOUDOS_URL: "https://dev.sdlc.lifebit.ai"
PROJECT_NAME: "cloudos-cli-tests"
WORKFLOW: "GH-rnatoy"
PARAMS_FILE: "Data/rnatoy_params.json"
JOB_NAME_BASE: "cloudos_cli_CI_test_params_file"
COMMIT_HASH: ${{ github.event.pull_request.head.sha || github.sha }}
PR_NUMBER: ${{ github.event.pull_request.number || 'manual' }}
run: |
cloudos job run \
--cloudos-url $CLOUDOS_URL \
--apikey $CLOUDOS_TOKEN \
--workspace-id $CLOUDOS_WORKSPACE_ID \
--project-name "$PROJECT_NAME" \
--workflow-name "$WORKFLOW" \
--job-name "$JOB_NAME_BASE|GitHubCommit:${COMMIT_HASH:0:6}|PR-NUMBER:$PR_NUMBER" \
--instance-type m4.xlarge \
--params-file "$PARAMS_FILE" \
--wait-completion
interactive_session_list:
runs-on: ubuntu-latest
strategy:
matrix:
python-version: [ "3.9" ]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: pip
cache-dependency-path: setup.py
- name: Install dependencies
run: |
pip install -e .
- name: Run tests
env:
CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN_DEV }}
CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID_DEV }}
CLOUDOS_URL: "https://dev.sdlc.lifebit.ai"
run: |
echo q |cloudos interactive-session list --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID
interactive_session_create:
outputs:
session_id: ${{ steps.get-session-id.outputs.session_id }}
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.9"]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: pip
cache-dependency-path: setup.py
- name: Install dependencies
run: |
pip install -e .
- name: Create interactive session
id: get-session-id
env:
CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN_DEV }}
CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID_DEV }}
CLOUDOS_URL: "https://dev.sdlc.lifebit.ai"
PROJECT_NAME: "cloudos-cli-tests"
SESSION_NAME: "ci_test_cli"
SESSION_TYPE: "jupyter"
SHUTDOWN_IN: "10m"
run: |
cloudos interactive-session create --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --project-name "$PROJECT_NAME" --name $SESSION_NAME --session-type $SESSION_TYPE --shutdown-in $SHUTDOWN_IN 2>&1 | tee out.txt
SESSION_ID=$(grep -oP '(?<=/view/)[a-f0-9]{24}' out.txt | head -1)
echo "session_id=$SESSION_ID" >> $GITHUB_OUTPUT
interactive_session_status:
needs: interactive_session_create
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.9"]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: pip
cache-dependency-path: setup.py
- name: Install dependencies
run: |
pip install -e .
- name: Get session status
env:
CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN_DEV }}
CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID_DEV }}
CLOUDOS_URL: "https://dev.sdlc.lifebit.ai"
SESSION_ID: ${{ needs.interactive_session_create.outputs.session_id }}
run: |
cloudos interactive-session status --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --session-id $SESSION_ID
interactive_session_pause:
needs: interactive_session_create
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.9"]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: pip
cache-dependency-path: setup.py
- name: Install dependencies
run: |
pip install -e .
- name: Pause interactive session
env:
CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN_DEV }}
CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID_DEV }}
CLOUDOS_URL: "https://dev.sdlc.lifebit.ai"
SESSION_ID: ${{ needs.interactive_session_create.outputs.session_id }}
run: |
cloudos interactive-session pause --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --session-id $SESSION_ID --yes
interactive_session_resume:
needs: [interactive_session_pause, interactive_session_create]
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.9"]
steps:
- uses: actions/checkout@v3
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
cache: pip
cache-dependency-path: setup.py
- name: Install dependencies
run: |
pip install -e .
- name: Resume interactive session
env:
CLOUDOS_TOKEN: ${{ secrets.CLOUDOS_TOKEN_DEV }}
CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID_DEV }}
CLOUDOS_URL: "https://dev.sdlc.lifebit.ai"
SESSION_ID: ${{ needs.interactive_session_create.outputs.session_id }}
run: |
cloudos interactive-session resume --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --session-id $SESSION_ID --shutdown-in 1m