diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7b230a81..1d7292f4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -244,7 +244,7 @@ jobs: INSTANCE_TYPE: "m4.xlarge" run: | JOB_NAME="$JOB_NAME_BASE""|GitHubCommit:""${COMMIT_HASH:0:6}""|PR-NUMBER:""$PR_NUMBER" - cloudos job run --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --project-name "$PROJECT_NAME" --workflow-name "$WORKFLOW" --job-config $JOB_CONFIG --job-name "$JOB_NAME" --wait-completion --resumable --instance-type $INSTANCE_TYPE 2>&1 | tee out.txt + cloudos job run --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --project-name "$PROJECT_NAME" --workflow-name "$WORKFLOW" --nextflow-version 22.10.8 --job-config $JOB_CONFIG --job-name "$JOB_NAME" --wait-completion --resumable --instance-type $INSTANCE_TYPE 2>&1 | tee out.txt JOB_ID=$(grep -e "Your assigned job id is:" out.txt | rev | cut -f1 -d " " | rev) echo "job_id=$JOB_ID" >> $GITHUB_OUTPUT cloudos job status --cloudos-url $CLOUDOS_URL --workspace-id $CLOUDOS_WORKSPACE_ID --apikey $CLOUDOS_TOKEN --job-id $JOB_ID @@ -800,7 +800,7 @@ jobs: run: | # Step 1: Run job (without --wait-completion so it doesn't wait until completion) JOB_NAME="$JOB_NAME_BASE""|GitHubCommit:""${COMMIT_HASH:0:6}""|PR-NUMBER:""$PR_NUMBER" - cloudos job run --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --project-name "$PROJECT_NAME" --workflow-name "$WORKFLOW" --job-config $JOB_CONFIG --job-name "$JOB_NAME" --instance-type $INSTANCE_TYPE 2>&1 | tee out.txt + cloudos job run --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --project-name "$PROJECT_NAME" --workflow-name "$WORKFLOW" --nextflow-version 22.10.8 --job-config $JOB_CONFIG --job-name "$JOB_NAME" --instance-type $INSTANCE_TYPE 2>&1 | tee out.txt JOB_ID=$(grep -e "Your assigned job id is:" out.txt | rev | cut -f1 -d " " | rev) echo "Job ID: $JOB_ID" @@ -893,6 +893,7 @@ jobs: --workspace-id $CLOUDOS_WORKSPACE_ID \ --project-name "$PROJECT_NAME" \ --workflow-name "$WORKFLOW" \ + --nextflow-version 22.10.8 \ --job-name "$JOB_NAME_BASE|GitHubCommit:${COMMIT_HASH:0:6}|PR-NUMBER:$PR_NUMBER" \ --instance-type m4.xlarge \ --params-file "$PARAMS_FILE" \ diff --git a/.github/workflows/ci_az.yml b/.github/workflows/ci_az.yml index 2ccc8063..b8e5b893 100644 --- a/.github/workflows/ci_az.yml +++ b/.github/workflows/ci_az.yml @@ -36,7 +36,7 @@ jobs: CLOUDOS_WORKSPACE_ID: ${{ secrets.CLOUDOS_WORKSPACE_ID_AZURE }} CLOUDOS_URL: "https://dev.sdlc.lifebit.ai" run: | - cloudos job list --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID + echo q | cloudos job list --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID job_list_filtering_az: needs: job_run_and_status_az runs-on: ubuntu-latest @@ -62,15 +62,15 @@ jobs: run: | JOB_ID="${{ needs.job_run_and_status_az.outputs.job_id }}" # Test filtering by status, project and workflow name - cloudos job list --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --filter-status completed --filter-project cloudos-cli-tests --filter-workflow GH-rnatoy --last --last-n-jobs 10 + echo q | cloudos job list --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --filter-status completed --filter-project cloudos-cli-tests --filter-workflow GH-rnatoy --last --last-n-jobs 10 # Test filtering job id - cloudos job list --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --filter-job-id $JOB_ID + echo q | cloudos job list --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --filter-job-id $JOB_ID # Test filtering job name - cloudos job list --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --filter-job-name "cloudos-cli-CI-test" --last-n-jobs 10 + echo q | cloudos job list --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --filter-job-name "cloudos-cli-CI-test" --last-n-jobs 10 # Test filtering by only mine - cloudos job list --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --filter-only-mine --last-n-jobs 10 + echo q | cloudos job list --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --filter-only-mine --last-n-jobs 10 # Test filtering by queue - #cloudos job list --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --filter-queue "cost_saving_standard_nextflow" --last-n-jobs 10 + #echo q | cloudos job list --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --filter-queue "cost_saving_standard_nextflow" --last-n-jobs 10 job_details_az: needs: job_run_and_status_az runs-on: ubuntu-latest diff --git a/.github/workflows/ci_dev.yml b/.github/workflows/ci_dev.yml index fb576ae4..6f8d9595 100644 --- a/.github/workflows/ci_dev.yml +++ b/.github/workflows/ci_dev.yml @@ -225,7 +225,7 @@ jobs: INSTANCE_TYPE: "m4.xlarge" run: | JOB_NAME="$JOB_NAME_BASE""|GitHubCommit:""${COMMIT_HASH:0:6}""|PR-NUMBER:""$PR_NUMBER" - cloudos job run --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --project-name "$PROJECT_NAME" --workflow-name "$WORKFLOW" --job-config $JOB_CONFIG --job-name "$JOB_NAME" --wait-completion --resumable --instance-type $INSTANCE_TYPE 2>&1 | tee out.txt + cloudos job run --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --project-name "$PROJECT_NAME" --workflow-name "$WORKFLOW" --nextflow-version 22.10.8 --job-config $JOB_CONFIG --job-name "$JOB_NAME" --wait-completion --resumable --instance-type $INSTANCE_TYPE 2>&1 | tee out.txt JOB_ID=$(grep -e "Your assigned job id is:" out.txt | rev | cut -f1 -d " " | rev) echo "job_id=$JOB_ID" >> $GITHUB_OUTPUT cloudos job status --cloudos-url $CLOUDOS_URL --workspace-id $CLOUDOS_WORKSPACE_ID --apikey $CLOUDOS_TOKEN --job-id $JOB_ID @@ -782,7 +782,7 @@ jobs: run: | # Step 1: Run job (without --wait-completion so it doesn't wait until completion) JOB_NAME="$JOB_NAME_BASE""|GitHubCommit:""${COMMIT_HASH:0:6}""|PR-NUMBER:""$PR_NUMBER" - cloudos job run --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --project-name "$PROJECT_NAME" --workflow-name "$WORKFLOW" --job-config $JOB_CONFIG --job-name "$JOB_NAME" --instance-type $INSTANCE_TYPE 2>&1 | tee out.txt + cloudos job run --cloudos-url $CLOUDOS_URL --apikey $CLOUDOS_TOKEN --workspace-id $CLOUDOS_WORKSPACE_ID --project-name "$PROJECT_NAME" --workflow-name "$WORKFLOW" --nextflow-version 22.10.8 --job-config $JOB_CONFIG --job-name "$JOB_NAME" --instance-type $INSTANCE_TYPE 2>&1 | tee out.txt JOB_ID=$(grep -e "Your assigned job id is:" out.txt | rev | cut -f1 -d " " | rev) echo "Job ID: $JOB_ID" @@ -899,6 +899,7 @@ jobs: --workspace-id $CLOUDOS_WORKSPACE_ID \ --project-name "$PROJECT_NAME" \ --workflow-name "$WORKFLOW" \ + --nextflow-version 22.10.8 \ --job-name "$JOB_NAME_BASE|GitHubCommit:${COMMIT_HASH:0:6}|PR-NUMBER:$PR_NUMBER" \ --instance-type m4.xlarge \ --params-file "$PARAMS_FILE" \ diff --git a/CHANGELOG.md b/CHANGELOG.md index 3da37d7c..bf5bd29c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ ## lifebit-ai/cloudos-cli: changelog +## v2.88.0 (2026-04-03) + +### Feat + +- Adds support for nextflow versions '25.04.8' and '25.10.4' +- Updates nextflow versions defaults for user imported pipelines and lifebit-owned pipelines + ## v2.86.0 (2026-03-23) ### Feat diff --git a/README.md b/README.md index ec56aac1..83baaf73 100644 --- a/README.md +++ b/README.md @@ -646,6 +646,29 @@ When there are duplicate `--workflow-name` in the platform, you can add the `--l _For example, the pipeline `lifebit-process` was imported on May 23 2025 and again on May 30 2025; with the `--last` flag, it will use the import of May 30, 2025._ +**Nextflow Version Support** + +CloudOS CLI supports multiple Nextflow versions, with intelligent defaults based on your workflow type and execution platform. + +**Default Version Behavior:** + +The default Nextflow version is automatically selected based on your workflow type and execution platform: + +- **Azure Platform**: Always uses `22.11.1-edge` regardless of workflow type +- **Platform Workflows** (AWS/HPC): Uses `22.10.8` (Lifebit provided workflows and modules) +- **User-Imported Workflows** (AWS/HPC): Uses `24.04.4` + +**Specifying a Custom Version:** + +You can override the default by using the `--nextflow-version` parameter: + +```bash +cloudos job run --profile my_profile --workflow-name rnatoy --job-config cloudos_cli/examples/rnatoy.config --nextflow-version 25.04.8 --resumable +``` + +> [!NOTE] +> **Azure Platform Behavior**: If you specify a Nextflow version not supported by Azure (anything other than 22.11.1-edge), CloudOS CLI will display a warning and automatically use 22.11.1-edge instead. Your job will proceed without error. + **AWS Executor Support** CloudOS supports [AWS batch](https://www.nextflow.io/docs/latest/executor.html?highlight=executors#aws-batch) executor by default. @@ -664,6 +687,7 @@ cloudos job run --profile my_profile --workflow-name rnatoy --job-config cloudos CloudOS can also be configured to use Microsoft Azure compute platforms. If your CloudOS is configured to use Azure, you will need to take into consideration the following: - When sending jobs to CloudOS using `cloudos job run` command, please use the option `--execution-platform azure` +- Azure only supports Nextflow version `22.11.1-edge`. If you specify a different version, CloudOS CLI will display a warning and automatically use `22.11.1-edge` instead - Due to the lack of AWS batch queues in Azure, `cloudos queue list` command is not working Other than that, `cloudos-cli` will work very similarly. For instance, this is a typical send job command: diff --git a/cloudos_cli/_version.py b/cloudos_cli/_version.py index c27892cb..cc991665 100644 --- a/cloudos_cli/_version.py +++ b/cloudos_cli/_version.py @@ -1 +1 @@ -__version__ = '2.86.0' +__version__ = '2.88.0' diff --git a/cloudos_cli/constants.py b/cloudos_cli/constants.py index e3a53a80..43ab1d8e 100644 --- a/cloudos_cli/constants.py +++ b/cloudos_cli/constants.py @@ -6,10 +6,10 @@ JOB_ABORTED = 'aborted' # Nextflow version constants -AWS_NEXTFLOW_VERSIONS = ['22.10.8', '24.04.4'] +AWS_NEXTFLOW_VERSIONS = ['22.10.8', '24.04.4', '25.04.8', '25.10.4'] AZURE_NEXTFLOW_VERSIONS = ['22.11.1-edge'] HPC_NEXTFLOW_VERSIONS = ['22.10.8'] -AWS_NEXTFLOW_LATEST = '24.04.4' +AWS_NEXTFLOW_LATEST = '25.10.4' AZURE_NEXTFLOW_LATEST = '22.11.1-edge' HPC_NEXTFLOW_LATEST = '22.10.8' diff --git a/cloudos_cli/jobs/cli.py b/cloudos_cli/jobs/cli.py index 53bac81b..7eb7716f 100644 --- a/cloudos_cli/jobs/cli.py +++ b/cloudos_cli/jobs/cli.py @@ -83,9 +83,9 @@ def job(): 'to use with your job.')) @click.option('--nextflow-version', help=('Nextflow version to use when executing the workflow in CloudOS. ' + - 'Default=22.10.8.'), - type=click.Choice(['22.10.8', '24.04.4', '22.11.1-edge', 'latest']), - default='22.10.8') + 'Defaults to 22.10.8 for Platform Workflows or 24.04.4 for user-imported Workflows.'), + type=click.Choice(['22.10.8', '24.04.4', '25.04.8', '25.10.4', '22.11.1-edge', 'latest']), + default=None) @click.option('--git-commit', help=('The git commit hash to run for ' + 'the selected pipeline. ' + @@ -293,6 +293,26 @@ def run(ctx, cl = Cloudos(cloudos_url, apikey, cromwell_token) workflow_type = cl.detect_workflow(workflow_name, workspace_id, verify_ssl, last) is_module = cl.is_module(workflow_name, workspace_id, verify_ssl, last) + + # Set dynamic default for Nextflow version based on execution platform and workflow type + if nextflow_version is None: + if execution_platform == 'azure': + nextflow_version = '22.11.1-edge' # Azure has fixed Nextflow version + if verbose: + print('\t...Using default Nextflow version 22.11.1-edge for Azure') + elif execution_platform == 'hpc': + nextflow_version = '22.10.8' # HPC has fixed Nextflow version + if verbose: + print('\t...Using default Nextflow version 22.10.8 for HPC') + elif is_module: + nextflow_version = '22.10.8' # Lifebit Platform workflows (AWS) + if verbose: + print('\t...Using default Nextflow version 22.10.8 for Platform Workflow') + else: + nextflow_version = '24.04.4' # User-imported workflows (AWS only) + if verbose: + print('\t...Using default Nextflow version 24.04.4 for user-imported workflow') + if execution_platform == 'hpc' and workflow_type == 'wdl': raise ValueError(f'The workflow {workflow_name} is a WDL workflow. ' + 'WDL is not supported on HPC execution platform.') @@ -343,11 +363,19 @@ def run(ctx, f'Platform Workflow "{workflow_name}". Platform Workflows ' + 'use their own predetermined queues.') job_queue_id = None - if nextflow_version != '22.10.8': - print(f'The selected worflow \'{workflow_name}\' ' + - 'is a CloudOS module. CloudOS modules only work with ' + - 'Nextflow version 22.10.8. Switching to use 22.10.8') - nextflow_version = '22.10.8' + # Platform workflows have fixed Nextflow versions per execution platform + if execution_platform == 'azure': + if nextflow_version != '22.11.1-edge': + print(f'The selected workflow \'{workflow_name}\' ' + + 'is a CloudOS Platform Workflow on Azure. Platform Workflows on Azure only work with ' + + 'Nextflow version 22.11.1-edge. Switching to use 22.11.1-edge') + nextflow_version = '22.11.1-edge' + else: + if nextflow_version != '22.10.8': + print(f'The selected workflow \'{workflow_name}\' ' + + 'is a CloudOS Platform Workflow. Platform Workflows only work with ' + + 'Nextflow version 22.10.8. Switching to use 22.10.8') + nextflow_version = '22.10.8' if execution_platform == 'azure': print(f'The selected worflow \'{workflow_name}\' ' + 'is a CloudOS module. For these workflows, worker nodes ' + @@ -389,19 +417,28 @@ def run(ctx, f'latest version available on CloudOS: {nextflow_version}.') if execution_platform == 'aws': if nextflow_version not in AWS_NEXTFLOW_VERSIONS: - print('For execution platform \'aws\', the workflow will use the default ' + - '\'22.10.8\' version on CloudOS.') - nextflow_version = '22.10.8' + available_versions = ', '.join(AWS_NEXTFLOW_VERSIONS) + raise click.BadParameter( + f'Unsupported Nextflow version \'{nextflow_version}\' for AWS execution platform. ' + f'Supported versions are: {available_versions}.' + ) if execution_platform == 'azure': if nextflow_version not in AZURE_NEXTFLOW_VERSIONS: - print('For execution platform \'azure\', the workflow will use the \'22.11.1-edge\' ' + - 'version on CloudOS.') - nextflow_version = '22.11.1-edge' + available_versions = ', '.join(AZURE_NEXTFLOW_VERSIONS) + click.secho( + f'Warning: Nextflow version \'{nextflow_version}\' is not supported for Azure execution platform. ' + f'Azure only supports: {available_versions}. Switching to use {AZURE_NEXTFLOW_LATEST}.', + fg='yellow', bold=True + ) + nextflow_version = AZURE_NEXTFLOW_LATEST if execution_platform == 'hpc': if nextflow_version not in HPC_NEXTFLOW_VERSIONS: - print('For execution platform \'hpc\', the workflow will use the \'22.10.8\' version on CloudOS.') - nextflow_version = '22.10.8' - if nextflow_version != '22.10.8' and nextflow_version != '22.11.1-edge': + available_versions = ', '.join(HPC_NEXTFLOW_VERSIONS) + raise click.BadParameter( + f'Unsupported Nextflow version \'{nextflow_version}\' for HPC execution platform. ' + f'HPC only supports: {available_versions}.' + ) + if nextflow_version not in ['22.10.8', '22.11.1-edge']: click.secho(f'You have specified Nextflow version {nextflow_version}. This version requires the pipeline ' + 'to be written in DSL2 and does not support DSL1.', fg='yellow', bold=True) print('\nExecuting run...') @@ -1704,7 +1741,7 @@ def archive_unarchive_jobs(ctx, @click.option('--nextflow-version', help=('Nextflow version to use when executing the workflow in CloudOS. ' + 'Default=22.10.8.'), - type=click.Choice(['22.10.8', '24.04.4', '22.11.1-edge', 'latest'])) + type=click.Choice(['22.10.8', '24.04.4', '25.04.8', '25.10.4', '22.11.1-edge', 'latest'])) @click.option('--git-branch', help=('The branch to run for the selected pipeline. ' + 'If not specified it defaults to the last commit ' +