Added tests for the cwl.output.json feature

.github/workflows/cwltool.yml

@@ -21,10 +21,10 @@ jobs:
 
       - uses: actions/setup-python@v6
         with:
-          python-version: '3.9.x'
+          python-version: '3.10.x'
 
       - name: Setup prerequirements
-        run: pip install "cwltest>=2.3" pytest-xdist "cwltool>=3.1.20240909164951"
+        run: pip install "cwltest>=2.3" pytest-xdist git+https://github.com/common-workflow-language/cwltool.git@refs/pull/2177/head
 
       - name: Copy in cwltool-specific configuration for the cwltest pytest plugin
         run: cp "$(python -c 'from cwltool.tests.util import get_data; print(get_data("tests/cwl-conformance/cwltool-conftest.py"))')" conftest.py

CommandLineTool.yml

@@ -51,6 +51,17 @@ $graph:
 
       ## Changelog for v1.3.0-dev1
 
+      * [Output Binding](#Output_binding): Any output present in the
+        "cwl.output.json" file but not declared in the `outputs` section must
+        be discarded from the output object. Previous versions of CWL did not
+        define if undeclared outputs in "cwl.output.json" should be present
+        in the output object.
+
+      ### Updated Conformance Tests for v1.3.0
+
+      * Reduced the usage of `ShellCommandRequirement` in the conformance tests,
+        making several more of the `required`.
+
       See also the [CWL Workflow Description, v1.3.0-dev1 changelog](Workflow.html#Changelog).
       For other changes since CWL v1.0, see the
       [CWL Command Line Tool Description, v1.1 changelog](https://www.commonwl.org/v1.1/CommandLineTool.html#Changelog)

conformance_tests.yaml

@@ -109,7 +109,7 @@
       checksum: sha1$f1d2d2f924e986ac86fdf7b36c94bcdf32beec15
       size: 4
       location: error.txt
-  tags: [ shell_command, command_line_tool ]
+  tags: [ required, command_line_tool ]
 
 - job: tests/empty.json
   tool: tests/stderr-shortcut.cwl
@@ -121,7 +121,7 @@
       checksum: sha1$f1d2d2f924e986ac86fdf7b36c94bcdf32beec15
       size: 4
       location: Any
-  tags: [ shell_command, command_line_tool ]
+  tags: [ required, command_line_tool ]
 
 - output:
     output_file:
@@ -133,7 +133,7 @@
   tool: tests/stderr-mediumcut.cwl
   id: stderr_redirect_mediumcut
   doc: Test command line with stderr redirection, named brief syntax
-  tags: [ shell_command, command_line_tool ]
+  tags: [ required, command_line_tool ]
 
 - job: tests/cat-job.json
   output:
@@ -951,7 +951,7 @@
   doc: >-
     Test support for reading cwl.output.json when running in a Docker container
     and just 'path' is provided.
-  tags: [ shell_command, command_line_tool ]
+  tags: [ required, command_line_tool ]
 
 - job: tests/empty.json
   output: {
@@ -1110,7 +1110,7 @@
   tool: tests/dir.cwl
   id: directory_input_param_ref
   doc: Test directory input with parameter reference
-  tags: [ shell_command, command_line_tool ]
+  tags: [ required, command_line_tool ]
 
 - job: tests/dir-job.yml
   output:
@@ -1123,7 +1123,7 @@
   tool: tests/dir2.cwl
   id: directory_input_docker
   doc: Test directory input in Docker
-  tags: [ command_line_tool, shell_command ]
+  tags: [ required, command_line_tool ]
 
 - job: tests/dir3-job.yml
   output:
@@ -1161,7 +1161,7 @@
   tool: tests/dir4.cwl
   id: directory_secondaryfiles
   doc: Test directories in secondaryFiles
-  tags: [ shell_command, command_line_tool ]
+  tags: [ required, command_line_tool ]
 
 - job: tests/dir-job.yml
   output: {
@@ -1175,7 +1175,7 @@
   tool: tests/dir5.cwl
   id: dynamic_initial_workdir
   doc: Test dynamic initial work dir
-  tags: [ shell_command, initial_work_dir, command_line_tool ]
+  tags: [ initial_work_dir, command_line_tool ]
 
 - job: tests/stagefile-job.yml
   output: {
@@ -1257,14 +1257,14 @@
   tool: tests/envvar.cwl
   id: env_home_tmpdir
   doc: Test $HOME and $TMPDIR are set correctly
-  tags: [ shell_command, command_line_tool ]
+  tags: [ required, command_line_tool ]
 
 - job: tests/empty.json
   output: {}
   tool: tests/envvar2.cwl
   id: env_home_tmpdir_docker
   doc: Test $HOME and $TMPDIR are set correctly in Docker
-  tags: [ shell_command, command_line_tool ]
+  tags: [ required, command_line_tool ]
 
 - job: tests/empty.json
   output:
@@ -1316,14 +1316,14 @@
   tool: tests/docker-array-secondaryfiles.cwl
   id: filesarray_secondaryfiles
   doc: Test required, optional and null secondaryFiles on array of files.
-  tags: [ docker, inline_javascript, shell_command, command_line_tool ]
+  tags: [ docker, inline_javascript, command_line_tool ]
 
 - job: tests/docker-array-secondaryfiles-job2.json
   should_fail: true
   tool: tests/docker-array-secondaryfiles.cwl
   id: filesarray_secondaryfiles2
   doc: Test required, optional and null secondaryFiles on array of files.
-  tags: [ docker, inline_javascript, shell_command, command_line_tool ]
+  tags: [ docker, inline_javascript, command_line_tool ]
 
 - job: tests/dir7.yml
   output: {
@@ -1477,7 +1477,7 @@
   tool: tests/recursive-input-directory.cwl
   id: input_dir_recurs_copy_writable
   doc: Test if a writable input directory is recursively copied and writable
-  tags: [ initial_work_dir, shell_command, command_line_tool ]
+  tags: [ initial_work_dir, command_line_tool ]
 
 - output:
     out: "t\n"
@@ -1819,7 +1819,7 @@
   tool: tests/envvar3.cwl
   id: env_home_tmpdir_docker_no_return_code
   doc: Test $HOME and $TMPDIR are set correctly in Docker without using return code
-  tags: [ shell_command, command_line_tool ]
+  tags: [ required, command_line_tool ]
 
 - job: tests/scatter-valuefrom-job1.json
   output:
@@ -1877,7 +1877,7 @@
   tool: tests/dir4.cwl
   id: job_input_secondary_subdirs
   doc: Test specifying secondaryFiles in subdirectories of the job input document.
-  tags: [ shell_command, command_line_tool ]
+  tags: [ required, command_line_tool ]
 
 - job: tests/dir4-subdir-2-job.yml
   output: {
@@ -1891,7 +1891,7 @@
   tool: tests/dir4.cwl
   id: job_input_subdir_primary_and_secondary_subdirs
   doc: Test specifying secondaryFiles in same subdirectory of the job input as the primary input file.
-  tags: [ shell_command, command_line_tool ]
+  tags: [ required, command_line_tool ]
 
 - job: tests/count-lines3-job.json
   output:
@@ -2773,7 +2773,7 @@
         "location": "lsout",
         "size": 32
       }
-  tags: [ resource, command_line_tool, initial_work_dir ]
+  tags: [ command_line_tool, initial_work_dir ]
   id: initial_work_dir_for_null_and_arrays
 
 - job: tests/stage-array-dirs-job.yml
@@ -2869,7 +2869,7 @@
       "baesname": "symlink.txt"
       "checksum": "sha1$cd28ec34f3f9425aca544b6332453708e8aaa82a"
   should_fail: true
-  tags: [ command_line_tool ]
+  tags: [ required, command_line_tool ]
   id: illegal_symlink
 
 - job: tests/empty.json
@@ -2881,7 +2881,7 @@
       "size": 27
       "basename": "symlink.txt"
       "checksum": "sha1$cd28ec34f3f9425aca544b6332453708e8aaa82a"
-  tags: [ command_line_tool ]
+  tags: [ required, command_line_tool ]
   id: legal_symlink
 
 - job: tests/empty.json
@@ -3399,7 +3399,7 @@
   }
   tool: tests/stdout_chained_commands.cwl
   doc: |
-    Test that chaining two echo calls causes the workflow tool to emit the output to stdout.
+    Test that chaining two echo calls causes the tool to emit the output to stdout.
     This is to confirm that the workflow tool will **not** create an expression such as
     `echo a && echo b > out.txt`, but instead will produce the correct `echo a && echo b`,
     and capture the output correctly.
@@ -3618,4 +3618,28 @@
       an_array_of_trues: [ true, true, true ]
       an_int: 42
 
+- id: cwl_output_json_new_output
+  tool: tests/test-cwl-out5.cwl
+  job: tests/empty.json
+  doc: >-
+    Confirm that undeclared outputs from cwl.output.json are discarded
+  tags: [ required, command_line_tool ]
+  output: {}
+
+- id: cwl_output_json_wrong_type
+  tool: tests/test-cwl-out6.cwl
+  job: tests/empty.json
+  doc: Confirm that outputs from cwl.output.json are type-checked
+  tags: [ required, command_line_tool ]
+  should_fail: true
+
+- id: cwl_output_json_missing_required_output
+  tool: tests/test-cwl-out7.cwl
+  job: tests/empty.json
+  doc: >-
+    Confirm that non-optional outputs not present in cwl.output.json
+    are still checked
+  tags: [ required, command_line_tool ]
+  should_fail: true
+
 - $import: tests/loop/test-index.yaml

invocation.md

@@ -159,7 +159,9 @@ The exit code of the process is available to expressions in
 If the output directory contains a file named "cwl.output.json", that
 file must be loaded and used as the output object.  In this case, the
 output object should still be type-checked against the `outputs`
-section, but `outputBinding` is ignored.
+section, but `outputBinding` is ignored. Any output is present in the
+"cwl.output.json" file but not declared in the `outputs` section must
+be discarded from the output object.
 
 For Files and Directories, if the value of `path` is a relative path
 pattern (does not begin with a slash '/') then it is resolved relative

tests/dir.cwl

@@ -1,17 +1,15 @@
 class: CommandLineTool
 cwlVersion: v1.3.0-dev1
-requirements:
-  - class: ShellCommandRequirement
 inputs:
   indir: Directory
 outputs:
   outlist:
     type: File
     outputBinding:
       glob: output.txt
-arguments: ["cd", "$(inputs.indir.path)",
-  {shellQuote: false, valueFrom: "&&"},
-  "find", ".",
-  {shellQuote: false, valueFrom: "|"},
-  "sort"]
-stdout: output.txt
+arguments:
+  - sh
+  - -c
+  - |
+    cd $(inputs.indir.path) && find . | sort
+stdout: output.txt

tests/dir2.cwl

@@ -1,7 +1,5 @@
 class: CommandLineTool
 cwlVersion: v1.3.0-dev1
-requirements:
-  ShellCommandRequirement: {}
 hints:
   DockerRequirement:
     dockerPull: docker.io/debian:stable-slim
@@ -12,9 +10,9 @@ outputs:
     type: File
     outputBinding:
       glob: output.txt
-arguments: ["cd", "$(inputs.indir.path)",
-  {shellQuote: false, valueFrom: "&&"},
-  "find", ".",
-  {shellQuote: false, valueFrom: "|"},
-  "sort"]
+arguments:
+  - sh
+  - -c
+  - |
+    cd $(inputs.indir.path) && find . | sort
 stdout: output.txt

tests/dir4.cwl

@@ -1,17 +1,15 @@
 class: CommandLineTool
 cwlVersion: v1.3.0-dev1
-requirements:
-  - class: ShellCommandRequirement
 inputs:
   inf: File
 outputs:
   outlist:
     type: File
     outputBinding:
       glob: output.txt
-arguments: ["cd", "$(inputs.inf.dirname)/xtestdir",
-  {shellQuote: false, valueFrom: "&&"},
-  "find", ".",
-  {shellQuote: false, valueFrom: "|"},
-  "sort"]
+arguments:
+  - sh
+  - -c
+  - |
+    cd $(inputs.inf.dirname)/xtestdir && find . | sort
 stdout: output.txt

tests/dir5.cwl

@@ -1,7 +1,6 @@
 class: CommandLineTool
 cwlVersion: v1.3.0-dev1
 requirements:
-  - class: ShellCommandRequirement
   - class: InitialWorkDirRequirement
     listing: $(inputs.indir.listing)
 inputs:
@@ -13,7 +12,9 @@ outputs:
     type: File
     outputBinding:
       glob: output.txt
-arguments: ["find", "-L", ".", "!", "-path", "*.txt",
-  {shellQuote: false, valueFrom: "|"},
-  "sort"]
+arguments:
+  - sh
+  - -c
+  - |
+    find -L . ! -path "*.txt" | sort
 stdout: output.txt

tests/dir6.cwl

@@ -18,4 +18,4 @@ arguments: [
   "find", ".",
   {shellQuote: false, valueFrom: "|"},
   "sort"]
-stdout: output.txt
+stdout: output.txt

tests/docker-array-secondaryfiles.cwl

@@ -6,7 +6,6 @@ requirements:
   - class: DockerRequirement
     dockerPull: docker.io/debian:stable-slim
   - class: InlineJavascriptRequirement
-  - class: ShellCommandRequirement
 
 class: CommandLineTool
 
@@ -37,14 +36,15 @@ outputs:
       - pattern: "${ return null }"
 
 arguments:
-  - valueFrom: ${
+  - sh
+  - -c
+  - |
+    ${
         var fai_list = "";
         for (var i = 0; i < inputs.fasta_path.length; i ++) {
           fai_list += " cat " + inputs.fasta_path[i].path +".fai" + " >> fai.list && "
         }
         return fai_list.slice(0,-3)
-        }
-    position: 1
-    shellQuote: false
+     }
 
 baseCommand: []

tests/envvar.cwl

@@ -2,10 +2,8 @@ class: CommandLineTool
 cwlVersion: v1.3.0-dev1
 inputs: []
 outputs: []
-requirements:
-  ShellCommandRequirement: {}
-arguments: [
-  echo, {valueFrom: '"HOME=$HOME"', shellQuote: false}, {valueFrom: '"TMPDIR=$TMPDIR"', shellQuote: false},
-  {valueFrom: '&&', shellQuote: false},
-  test, {valueFrom: '"$HOME"', shellQuote: false}, "=", $(runtime.outdir),
-  "-a", {valueFrom: '"$TMPDIR"', shellQuote: false}, "=", $(runtime.tmpdir)]
+arguments:
+  - sh
+  - -c
+  - |
+    echo "HOME=$HOME" "TMPDIR=$TMPDIR" && test "$HOME"=$(runtime.outdir) -a "$TMPDIR"=$(runtime.tmpdir)