Reinitialization async

.github/workflows/test.yml

@@ -46,7 +46,7 @@ jobs:
 
       - name: Setup Rust
         uses: dtolnay/rust-toolchain@stable
-        
+
       - name: Ensure cargo in PATH (Windows)
         if: matrix.os == 'windows-latest'
         run: echo "$HOME/.cargo/bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append

api/openapi.yaml

@@ -4930,6 +4930,15 @@ paths:
           default: true
           type: boolean
         style: form
+      - description: Run initialization asynchronously and return a task ID.
+        explode: true
+        in: query
+        name: async
+        required: false
+        schema:
+          default: false
+          type: boolean
+        style: form
       requestBody:
         content:
           application/json:
@@ -4943,6 +4952,18 @@ paths:
               schema:
                 type: object
           description: Successful response
+        "202":
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/task_model"
+          description: Accepted - initialization task created
+        "409":
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/default_error_response"
+          description: Conflict - initialization already in progress
         "403":
           content:
             application/json:
@@ -4965,6 +4986,41 @@ paths:
       summary: Initialize job relationships based on file and user_data relationships.
       tags: []
       x-codegen-request-body-name: body
+  /tasks/{id}:
+    get:
+      description: Get the status of an asynchronous task.
+      operationId: get_task
+      parameters:
+      - description: Task ID
+        explode: false
+        in: path
+        name: id
+        required: true
+        schema:
+          type: integer
+        style: simple
+      responses:
+        "200":
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/task_model"
+          description: Successful response
+        "404":
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/not_found_error_response"
+          description: Not found error response
+        "500":
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/default_error_response"
+          description: Default error response
+      security: []
+      summary: Get the status of an asynchronous task.
+      tags: []
   /workflows/{id}/is_complete:
     get:
       description: Return true if all jobs in the workflow are complete.
@@ -8659,4 +8715,44 @@ components:
         - total_count
         - has_more
       type: object
+    task_status:
+      description: Status of an asynchronous task.
+      enum:
+        - queued
+        - running
+        - succeeded
+        - failed
+      type: string
+    task_model:
+      description: Asynchronous task status record.
+      properties:
+        id:
+          type: integer
+        workflow_id:
+          type: integer
+        operation:
+          type: string
+        status:
+          $ref: "#/components/schemas/task_status"
+        created_at_ms:
+          type: integer
+        started_at_ms:
+          type:
+            - integer
+            - "null"
+        finished_at_ms:
+          type:
+            - integer
+            - "null"
+        error:
+          type:
+            - string
+            - "null"
+      required:
+        - id
+        - workflow_id
+        - operation
+        - status
+        - created_at_ms
+      type: object
 x-original-swagger-version: "2.0"

docs/src/core/concepts/job-runners.md

@@ -78,13 +78,13 @@ resource_requirements:
     memory: "200g"
     runtime: "PT4H"
     num_nodes: 1
-    
+
 parameters:
   i: "1:100"
 jobs:
   - name: "work_{i}"
     command: bash my_script.sh {i}
-    resource_requirements: work_resources  
+    resource_requirements: work_resources
     use_parameters:
     - i
 ```

examples/json/data_pipeline_parameterized.json5

@@ -79,7 +79,7 @@
       }
     }
   ],
-  
+
   // File definitions
   "files": [
     {
@@ -105,7 +105,7 @@
       }
     }
   ],
-  
+
   // Resource requirements
   "resource_requirements": [
     {

examples/json/diamond_workflow.json5

@@ -23,7 +23,7 @@
       "command": "bash tests/scripts/postprocess.sh -i ${files.input.f4} -i ${files.input.f5} -o ${files.output.f6}"
     }
   ],
-  
+
   // File definitions
   "files": [
     {

examples/json/hundred_jobs_parameterized.json5

@@ -15,7 +15,7 @@
       }
     }
   ],
-  
+
   // Resource requirements
   "resource_requirements": [
     {

examples/json/hundred_jobs_workflow.json5

@@ -507,7 +507,7 @@
       "resource_requirements": "minimal"
     }
   ],
-  
+
   // Resource requirements
   "resource_requirements": [
     {

examples/json/hyperparameter_sweep.json5

@@ -61,7 +61,7 @@
       }
     }
   ],
-  
+
   // File definitions
   "files": [
     {
@@ -91,7 +91,7 @@
       }
     }
   ],
-  
+
   // Resource requirements
   "resource_requirements": [
     {

examples/json/multi_stage_barrier_pattern.json5

@@ -68,7 +68,7 @@
       ]
     }
   ],
-  
+
   // Resource requirements
   "resource_requirements": [
     {

examples/json/resource_monitoring_demo.json5

@@ -29,13 +29,13 @@
       "resource_requirements": "medium"
     }
   ],
-  
+
   // File definitions
   "files": [],
-  
+
   // User data
   "user_data": [],
-  
+
   // Resource requirements
   "resource_requirements": [
     {
@@ -47,10 +47,10 @@
       "runtime": "PT5M"
     }
   ],
-  
+
   // Slurm schedulers
   "slurm_schedulers": [],
-  
+
     "resource_monitor": {
     "enabled": true,
     "granularity": "time_series",

examples/json/sample_workflow.json

@@ -102,7 +102,7 @@
       "runtime": "PT30M"
     },
     {
-      "name": "large_job", 
+      "name": "large_job",
       "num_cpus": 4,
       "num_gpus": 1,
       "num_nodes": 1,
@@ -123,11 +123,11 @@
     },
     {
       "name": "gpu_scheduler",
-      "account": "gpu_project", 
+      "account": "gpu_project",
       "gres": "gpu:1",
       "nodes": 1,
       "partition": "gpu",
-      "tmp": "50G", 
+      "tmp": "50G",
       "walltime": "04:00:00",
       "extra": "--constraint=v100"
     }