diff --git a/.changeset/event-sourced-entities.md b/.changeset/event-sourced-entities.md new file mode 100644 index 000000000..3a528c589 --- /dev/null +++ b/.changeset/event-sourced-entities.md @@ -0,0 +1,19 @@ +--- +"@workflow/core": patch +"@workflow/world": patch +"@workflow/world-local": patch +"@workflow/world-postgres": patch +"@workflow/world-vercel": patch +"@workflow/web": patch +"@workflow/web-shared": patch +--- + +perf: implement event-sourced architecture for runs, steps, and hooks + +- Add run lifecycle events (run_created, run_started, run_completed, run_failed, run_cancelled) +- Add step_retrying event for non-fatal step failures that will be retried +- Remove `fatal` field from step_failed event (step_failed now implies terminal failure) +- Rename step's `lastKnownError` to `error` for consistency with server +- Update world implementations to create/update entities from events via events.create() +- Entities (runs, steps, hooks) are now materializations of the event log +- This makes the system faster, easier to reason about, and resilient to data inconsistencies diff --git a/.changeset/remove-paused-resumed.md b/.changeset/remove-paused-resumed.md new file mode 100644 index 000000000..0090272f5 --- /dev/null +++ b/.changeset/remove-paused-resumed.md @@ -0,0 +1,15 @@ +--- +"@workflow/world": patch +"@workflow/world-local": patch +"@workflow/world-vercel": patch +"@workflow/cli": patch +"@workflow/web": patch +"@workflow/web-shared": patch +--- + +**BREAKING CHANGE**: Remove unused paused/resumed run events and states + +- Remove `run_paused` and `run_resumed` event types +- Remove `paused` status from `WorkflowRunStatus` +- Remove `PauseWorkflowRunParams` and `ResumeWorkflowRunParams` types +- Remove `pauseWorkflowRun` and `resumeWorkflowRun` functions from world-vercel diff --git a/docs/README.md b/docs/README.md index 8269d622b..d4520e10a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,3 +1,66 @@ # Workflow DevKit Docs Check out the docs [here](https://useworkflow.dev/) + +## Mermaid Diagram Style Guide + +When adding diagrams to documentation, follow these conventions for consistency. + +### Diagram Type + +Use `flowchart TD` (top-down) or `flowchart LR` (left-right) for flow diagrams: + +```mermaid +flowchart TD + A["Source Code"] --> B["Transform"] + B --> C["Output"] +``` + +### Node Syntax + +Use square brackets with double quotes for rectangular nodes: + +``` +A["Label Text"] # Correct - rectangular node +A[Label Text] # Avoid - can cause parsing issues +A(Label Text) # Avoid - rounded node, inconsistent style +``` + +### Edge Labels + +Use the pipe syntax with double quotes for edge labels: + +``` +A -->|"label"| B # Correct +A --> B # Correct (no label) +``` + +### Highlighting Important Nodes + +Use the purple color scheme to highlight terminal states or key components: + +``` +style NodeId fill:#a78bfa,stroke:#8b5cf6,color:#000 +``` + +Place all `style` declarations at the end of the diagram. + +### Complete Example + +```mermaid +flowchart TD + A["(start)"] --> B["pending"] + B -->|"started"| C["running"] + C -->|"completed"| D["completed"] + C -->|"failed"| E["failed"] + + style D fill:#a78bfa,stroke:#8b5cf6,color:#000 + style E fill:#a78bfa,stroke:#8b5cf6,color:#000 +``` + +### Guidelines + +- Keep diagrams simple and readable +- Use meaningful node labels +- Limit complexity - split into multiple diagrams if needed +- Add a legend or callout explaining highlighted nodes when appropriate diff --git a/docs/content/docs/foundations/streaming.mdx b/docs/content/docs/foundations/streaming.mdx index fdb684708..844f99779 100644 --- a/docs/content/docs/foundations/streaming.mdx +++ b/docs/content/docs/foundations/streaming.mdx @@ -83,7 +83,7 @@ This allows clients to reconnect and continue receiving data from where they lef [`ReadableStream`](https://developer.mozilla.org/en-US/docs/Web/API/ReadableStream) and [`WritableStream`](https://developer.mozilla.org/en-US/docs/Web/API/WritableStream) are standard Web Streams API types that Workflow DevKit makes serializable. These are not custom types - they follow the web standard - but Workflow DevKit adds the ability to pass them between functions while maintaining their streaming capabilities. -Unlike regular values that are fully serialized to the event log, streams maintain their streaming capabilities when passed between functions. +Unlike regular values that are fully serialized to the [event log](/docs/how-it-works/event-sourcing), streams maintain their streaming capabilities when passed between functions. **Key properties:** - Stream references can be passed between workflow and step functions @@ -151,7 +151,7 @@ async function processInputStream(input: ReadableStream) { You cannot read from or write to streams directly within a workflow function. All stream operations must happen in step functions. -Workflow functions must be deterministic to support replay. Since streams bypass the event log for performance, reading stream data in a workflow would break determinism - each replay could see different data. By requiring all stream operations to happen in steps, the framework ensures consistent behavior. +Workflow functions must be deterministic to support replay. Since streams bypass the [event log](/docs/how-it-works/event-sourcing) for performance, reading stream data in a workflow would break determinism - each replay could see different data. By requiring all stream operations to happen in steps, the framework ensures consistent behavior. For more on determinism and replay, see [Workflows and Steps](/docs/foundations/workflows-and-steps). diff --git a/docs/content/docs/foundations/workflows-and-steps.mdx b/docs/content/docs/foundations/workflows-and-steps.mdx index 15537d13e..3b2909bdd 100644 --- a/docs/content/docs/foundations/workflows-and-steps.mdx +++ b/docs/content/docs/foundations/workflows-and-steps.mdx @@ -36,10 +36,10 @@ export async function processOrderWorkflow(orderId: string) { **Key Characteristics:** - Runs in a sandboxed environment without full Node.js access -- All step results are persisted to the event log +- All step results are persisted to the [event log](/docs/how-it-works/event-sourcing) - Must be **deterministic** to allow resuming after failures -Determinism in the workflow is required to resume the workflow from a suspension. Essentially, the workflow code gets re-run multiple times during its lifecycle, each time using an event log to resume the workflow to the correct spot. +Determinism in the workflow is required to resume the workflow from a suspension. Essentially, the workflow code gets re-run multiple times during its lifecycle, each time using the [event log](/docs/how-it-works/event-sourcing) to resume the workflow to the correct spot. The sandboxed environment that workflows run in already ensures determinism. For instance, `Math.random` and `Date` constructors are fixed in workflow runs, so you are safe to use them, and the framework ensures that the values don't change across replays. @@ -111,7 +111,7 @@ Keep in mind that calling a step function outside of a workflow function will no ### Suspension and Resumption -Workflow functions have the ability to automatically suspend while they wait on asynchronous work. While suspended, the workflow's state is stored via the event log and no compute resources are used until the workflow resumes execution. +Workflow functions have the ability to automatically suspend while they wait on asynchronous work. While suspended, the workflow's state is stored via the [event log](/docs/how-it-works/event-sourcing) and no compute resources are used until the workflow resumes execution. There are multiple ways a workflow can suspend: diff --git a/docs/content/docs/how-it-works/code-transform.mdx b/docs/content/docs/how-it-works/code-transform.mdx index 1b13797c5..396a2acab 100644 --- a/docs/content/docs/how-it-works/code-transform.mdx +++ b/docs/content/docs/how-it-works/code-transform.mdx @@ -140,7 +140,7 @@ handleUserSignup.workflowId = "workflow//workflows/user.js//handleUserSignup"; / - The workflow function gets a `workflowId` property for runtime identification - The `"use workflow"` directive is removed -**Why this transformation?** When a workflow executes, it needs to replay past steps from the event log rather than re-executing them. The `WORKFLOW_USE_STEP` symbol is a special runtime hook that: +**Why this transformation?** When a workflow executes, it needs to replay past steps from the [event log](/docs/how-it-works/event-sourcing) rather than re-executing them. The `WORKFLOW_USE_STEP` symbol is a special runtime hook that: 1. Checks if the step has already been executed (in the event log) 2. If yes: Returns the cached result @@ -283,7 +283,7 @@ Because workflow functions are deterministic and have no side effects, they can - Can make API calls, database queries, etc. - Have full access to Node.js runtime and APIs -- Results are cached in the event log after first execution +- Results are cached in the [event log](/docs/how-it-works/event-sourcing) after first execution Learn more about [Workflows and Steps](/docs/foundations/workflows-and-steps). diff --git a/docs/content/docs/how-it-works/event-sourcing.mdx b/docs/content/docs/how-it-works/event-sourcing.mdx new file mode 100644 index 000000000..c2ca4e6b3 --- /dev/null +++ b/docs/content/docs/how-it-works/event-sourcing.mdx @@ -0,0 +1,247 @@ +--- +title: Event Sourcing +--- + + +This guide explores how the Workflow DevKit uses event sourcing internally. Understanding these concepts is helpful for debugging and building observability tools, but is not required to use workflows. For getting started with workflows, see the [getting started](/docs/getting-started) guides for your framework. + + +The Workflow DevKit uses event sourcing to track all state changes in workflow executions. Every mutation creates an event that is persisted to the event log, and entity state is derived by replaying these events. + +This page explains the event sourcing model and entity lifecycles. + +## Event Sourcing Overview + +Event sourcing is a persistence pattern where state changes are stored as a sequence of events rather than by updating records in place. The current state of any entity is reconstructed by replaying its events from the beginning. + +**Benefits for durable workflows:** + +- **Complete audit trail**: Every state change is recorded with its timestamp and context +- **Debugging**: Replay the exact sequence of events that led to any state +- **Consistency**: Events provide a single source of truth for all entity state +- **Recoverability**: State can be reconstructed from the event log after failures + +In the Workflow DevKit, the following entity types are managed through events: + +- **Runs**: Workflow execution instances (materialized in storage) +- **Steps**: Individual atomic operations within a workflow (materialized in storage) +- **Hooks**: Suspension points that can receive external data (materialized in storage) +- **Waits**: Sleep or delay operations (tracked via events only, not materialized) + +## Entity Lifecycles + +Each entity type follows a specific lifecycle defined by the events that can affect it. Events transition entities between states, and certain states are terminal—once reached, no further transitions are possible. + + +In the diagrams below, purple nodes indicate terminal states that cannot be transitioned out of. + + +### Run Lifecycle + +A run represents a single execution of a workflow function. Runs begin in `pending` state when created, transition to `running` when execution starts, and end in one of three terminal states. + +```mermaid +flowchart TD + A["(start)"] -->|"run_created"| B["pending"] + B -->|"run_started"| C["running"] + C -->|"run_completed"| D["completed"] + C -->|"run_failed"| E["failed"] + C -->|"run_cancelled"| F["cancelled"] + B -->|"run_cancelled"| F + + style D fill:#a78bfa,stroke:#8b5cf6,color:#000 + style E fill:#a78bfa,stroke:#8b5cf6,color:#000 + style F fill:#a78bfa,stroke:#8b5cf6,color:#000 +``` + +**Run states:** + +- `pending`: Created but not yet executing +- `running`: Actively executing workflow code +- `completed`: Finished successfully with an output value +- `failed`: Terminated due to an unrecoverable error +- `cancelled`: Explicitly cancelled by the user or system + +### Step Lifecycle + +A step represents a single invocation of a step function. Steps can retry on failure, either transitioning back to `pending` via `step_retrying` or being re-executed directly with another `step_started` event. + +```mermaid +flowchart TD + A["(start)"] -->|"step_created"| B["pending"] + B -->|"step_started"| C["running"] + C -->|"step_completed"| D["completed"] + C -->|"step_failed"| E["failed"] + C -.->|"step_retrying"| B + + style D fill:#a78bfa,stroke:#8b5cf6,color:#000 + style E fill:#a78bfa,stroke:#8b5cf6,color:#000 +``` + +**Step states:** + +- `pending`: Created but not yet executing, or waiting to retry +- `running`: Actively executing step code +- `completed`: Finished successfully with a result value +- `failed`: Terminated after exhausting all retry attempts +- `cancelled`: Reserved for future use (not currently emitted) + + +The `step_retrying` event is optional. Steps can retry without it - the retry mechanism works regardless of whether this event is emitted. You may see back-to-back `step_started` events in logs when a step retries after a timeout or when the error is not explicitly captured. See [Errors and Retries](/docs/foundations/errors-and-retries) for more on how retries work. + + +When present, the `step_retrying` event moves a step back to `pending` state and records the error that caused the retry. This provides two benefits: + +- **Cleaner observability**: The event log explicitly shows retry transitions rather than consecutive `step_started` events +- **Error history**: The error that triggered the retry is preserved for debugging + +### Hook Lifecycle + +A hook represents a suspension point that can receive external data, created by [`createHook()`](/docs/api-reference/workflow/create-hook). Hooks enable workflows to pause and wait for external events, user interactions, or HTTP requests. Webhooks (created with [`createWebhook()`](/docs/api-reference/workflow/create-webhook)) are a higher-level abstraction built on hooks that adds automatic HTTP request/response handling. + +Hooks can receive multiple payloads while active and are disposed when no longer needed. + +```mermaid +flowchart TD + A["(start)"] -->|"hook_created"| B["active"] + B -->|"hook_received"| B + B -->|"hook_disposed"| C["disposed"] + + style C fill:#a78bfa,stroke:#8b5cf6,color:#000 +``` + +**Hook states:** + +- `active`: Ready to receive payloads (hook exists in storage) +- `disposed`: No longer accepting payloads (hook is deleted from storage) + +Unlike other entities, hooks don't have a `status` field—the states above are conceptual. An "active" hook is one that exists in storage, while "disposed" means the hook has been deleted. When a `hook_disposed` event is created, the hook record is removed rather than updated. + +While a hook is active, its token is reserved and cannot be used by other workflows. This prevents token reuse conflicts across concurrent workflows. When a hook is disposed (either explicitly or when its workflow completes), the token is released and can be claimed by future workflows. Hooks are automatically disposed when a workflow reaches a terminal state (`completed`, `failed`, or `cancelled`). The `hook_disposed` event is only needed for explicit disposal before workflow completion. + +See [Hooks & Webhooks](/docs/foundations/hooks) for more on how hooks and webhooks work. + +### Wait Lifecycle + +A wait represents a sleep operation created by [`sleep()`](/docs/api-reference/workflow/sleep). Waits track when a delay period has elapsed. + +```mermaid +flowchart TD + A["(start)"] -->|"wait_created"| B["waiting"] + B -->|"wait_completed"| C["completed"] + + style C fill:#a78bfa,stroke:#8b5cf6,color:#000 +``` + +**Wait states:** + +- `waiting`: Delay period has not yet elapsed +- `completed`: Delay period has elapsed, workflow can resume + + +Unlike Runs, Steps, and Hooks, waits are conceptual entities tracked only through events. There is no separate "Wait" record in storage that can be queried—the wait state is derived entirely from the `wait_created` and `wait_completed` events in the event log. + + +## Event Types Reference + +Events are categorized by the entity type they affect. Each event contains metadata including a timestamp and a `correlationId` that links the event to a specific entity: + +- Step events use the `stepId` as the correlation ID +- Hook events use the `hookId` as the correlation ID +- Wait events use the `waitId` as the correlation ID +- Run events do not require a correlation ID since the `runId` itself identifies the entity + +### Run Events + +| Event | Description | +|-------|-------------| +| `run_created` | Creates a new workflow run in `pending` state. Contains the deployment ID, workflow name, input arguments, and optional execution context. | +| `run_started` | Transitions the run to `running` state when execution begins. | +| `run_completed` | Transitions the run to `completed` state with the workflow's return value. | +| `run_failed` | Transitions the run to `failed` state with error details and optional error code. | +| `run_cancelled` | Transitions the run to `cancelled` state. Can be triggered from `pending` or `running` states. | + +### Step Events + +| Event | Description | +|-------|-------------| +| `step_created` | Creates a new step in `pending` state. Contains the step name and serialized input arguments. | +| `step_started` | Transitions the step to `running` state. Includes the current attempt number for retries. | +| `step_completed` | Transitions the step to `completed` state with the step's return value. | +| `step_failed` | Transitions the step to `failed` state with error details. The step will not be retried. | +| `step_retrying` | (Optional) Transitions the step back to `pending` state for retry. Contains the error that caused the retry and optional delay before the next attempt. When not emitted, retries appear as consecutive `step_started` events. | + +### Hook Events + +| Event | Description | +|-------|-------------| +| `hook_created` | Creates a new hook in `active` state. Contains the hook token and optional metadata. | +| `hook_received` | Records that a payload was delivered to the hook. The hook remains `active` and can receive more payloads. | +| `hook_disposed` | Deletes the hook from storage (conceptually transitioning to `disposed` state). The token is released for reuse by future workflows. | + +### Wait Events + +| Event | Description | +|-------|-------------| +| `wait_created` | Creates a new wait in `waiting` state. Contains the timestamp when the wait should complete. | +| `wait_completed` | Transitions the wait to `completed` state when the delay period has elapsed. | + +## Terminal States + +Terminal states represent the end of an entity's lifecycle. Once an entity reaches a terminal state, no further events can transition it to another state. + +**Run terminal states:** + +- `completed`: Workflow finished successfully +- `failed`: Workflow encountered an unrecoverable error +- `cancelled`: Workflow was explicitly cancelled + +**Step terminal states:** + +- `completed`: Step finished successfully +- `failed`: Step failed after all retry attempts + +**Hook terminal states:** + +- `disposed`: Hook has been deleted from storage and is no longer active + +**Wait terminal states:** + +- `completed`: Delay period has elapsed + +Attempting to create an event that would transition an entity out of a terminal state will result in an error. This prevents inconsistent state and ensures the integrity of the event log. + +## Event Correlation + +Events use a `correlationId` to link related events together. For step, hook, and wait events, the correlation ID identifies the specific entity instance: + +- Step events share the same `correlationId` (the step ID) across all events for that step execution +- Hook events share the same `correlationId` (the hook ID) across all events for that hook +- Wait events share the same `correlationId` (the wait ID) across creation and completion + +Run events do not require a correlation ID since the `runId` itself provides the correlation. + +This correlation enables: + +- Querying all events for a specific step, hook, or wait +- Building timelines of entity lifecycle transitions +- Debugging by tracing the complete history of any entity + +## Entity IDs + +All entities in the Workflow DevKit use a consistent ID format: a 4-character prefix followed by an underscore and a [ULID](https://github.com/ulid/spec) (Universally Unique Lexicographically Sortable Identifier). + +| Entity | Prefix | Example | +|--------|--------|---------| +| Run | `wrun_` | `wrun_01HXYZ123ABC456DEF789GHJ` | +| Step | `step_` | `step_01HXYZ123ABC456DEF789GHJ` | +| Hook | `hook_` | `hook_01HXYZ123ABC456DEF789GHJ` | +| Wait | `wait_` | `wait_01HXYZ123ABC456DEF789GHJ` | +| Event | `evnt_` | `evnt_01HXYZ123ABC456DEF789GHJ` | +| Stream | `strm_` | `strm_01HXYZ123ABC456DEF789GHJ` | + +**Why this format?** + +- **Prefixes enable introspection**: Given any ID, you can immediately identify what type of entity it refers to. This makes debugging, logging, and cross-referencing entities across the system straightforward. + +- **ULIDs enable chronological ordering**: Unlike UUIDs, ULIDs encode a timestamp in their first 48 bits, making them lexicographically sortable by creation time. This property is essential for the event log—events are always stored and retrieved in the correct chronological order simply by sorting their IDs. diff --git a/docs/content/docs/how-it-works/meta.json b/docs/content/docs/how-it-works/meta.json index 09d452d82..a69c654e4 100644 --- a/docs/content/docs/how-it-works/meta.json +++ b/docs/content/docs/how-it-works/meta.json @@ -3,7 +3,8 @@ "pages": [ "understanding-directives", "code-transform", - "framework-integrations" + "framework-integrations", + "event-sourcing" ], "defaultOpen": false } diff --git a/docs/content/docs/how-it-works/understanding-directives.mdx b/docs/content/docs/how-it-works/understanding-directives.mdx index 5f2787459..d543b793c 100644 --- a/docs/content/docs/how-it-works/understanding-directives.mdx +++ b/docs/content/docs/how-it-works/understanding-directives.mdx @@ -47,7 +47,7 @@ export async function onboardUser(userId: string) { } ``` -**The key insight:** Workflows resume from suspension by replaying their code using cached step results from the event log. When a step like `await fetchUserData(userId)` is called: +**The key insight:** Workflows resume from suspension by replaying their code using cached step results from the [event log](/docs/how-it-works/event-sourcing). When a step like `await fetchUserData(userId)` is called: - **If already executed:** Returns the cached result immediately from the event log - **If not yet executed:** Suspends the workflow, enqueues the step for background execution, and resumes later with the result diff --git a/docs/content/docs/observability/index.mdx b/docs/content/docs/observability/index.mdx index 4daf9b867..841a8edef 100644 --- a/docs/content/docs/observability/index.mdx +++ b/docs/content/docs/observability/index.mdx @@ -2,7 +2,7 @@ title: Observability --- -Workflow DevKit provides powerful tools to inspect, monitor, and debug your workflows through the CLI and Web UI. These tools allow you to inspect workflow runs, steps, webhooks, events, and stream output. +Workflow DevKit provides powerful tools to inspect, monitor, and debug your workflows through the CLI and Web UI. These tools allow you to inspect workflow runs, steps, webhooks, [events](/docs/how-it-works/event-sourcing), and stream output. ## Quick Start diff --git a/docs/content/docs/worlds/index.mdx b/docs/content/docs/worlds/index.mdx index 67c4d7c7e..5549ae93d 100644 --- a/docs/content/docs/worlds/index.mdx +++ b/docs/content/docs/worlds/index.mdx @@ -14,7 +14,7 @@ The Workflow `World` is an interface that abstracts how workflows and steps comm ## What is a World? A World implementation handles: -- **Workflow Storage**: Persisting workflow state and event logs +- **Workflow Storage**: Persisting workflow state and [event logs](/docs/how-it-works/event-sourcing) - **Step Execution**: Managing step function invocations - **Message Passing**: Communication between workflow orchestrator and step functions diff --git a/packages/cli/src/lib/inspect/output.ts b/packages/cli/src/lib/inspect/output.ts index 75700d15f..67cf974b0 100644 --- a/packages/cli/src/lib/inspect/output.ts +++ b/packages/cli/src/lib/inspect/output.ts @@ -101,7 +101,6 @@ const STATUS_COLORS: Record< failed: chalk.red, cancelled: chalk.strikethrough.yellow, pending: chalk.blue, - paused: chalk.yellow, }; const isStreamId = (value: string) => { @@ -116,7 +115,6 @@ const showStatusLegend = () => { 'failed', 'cancelled', 'pending', - 'paused', ]; const legendItems = statuses.map((status) => { diff --git a/packages/cli/src/lib/inspect/run.ts b/packages/cli/src/lib/inspect/run.ts index 8a4f0f2f2..6827e3aa8 100644 --- a/packages/cli/src/lib/inspect/run.ts +++ b/packages/cli/src/lib/inspect/run.ts @@ -68,6 +68,6 @@ export const startRun = async ( }; export const cancelRun = async (world: World, runId: string) => { - await world.runs.cancel(runId); + await world.events.create(runId, { eventType: 'run_cancelled' }); logger.log(chalk.green(`Cancel signal sent to run ${runId}`)); }; diff --git a/packages/core/e2e/e2e.test.ts b/packages/core/e2e/e2e.test.ts index e39107b1b..5f7a173d1 100644 --- a/packages/core/e2e/e2e.test.ts +++ b/packages/core/e2e/e2e.test.ts @@ -799,6 +799,11 @@ describe('e2e', () => { } ); + // TODO: Add test for concurrent hook token conflict once workflow-server PR is merged and deployed + // PR: https://github.com/vercel/workflow-server/pull/XXX (pranaygp/event-sourced-api-v3 branch) + // The test should verify that two concurrent workflows cannot use the same hook token + // See: hookCleanupTestWorkflow for sequential token reuse (after workflow completion) + test( 'stepFunctionPassingWorkflow - step function references can be passed as arguments (without closure vars)', { timeout: 60_000 }, diff --git a/packages/core/src/events-consumer.test.ts b/packages/core/src/events-consumer.test.ts index dfb73e2ae..90fd141ab 100644 --- a/packages/core/src/events-consumer.test.ts +++ b/packages/core/src/events-consumer.test.ts @@ -73,6 +73,7 @@ describe('EventsConsumer', () => { await waitForNextTick(); expect(callback).toHaveBeenCalledWith(event); + // Without auto-advance, callback is only called once expect(callback).toHaveBeenCalledTimes(1); }); }); @@ -87,6 +88,7 @@ describe('EventsConsumer', () => { await waitForNextTick(); expect(callback).toHaveBeenCalledWith(event); + // Without auto-advance, callback is only called once expect(callback).toHaveBeenCalledTimes(1); }); @@ -109,23 +111,27 @@ describe('EventsConsumer', () => { consumer.subscribe(callback); await waitForNextTick(); + // callback finishes at event1, index advances to 1 + // Without auto-advance, event2 is NOT processed expect(consumer.eventIndex).toBe(1); expect(consumer.callbacks).toHaveLength(0); }); - it('should not increment event index when callback returns false', async () => { + it('should NOT auto-advance when all callbacks return NotConsumed', async () => { const event = createMockEvent(); const consumer = new EventsConsumer([event]); const callback = vi.fn().mockReturnValue(EventConsumerResult.NotConsumed); consumer.subscribe(callback); await waitForNextTick(); + await waitForNextTick(); // Extra tick to confirm no auto-advance + // Without auto-advance, eventIndex stays at 0 expect(consumer.eventIndex).toBe(0); expect(consumer.callbacks).toContain(callback); }); - it('should process multiple callbacks until one returns true', async () => { + it('should process multiple callbacks until one returns Consumed or Finished', async () => { const event = createMockEvent(); const consumer = new EventsConsumer([event]); const callback1 = vi @@ -140,15 +146,17 @@ describe('EventsConsumer', () => { consumer.subscribe(callback2); consumer.subscribe(callback3); await waitForNextTick(); + await waitForNextTick(); // For next event processing expect(callback1).toHaveBeenCalledWith(event); expect(callback2).toHaveBeenCalledWith(event); + // callback3 sees the next event (null since we only have one event) expect(callback3).toHaveBeenCalledWith(null); expect(consumer.eventIndex).toBe(1); expect(consumer.callbacks).toEqual([callback1, callback3]); }); - it('should process all callbacks when none return true', async () => { + it('should NOT advance when all callbacks return NotConsumed', async () => { const event = createMockEvent(); const consumer = new EventsConsumer([event]); const callback1 = vi @@ -169,6 +177,7 @@ describe('EventsConsumer', () => { expect(callback1).toHaveBeenCalledWith(event); expect(callback2).toHaveBeenCalledWith(event); expect(callback3).toHaveBeenCalledWith(event); + // Without auto-advance, eventIndex stays at 0 expect(consumer.eventIndex).toBe(0); expect(consumer.callbacks).toEqual([callback1, callback2, callback3]); }); @@ -211,7 +220,7 @@ describe('EventsConsumer', () => { expect(callback2).toHaveBeenCalledWith(null); }); - it('should handle complex event processing scenario', async () => { + it('should handle complex event processing with multiple consumers', async () => { const events = [ createMockEvent({ id: 'event-1', event_type: 'type-a' }), createMockEvent({ id: 'event-2', event_type: 'type-b' }), @@ -241,13 +250,14 @@ describe('EventsConsumer', () => { consumer.subscribe(typeBCallback); await waitForNextTick(); await waitForNextTick(); // Wait for recursive processing - await waitForNextTick(); // Wait for final processing - // typeACallback processes event-1 and gets removed, so it won't process event-3 + // typeACallback processes event-1 and gets removed expect(typeACallback).toHaveBeenCalledTimes(1); // Called for event-1 only + // typeBCallback processes event-2 and gets removed expect(typeBCallback).toHaveBeenCalledTimes(1); // Called for event-2 - expect(consumer.eventIndex).toBe(2); // Only 2 events processed (event-3 remains) - expect(consumer.callbacks).toHaveLength(0); // Both callbacks removed after consuming their events + // eventIndex is at 2 (after event-1 and event-2 were consumed) + expect(consumer.eventIndex).toBe(2); + expect(consumer.callbacks).toHaveLength(0); }); }); @@ -297,8 +307,9 @@ describe('EventsConsumer', () => { consumer.subscribe(callback3); await waitForNextTick(); - // callback2 should be removed when it returns true + // callback2 should be removed when it returns Finished expect(consumer.callbacks).toEqual([callback1, callback3]); + // callback3 is called with the next event (null after event-1) expect(callback3).toHaveBeenCalledWith(null); }); @@ -314,25 +325,6 @@ describe('EventsConsumer', () => { expect(consumer.eventIndex).toBe(1); }); - it('should handle multiple subscriptions happening in sequence', async () => { - const event1 = createMockEvent({ id: 'event-1' }); - const event2 = createMockEvent({ id: 'event-2' }); - const consumer = new EventsConsumer([event1, event2]); - - const callback1 = vi.fn().mockReturnValue(EventConsumerResult.Finished); - const callback2 = vi.fn().mockReturnValue(EventConsumerResult.Finished); - - consumer.subscribe(callback1); - await waitForNextTick(); - - consumer.subscribe(callback2); - await waitForNextTick(); - - expect(callback1).toHaveBeenCalledWith(event1); - expect(callback2).toHaveBeenCalledWith(event2); - expect(consumer.eventIndex).toBe(2); - }); - it('should handle empty events array gracefully', async () => { const consumer = new EventsConsumer([]); const callback = vi.fn().mockReturnValue(EventConsumerResult.NotConsumed); @@ -343,5 +335,49 @@ describe('EventsConsumer', () => { expect(callback).toHaveBeenCalledWith(null); expect(consumer.eventIndex).toBe(0); }); + + it('should process events in order with proper consumers', async () => { + // This test simulates the workflow scenario: + // - run_created consumer consumes it + // - step consumer gets step_created, step_completed + const events = [ + createMockEvent({ id: 'run-created', event_type: 'run_created' }), + createMockEvent({ id: 'step-created', event_type: 'step_created' }), + createMockEvent({ id: 'step-completed', event_type: 'step_completed' }), + ]; + const consumer = new EventsConsumer(events); + + // Run lifecycle consumer - consumes run_created + const runConsumer = vi.fn().mockImplementation((event: Event | null) => { + if (event?.event_type === 'run_created') { + return EventConsumerResult.Consumed; + } + return EventConsumerResult.NotConsumed; + }); + + // Step consumer - consumes step_created, finishes on step_completed + const stepConsumer = vi.fn().mockImplementation((event: Event | null) => { + if (event?.event_type === 'step_created') { + return EventConsumerResult.Consumed; + } + if (event?.event_type === 'step_completed') { + return EventConsumerResult.Finished; + } + return EventConsumerResult.NotConsumed; + }); + + consumer.subscribe(runConsumer); + consumer.subscribe(stepConsumer); + await waitForNextTick(); + await waitForNextTick(); + await waitForNextTick(); + + // runConsumer consumes run_created + expect(runConsumer).toHaveBeenCalledWith(events[0]); + // stepConsumer consumes step_created, then finishes on step_completed + expect(stepConsumer).toHaveBeenCalledWith(events[1]); + expect(stepConsumer).toHaveBeenCalledWith(events[2]); + expect(consumer.eventIndex).toBe(3); + }); }); }); diff --git a/packages/core/src/events-consumer.ts b/packages/core/src/events-consumer.ts index f38d7fbd6..221d111fa 100644 --- a/packages/core/src/events-consumer.ts +++ b/packages/core/src/events-consumer.ts @@ -78,5 +78,10 @@ export class EventsConsumer { return; } } + + // If we reach here, all callbacks returned NotConsumed. + // We do NOT auto-advance - every event must have a consumer. + // With proper consumers for run_created/run_started/step_created, + // this should not cause events to get stuck. }; } diff --git a/packages/core/src/global.ts b/packages/core/src/global.ts index 17a680dc4..7f1853512 100644 --- a/packages/core/src/global.ts +++ b/packages/core/src/global.ts @@ -6,6 +6,7 @@ export interface StepInvocationQueueItem { stepName: string; args: Serializable[]; closureVars?: Record; + hasCreatedEvent?: boolean; } export interface HookInvocationQueueItem { diff --git a/packages/core/src/runtime.ts b/packages/core/src/runtime.ts index cf38e3bcf..6cf16ff5f 100644 --- a/packages/core/src/runtime.ts +++ b/packages/core/src/runtime.ts @@ -104,7 +104,9 @@ export class Run { * Cancels the workflow run. */ async cancel(): Promise { - await this.world.runs.cancel(this.runId); + await this.world.events.create(this.runId, { + eventType: 'run_cancelled', + }); } /** @@ -272,10 +274,14 @@ export function workflowEntrypoint( let workflowRun = await world.runs.get(runId); if (workflowRun.status === 'pending') { - workflowRun = await world.runs.update(runId, { - // This sets the `startedAt` timestamp at the database level - status: 'running', + // Transition run to 'running' via event (event-sourced architecture) + const result = await world.events.create(runId, { + eventType: 'run_started', }); + // Use the run entity from the event response (no extra get call needed) + if (result.run) { + workflowRun = result.run; + } } // At this point, the workflow is "running" and `startedAt` should @@ -310,27 +316,35 @@ export function workflowEntrypoint( // Load all events into memory before running const events = await getAllWorkflowRunEvents(workflowRun.runId); - // Check for any elapsed waits and create wait_completed events + // Check for any elapsed waits and batch create wait_completed events const now = Date.now(); - for (const event of events) { - if (event.eventType === 'wait_created') { - const resumeAt = event.eventData.resumeAt as Date; - const hasCompleted = events.some( - (e) => - e.eventType === 'wait_completed' && - e.correlationId === event.correlationId - ); - // If wait has elapsed and hasn't been completed yet - if (!hasCompleted && now >= resumeAt.getTime()) { - const completedEvent = await world.events.create(runId, { - eventType: 'wait_completed', - correlationId: event.correlationId, - }); - // Add the event to the events array so the workflow can see it - events.push(completedEvent); - } - } + // Pre-compute completed correlation IDs for O(n) lookup instead of O(n²) + const completedWaitIds = new Set( + events + .filter((e) => e.eventType === 'wait_completed') + .map((e) => e.correlationId) + ); + + // Collect all waits that need completion + const waitsToComplete = events + .filter( + (e): e is typeof e & { correlationId: string } => + e.eventType === 'wait_created' && + e.correlationId !== undefined && + !completedWaitIds.has(e.correlationId) && + now >= (e.eventData.resumeAt as Date).getTime() + ) + .map((e) => ({ + eventType: 'wait_completed' as const, + correlationId: e.correlationId, + })); + + // Create all wait_completed events + for (const waitEvent of waitsToComplete) { + const result = await world.events.create(runId, waitEvent); + // Add the event to the events array so the workflow can see it + events.push(result.event); } const result = await runWorkflow( @@ -339,10 +353,12 @@ export function workflowEntrypoint( events ); - // Update the workflow run with the result - await world.runs.update(runId, { - status: 'completed', - output: result as Serializable, + // Complete the workflow run via event (event-sourced architecture) + await world.events.create(runId, { + eventType: 'run_completed', + eventData: { + output: result as Serializable, + }, }); span?.setAttributes({ @@ -393,14 +409,18 @@ export function workflowEntrypoint( console.error( `${errorName} while running "${runId}" workflow:\n\n${errorStack}` ); - await world.runs.update(runId, { - status: 'failed', - error: { - message: errorMessage, - stack: errorStack, + // Fail the workflow run via event (event-sourced architecture) + await world.events.create(runId, { + eventType: 'run_failed', + eventData: { + error: { + message: errorMessage, + stack: errorStack, + }, // TODO: include error codes when we define them }, }); + span?.setAttributes({ ...Attribute.WorkflowRunStatus('failed'), ...Attribute.WorkflowErrorName(errorName), diff --git a/packages/core/src/runtime/start.ts b/packages/core/src/runtime/start.ts index 687bf012b..2c43395f5 100644 --- a/packages/core/src/runtime/start.ts +++ b/packages/core/src/runtime/start.ts @@ -92,22 +92,30 @@ export async function start( const { promise: runIdPromise, resolve: resolveRunId } = withResolvers(); + // Serialize current trace context to propagate across queue boundary + const traceCarrier = await serializeTraceCarrier(); + + // Create run via run_created event (event-sourced architecture) + // Pass null for runId - the server generates it and returns it in the response const workflowArguments = dehydrateWorkflowArguments( args, ops, runIdPromise ); - // Serialize current trace context to propagate across queue boundary - const traceCarrier = await serializeTraceCarrier(); - const runResponse = await world.runs.create({ - deploymentId: deploymentId, - workflowName: workflowName, - input: workflowArguments, - executionContext: { traceCarrier }, + const result = await world.events.create(null, { + eventType: 'run_created', + eventData: { + deploymentId: deploymentId, + workflowName: workflowName, + input: workflowArguments, + executionContext: { traceCarrier }, + }, }); - resolveRunId(runResponse.runId); + // Get the server-generated runId from the event response + const runId = result.event.runId; + resolveRunId(runId); waitUntil( Promise.all(ops).catch((err) => { @@ -119,15 +127,15 @@ export async function start( ); span?.setAttributes({ - ...Attribute.WorkflowRunId(runResponse.runId), - ...Attribute.WorkflowRunStatus(runResponse.status), + ...Attribute.WorkflowRunId(runId), + ...Attribute.WorkflowRunStatus('pending'), ...Attribute.DeploymentId(deploymentId), }); await world.queue( `__wkf_workflow_${workflowName}`, { - runId: runResponse.runId, + runId, traceCarrier, } satisfies WorkflowInvokePayload, { @@ -135,7 +143,7 @@ export async function start( } ); - return new Run(runResponse.runId); + return new Run(runId); }); }); } diff --git a/packages/core/src/runtime/step-handler.ts b/packages/core/src/runtime/step-handler.ts index 5390ccada..771c7f25a 100644 --- a/packages/core/src/runtime/step-handler.ts +++ b/packages/core/src/runtime/step-handler.ts @@ -114,29 +114,22 @@ const stepHandler = getWorldHandlers().createQueueHandler( } let result: unknown; - const attempt = step.attempt + 1; // Check max retries FIRST before any state changes. + // step.attempt tracks how many times step_started has been called. + // If step.attempt >= maxRetries, we've already tried maxRetries times. // This handles edge cases where the step handler is invoked after max retries have been exceeded - // (e.g., when the step repeatedly times out or fails before reaching the catch handler at line 822). + // (e.g., when the step repeatedly times out or fails before reaching the catch handler). // Without this check, the step would retry forever. - if (attempt > maxRetries) { - const errorMessage = `Step "${stepName}" exceeded max retries (${attempt} attempts)`; + if (step.attempt >= maxRetries) { + const errorMessage = `Step "${stepName}" exceeded max retries (${step.attempt} attempts)`; console.error(`[Workflows] "${workflowRunId}" - ${errorMessage}`); - // Update step status first (idempotent), then create event - await world.steps.update(workflowRunId, stepId, { - status: 'failed', - error: { - message: errorMessage, - stack: undefined, - }, - }); + // Fail the step via event (event-sourced architecture) await world.events.create(workflowRunId, { eventType: 'step_failed', correlationId: stepId, eventData: { error: errorMessage, - fatal: true, }, }); @@ -192,15 +185,23 @@ const stepHandler = getWorldHandlers().createQueueHandler( return; } - await world.events.create(workflowRunId, { - eventType: 'step_started', // TODO: Replace with 'step_retrying' + // Start the step via event (event-sourced architecture) + // step_started increments the attempt counter in the World implementation + const startResult = await world.events.create(workflowRunId, { + eventType: 'step_started', correlationId: stepId, }); - step = await world.steps.update(workflowRunId, stepId, { - attempt, - status: 'running', - }); + // Use the step entity from the event response (no extra get call needed) + if (!startResult.step) { + throw new WorkflowRuntimeError( + `step_started event for "${stepId}" did not return step entity` + ); + } + step = startResult.step; + + // step.attempt is now the current attempt number (after increment) + const attempt = step.attempt; if (!step.startedAt) { throw new WorkflowRuntimeError( @@ -257,16 +258,8 @@ const stepHandler = getWorldHandlers().createQueueHandler( }) ); - // Mark the step as completed first. This order is important. If a concurrent - // execution marked the step as complete, this request should throw, and - // this prevent the step_completed event in the event log - // TODO: this should really be atomic and handled by the world - await world.steps.update(workflowRunId, stepId, { - status: 'completed', - output: result as Serializable, - }); - - // Then, append the event log with the step result + // Complete the step via event (event-sourced architecture) + // The event creation atomically updates the step entity await world.events.create(workflowRunId, { eventType: 'step_completed', correlationId: stepId, @@ -301,22 +294,13 @@ const stepHandler = getWorldHandlers().createQueueHandler( console.error( `[Workflows] "${workflowRunId}" - Encountered \`FatalError\` while executing step "${stepName}":\n > ${stackLines.join('\n > ')}\n\nBubbling up error to parent workflow` ); - // Fatal error - store the error in the event log and re-invoke the workflow + // Fail the step via event (event-sourced architecture) await world.events.create(workflowRunId, { eventType: 'step_failed', correlationId: stepId, eventData: { error: String(err), stack: errorStack, - fatal: true, - }, - }); - await world.steps.update(workflowRunId, stepId, { - status: 'failed', - error: { - message: err.message || String(err), - stack: errorStack, - // TODO: include error codes when we define them }, }); @@ -326,34 +310,29 @@ const stepHandler = getWorldHandlers().createQueueHandler( }); } else { const maxRetries = stepFn.maxRetries ?? DEFAULT_STEP_MAX_RETRIES; + // step.attempt was incremented by step_started, use it here + const currentAttempt = step.attempt; span?.setAttributes({ - ...Attribute.StepAttempt(attempt), + ...Attribute.StepAttempt(currentAttempt), ...Attribute.StepMaxRetries(maxRetries), }); - if (attempt > maxRetries) { - // Max retries reached + if (currentAttempt >= maxRetries) { + // Max retries reached (consistent with pre-execution check) const errorStack = getErrorStack(err); const stackLines = errorStack.split('\n').slice(0, 4); console.error( - `[Workflows] "${workflowRunId}" - Encountered \`Error\` while executing step "${stepName}" (attempt ${attempt}):\n > ${stackLines.join('\n > ')}\n\n Max retries reached\n Bubbling error to parent workflow` + `[Workflows] "${workflowRunId}" - Encountered \`Error\` while executing step "${stepName}" (attempt ${currentAttempt}):\n > ${stackLines.join('\n > ')}\n\n Max retries reached\n Bubbling error to parent workflow` ); const errorMessage = `Step "${stepName}" failed after max retries: ${String(err)}`; + // Fail the step via event (event-sourced architecture) await world.events.create(workflowRunId, { eventType: 'step_failed', correlationId: stepId, eventData: { error: errorMessage, stack: errorStack, - fatal: true, - }, - }); - await world.steps.update(workflowRunId, stepId, { - status: 'failed', - error: { - message: errorMessage, - stack: errorStack, }, }); @@ -365,30 +344,29 @@ const stepHandler = getWorldHandlers().createQueueHandler( // Not at max retries yet - log as a retryable error if (RetryableError.is(err)) { console.warn( - `[Workflows] "${workflowRunId}" - Encountered \`RetryableError\` while executing step "${stepName}" (attempt ${attempt}):\n > ${String(err.message)}\n\n This step has failed but will be retried` + `[Workflows] "${workflowRunId}" - Encountered \`RetryableError\` while executing step "${stepName}" (attempt ${currentAttempt}):\n > ${String(err.message)}\n\n This step has failed but will be retried` ); } else { const stackLines = getErrorStack(err).split('\n').slice(0, 4); console.error( - `[Workflows] "${workflowRunId}" - Encountered \`Error\` while executing step "${stepName}" (attempt ${attempt}):\n > ${stackLines.join('\n > ')}\n\n This step has failed but will be retried` + `[Workflows] "${workflowRunId}" - Encountered \`Error\` while executing step "${stepName}" (attempt ${currentAttempt}):\n > ${stackLines.join('\n > ')}\n\n This step has failed but will be retried` ); } + // Set step to pending for retry via event (event-sourced architecture) + // step_retrying records the error and sets status to pending + const errorStack = getErrorStack(err); await world.events.create(workflowRunId, { - eventType: 'step_failed', + eventType: 'step_retrying', correlationId: stepId, eventData: { error: String(err), - stack: getErrorStack(err), + stack: errorStack, + ...(RetryableError.is(err) && { + retryAfter: err.retryAfter, + }), }, }); - await world.steps.update(workflowRunId, stepId, { - status: 'pending', // TODO: Should be "retrying" once we have that status - ...(RetryableError.is(err) && { - retryAfter: err.retryAfter, - }), - }); - const timeoutSeconds = Math.max( 1, RetryableError.is(err) diff --git a/packages/core/src/runtime/suspension-handler.ts b/packages/core/src/runtime/suspension-handler.ts index 11dcc81e7..493909c07 100644 --- a/packages/core/src/runtime/suspension-handler.ts +++ b/packages/core/src/runtime/suspension-handler.ts @@ -1,7 +1,7 @@ import type { Span } from '@opentelemetry/api'; import { waitUntil } from '@vercel/functions'; import { WorkflowAPIError } from '@workflow/errors'; -import type { World } from '@workflow/world'; +import type { CreateEventRequest, World } from '@workflow/world'; import type { HookInvocationQueueItem, StepInvocationQueueItem, @@ -27,178 +27,14 @@ export interface SuspensionHandlerResult { timeoutSeconds?: number; } -interface ProcessHookParams { - queueItem: HookInvocationQueueItem; - world: World; - runId: string; - global: typeof globalThis; -} - -/** - * Processes a single hook by creating it in the database and event log. - */ -async function processHook({ - queueItem, - world, - runId, - global, -}: ProcessHookParams): Promise { - try { - // Create hook in database - const hookMetadata = - typeof queueItem.metadata === 'undefined' - ? undefined - : dehydrateStepArguments(queueItem.metadata, global); - await world.hooks.create(runId, { - hookId: queueItem.correlationId, - token: queueItem.token, - metadata: hookMetadata, - }); - - // Create hook_created event in event log - await world.events.create(runId, { - eventType: 'hook_created', - correlationId: queueItem.correlationId, - }); - } catch (err) { - if (WorkflowAPIError.is(err)) { - if (err.status === 409) { - // Hook already exists (duplicate hook_id constraint), so we can skip it - console.warn( - `Hook with correlation ID "${queueItem.correlationId}" already exists, skipping: ${err.message}` - ); - return; - } else if (err.status === 410) { - // Workflow has already completed, so no-op - console.warn( - `Workflow run "${runId}" has already completed, skipping hook "${queueItem.correlationId}": ${err.message}` - ); - return; - } - } - throw err; - } -} - -interface ProcessStepParams { - queueItem: StepInvocationQueueItem; - world: World; - runId: string; - workflowName: string; - workflowStartedAt: number; - global: typeof globalThis; -} - -/** - * Processes a single step by creating it in the database and queueing execution. - */ -async function processStep({ - queueItem, - world, - runId, - workflowName, - workflowStartedAt, - global, -}: ProcessStepParams): Promise { - const ops: Promise[] = []; - const dehydratedInput = dehydrateStepArguments( - { - args: queueItem.args, - closureVars: queueItem.closureVars, - }, - global - ); - - try { - const step = await world.steps.create(runId, { - stepId: queueItem.correlationId, - stepName: queueItem.stepName, - input: dehydratedInput as Serializable, - }); - - waitUntil( - Promise.all(ops).catch((opErr) => { - // Ignore expected client disconnect errors (e.g., browser refresh during streaming) - const isAbortError = - opErr?.name === 'AbortError' || opErr?.name === 'ResponseAborted'; - if (!isAbortError) throw opErr; - }) - ); - - await queueMessage( - world, - `__wkf_step_${queueItem.stepName}`, - { - workflowName, - workflowRunId: runId, - workflowStartedAt, - stepId: step.stepId, - traceCarrier: await serializeTraceCarrier(), - requestedAt: new Date(), - }, - { - idempotencyKey: queueItem.correlationId, - } - ); - } catch (err) { - if (WorkflowAPIError.is(err) && err.status === 409) { - // Step already exists, so we can skip it - console.warn( - `Step "${queueItem.stepName}" with correlation ID "${queueItem.correlationId}" already exists, skipping: ${err.message}` - ); - return; - } - throw err; - } -} - -interface ProcessWaitParams { - queueItem: WaitInvocationQueueItem; - world: World; - runId: string; -} - -/** - * Processes a single wait by creating the event and calculating timeout. - * @returns The timeout in seconds, or null if the wait already exists. - */ -async function processWait({ - queueItem, - world, - runId, -}: ProcessWaitParams): Promise { - try { - // Only create wait_created event if it hasn't been created yet - if (!queueItem.hasCreatedEvent) { - await world.events.create(runId, { - eventType: 'wait_created', - correlationId: queueItem.correlationId, - eventData: { - resumeAt: queueItem.resumeAt, - }, - }); - } - - // Calculate how long to wait before resuming - const now = Date.now(); - const resumeAtMs = queueItem.resumeAt.getTime(); - const delayMs = Math.max(1000, resumeAtMs - now); - return Math.ceil(delayMs / 1000); - } catch (err) { - if (WorkflowAPIError.is(err) && err.status === 409) { - // Wait already exists, so we can skip it - console.warn( - `Wait with correlation ID "${queueItem.correlationId}" already exists, skipping: ${err.message}` - ); - return null; - } - throw err; - } -} - /** * Handles a workflow suspension by processing all pending operations (hooks, steps, waits). - * Hooks are processed first to prevent race conditions, then steps and waits in parallel. + * Uses an event-sourced architecture where entities (steps, hooks) are created atomically + * with their corresponding events via events.create(). + * + * Processing order: + * 1. Hooks are processed first to prevent race conditions with webhook receivers + * 2. Steps and waits are processed in parallel after hooks complete */ export async function handleSuspension({ suspension, @@ -208,7 +44,7 @@ export async function handleSuspension({ workflowStartedAt, span, }: SuspensionHandlerParams): Promise { - // Separate queue items by type for parallel processing + // Separate queue items by type const stepItems = suspension.steps.filter( (item): item is StepInvocationQueueItem => item.type === 'step' ); @@ -219,49 +55,157 @@ export async function handleSuspension({ (item): item is WaitInvocationQueueItem => item.type === 'wait' ); - // Process all hooks first to prevent race conditions - await Promise.all( - hookItems.map((queueItem) => - processHook({ - queueItem, - world, - runId, - global: suspension.globalThis, + // Build hook_created events (World will atomically create hook entities) + const hookEvents: CreateEventRequest[] = hookItems.map((queueItem) => { + const hookMetadata = + typeof queueItem.metadata === 'undefined' + ? undefined + : dehydrateStepArguments(queueItem.metadata, suspension.globalThis); + return { + eventType: 'hook_created' as const, + correlationId: queueItem.correlationId, + eventData: { + token: queueItem.token, + metadata: hookMetadata, + }, + }; + }); + + // Process hooks first to prevent race conditions with webhook receivers + // All hook creations run in parallel + if (hookEvents.length > 0) { + await Promise.all( + hookEvents.map(async (hookEvent) => { + try { + await world.events.create(runId, hookEvent); + } catch (err) { + if (WorkflowAPIError.is(err)) { + if (err.status === 409) { + console.warn(`Hook already exists, continuing: ${err.message}`); + } else if (err.status === 410) { + console.warn( + `Workflow run "${runId}" has already completed, skipping hook: ${err.message}` + ); + } else { + throw err; + } + } else { + throw err; + } + } }) - ) + ); + } + + // Build a map of stepId -> step event for steps that need creation + const stepsNeedingCreation = new Set( + stepItems + .filter((queueItem) => !queueItem.hasCreatedEvent) + .map((queueItem) => queueItem.correlationId) ); - // Then process steps and waits in parallel - const [, waitTimeouts] = await Promise.all([ - Promise.all( - stepItems.map((queueItem) => - processStep({ - queueItem, - world, - runId, - workflowName, - workflowStartedAt, - global: suspension.globalThis, - }) - ) - ), - Promise.all( - waitItems.map((queueItem) => - processWait({ - queueItem, + // Process steps and waits in parallel + // Each step: create event (if needed) -> queue message + // Each wait: create event (if needed) + const ops: Promise[] = []; + + // Steps: create event then queue message, all in parallel + for (const queueItem of stepItems) { + ops.push( + (async () => { + // Create step event if not already created + if (stepsNeedingCreation.has(queueItem.correlationId)) { + const dehydratedInput = dehydrateStepArguments( + { + args: queueItem.args, + closureVars: queueItem.closureVars, + }, + suspension.globalThis + ); + const stepEvent: CreateEventRequest = { + eventType: 'step_created' as const, + correlationId: queueItem.correlationId, + eventData: { + stepName: queueItem.stepName, + input: dehydratedInput as Serializable, + }, + }; + try { + await world.events.create(runId, stepEvent); + } catch (err) { + if (WorkflowAPIError.is(err) && err.status === 409) { + console.warn(`Step already exists, continuing: ${err.message}`); + } else { + throw err; + } + } + } + + // Queue step execution message + await queueMessage( world, - runId, - }) - ) - ), - ]); + `__wkf_step_${queueItem.stepName}`, + { + workflowName, + workflowRunId: runId, + workflowStartedAt, + stepId: queueItem.correlationId, + traceCarrier: await serializeTraceCarrier(), + requestedAt: new Date(), + }, + { + idempotencyKey: queueItem.correlationId, + } + ); + })() + ); + } + + // Waits: create events in parallel (no queueing needed for waits) + for (const queueItem of waitItems) { + if (!queueItem.hasCreatedEvent) { + ops.push( + (async () => { + const waitEvent: CreateEventRequest = { + eventType: 'wait_created' as const, + correlationId: queueItem.correlationId, + eventData: { + resumeAt: queueItem.resumeAt, + }, + }; + try { + await world.events.create(runId, waitEvent); + } catch (err) { + if (WorkflowAPIError.is(err) && err.status === 409) { + console.warn(`Wait already exists, continuing: ${err.message}`); + } else { + throw err; + } + } + })() + ); + } + } - // Find minimum timeout from waits - const minTimeoutSeconds = waitTimeouts.reduce( - (min, timeout) => { - if (timeout === null) return min; - if (min === null) return timeout; - return Math.min(min, timeout); + // Wait for all step and wait operations to complete + waitUntil( + Promise.all(ops).catch((opErr) => { + const isAbortError = + opErr?.name === 'AbortError' || opErr?.name === 'ResponseAborted'; + if (!isAbortError) throw opErr; + }) + ); + await Promise.all(ops); + + // Calculate minimum timeout from waits + const now = Date.now(); + const minTimeoutSeconds = waitItems.reduce( + (min, queueItem) => { + const resumeAtMs = queueItem.resumeAt.getTime(); + const delayMs = Math.max(1000, resumeAtMs - now); + const timeoutSeconds = Math.ceil(delayMs / 1000); + if (min === null) return timeoutSeconds; + return Math.min(min, timeoutSeconds); }, null ); @@ -273,7 +217,6 @@ export async function handleSuspension({ ...Attribute.WorkflowWaitsCreated(waitItems.length), }); - // If we encountered any waits, return the minimum timeout if (minTimeoutSeconds !== null) { return { timeoutSeconds: minTimeoutSeconds }; } diff --git a/packages/core/src/step.test.ts b/packages/core/src/step.test.ts index cd49e9c6c..8a3e129ca 100644 --- a/packages/core/src/step.test.ts +++ b/packages/core/src/step.test.ts @@ -59,7 +59,6 @@ describe('createUseStep', () => { correlationId: 'step_01K11TFZ62YS0YYFDQ3E8B9YCV', eventData: { error: 'test', - fatal: true, }, createdAt: new Date(), }, diff --git a/packages/core/src/step.ts b/packages/core/src/step.ts index f356f81f2..78af0930a 100644 --- a/packages/core/src/step.ts +++ b/packages/core/src/step.ts @@ -32,11 +32,6 @@ export function createUseStep(ctx: WorkflowOrchestratorContext) { ctx.invocationsQueue.set(correlationId, queueItem); - // Track whether we've already seen a "step_started" event for this step. - // This is important because after a retryable failure, the step moves back to - // "pending" status which causes another "step_started" event to be emitted. - let hasSeenStepStarted = false; - stepLogger.debug('Step consumer setup', { correlationId, stepName, @@ -70,43 +65,55 @@ export function createUseStep(ctx: WorkflowOrchestratorContext) { return EventConsumerResult.NotConsumed; } - if (event.eventType === 'step_started') { - // Step has started - so remove from the invocations queue (only on the first "step_started" event) - if (!hasSeenStepStarted) { - // O(1) lookup and delete using Map - if (ctx.invocationsQueue.has(correlationId)) { - ctx.invocationsQueue.delete(correlationId); - } else { - setTimeout(() => { - reject( - new WorkflowRuntimeError( - `Corrupted event log: step ${correlationId} (${stepName}) started but not found in invocation queue` - ) - ); - }, 0); - return EventConsumerResult.Finished; - } - hasSeenStepStarted = true; + if (event.eventType === 'step_created') { + // Step has been created (registered for execution) - mark as having event + // but keep in queue so suspension handler knows to queue execution without + // creating a duplicate step_created event + const queueItem = ctx.invocationsQueue.get(correlationId); + if (!queueItem || queueItem.type !== 'step') { + // This indicates event log corruption - step_created received + // but the step was never invoked in the workflow during replay. + setTimeout(() => { + reject( + new WorkflowRuntimeError( + `Corrupted event log: step ${correlationId} (${stepName}) created but not found in invocation queue` + ) + ); + }, 0); + return EventConsumerResult.Finished; } - // If this is a subsequent "step_started" event (after a retry), we just consume it - // without trying to remove from the queue again or logging a warning + queueItem.hasCreatedEvent = true; + // Continue waiting for step_started/step_completed/step_failed events + return EventConsumerResult.Consumed; + } + + if (event.eventType === 'step_started') { + // Step was started - don't do anything. The step is left in the invocationQueue which + // will allow it to be re-enqueued. We rely on the queue's idempotency to prevent it from + // actually being over enqueued. + return EventConsumerResult.Consumed; + } + + if (event.eventType === 'step_retrying') { + // Step is being retried - just consume the event and wait for next step_started return EventConsumerResult.Consumed; } if (event.eventType === 'step_failed') { + // Terminal state - we can remove the invocationQueue item + ctx.invocationsQueue.delete(event.correlationId); // Step failed - bubble up to workflow - if (event.eventData.fatal) { - setTimeout(() => { - reject(new FatalError(event.eventData.error)); - }, 0); - return EventConsumerResult.Finished; - } else { - // This is a retryable error, so nothing to do here, - // but we will consume the event - return EventConsumerResult.Consumed; - } - } else if (event.eventType === 'step_completed') { - // Step has already completed, so resolve the Promise with the cached result + setTimeout(() => { + reject(new FatalError(event.eventData.error)); + }, 0); + return EventConsumerResult.Finished; + } + + if (event.eventType === 'step_completed') { + // Terminal state - we can remove the invocationQueue item + ctx.invocationsQueue.delete(event.correlationId); + + // Step has completed, so resolve the Promise with the cached result const hydratedResult = hydrateStepReturnValue( event.eventData.result, ctx.globalThis @@ -115,17 +122,17 @@ export function createUseStep(ctx: WorkflowOrchestratorContext) { resolve(hydratedResult); }, 0); return EventConsumerResult.Finished; - } else { - // An unexpected event type has been received, but it does belong to this step (matching `correlationId`) - setTimeout(() => { - reject( - new WorkflowRuntimeError( - `Unexpected event type: "${event.eventType}"` - ) - ); - }, 0); - return EventConsumerResult.Finished; } + + // An unexpected event type has been received, but it does belong to this step (matching `correlationId`) + setTimeout(() => { + reject( + new WorkflowRuntimeError( + `Unexpected event type for step ${correlationId} (${stepName}) "${event.eventType}"` + ) + ); + }, 0); + return EventConsumerResult.Finished; }); return promise; diff --git a/packages/core/src/workflow.test.ts b/packages/core/src/workflow.test.ts index 6e7b53787..707bd200e 100644 --- a/packages/core/src/workflow.test.ts +++ b/packages/core/src/workflow.test.ts @@ -144,6 +144,7 @@ describe('runWorkflow', () => { expect(hydrateWorkflowReturnValue(result as any, ops)).toEqual(3); }); + // Test that timestamps update correctly as events are consumed it('should update the timestamp in the vm context as events are replayed', async () => { const ops: Promise[] = []; const workflowRunId = 'wrun_123'; @@ -158,7 +159,27 @@ describe('runWorkflow', () => { deploymentId: 'test-deployment', }; + // Events now include run_created, run_started, and step_created for proper consumption const events: Event[] = [ + { + eventId: 'event-run-created', + runId: workflowRunId, + eventType: 'run_created', + createdAt: new Date('2024-01-01T00:00:00.000Z'), + }, + { + eventId: 'event-run-started', + runId: workflowRunId, + eventType: 'run_started', + createdAt: new Date('2024-01-01T00:00:00.500Z'), + }, + { + eventId: 'event-step1-created', + runId: workflowRunId, + eventType: 'step_created', + correlationId: 'step_01HK153X00Y11PCQTCHQRK34HF', + createdAt: new Date('2024-01-01T00:00:00.600Z'), + }, { eventId: 'event-0', runId: workflowRunId, @@ -176,6 +197,13 @@ describe('runWorkflow', () => { }, createdAt: new Date('2024-01-01T00:00:02.000Z'), }, + { + eventId: 'event-step2-created', + runId: workflowRunId, + eventType: 'step_created', + correlationId: 'step_01HK153X00Y11PCQTCHQRK34HG', + createdAt: new Date('2024-01-01T00:00:02.500Z'), + }, { eventId: 'event-2', runId: workflowRunId, @@ -193,6 +221,13 @@ describe('runWorkflow', () => { }, createdAt: new Date('2024-01-01T00:00:04.000Z'), }, + { + eventId: 'event-step3-created', + runId: workflowRunId, + eventType: 'step_created', + correlationId: 'step_01HK153X00Y11PCQTCHQRK34HH', + createdAt: new Date('2024-01-01T00:00:04.500Z'), + }, { eventId: 'event-4', runId: workflowRunId, @@ -228,10 +263,15 @@ describe('runWorkflow', () => { workflowRun, events ); + // Timestamps: + // - Initial: 0s (from startedAt) + // - After step 1 completes (at 2s), timestamp advances to step2_created (2.5s) + // - After step 2 completes (at 4s), timestamp advances to step3_created (4.5s) + // - After step 3 completes: 6s expect(hydrateWorkflowReturnValue(result as any, ops)).toEqual([ new Date('2024-01-01T00:00:00.000Z'), - 1704067203000, - 1704067205000, + 1704067202500, // 2.5s (step2_created timestamp) + 1704067204500, // 4.5s (step3_created timestamp) new Date('2024-01-01T00:00:06.000Z'), ]); }); @@ -855,8 +895,9 @@ describe('runWorkflow', () => { } assert(error); expect(error.name).toEqual('WorkflowSuspension'); - expect(error.message).toEqual('0 steps have not been run yet'); - expect((error as WorkflowSuspension).steps).toEqual([]); + // step_started no longer removes from queue - step stays in queue for re-enqueueing + expect(error.message).toEqual('1 step has not been run yet'); + expect((error as WorkflowSuspension).steps).toHaveLength(1); }); it('should throw `WorkflowSuspension` for multiple steps with `Promise.all()`', async () => { diff --git a/packages/core/src/workflow.ts b/packages/core/src/workflow.ts index 6ecfba7fd..bd466d302 100644 --- a/packages/core/src/workflow.ts +++ b/packages/core/src/workflow.ts @@ -90,6 +90,27 @@ export async function runWorkflow( return EventConsumerResult.NotConsumed; }); + // Consume run lifecycle events - these are structural events that don't + // need special handling in the workflow, but must be consumed to advance + // past them in the event log + workflowContext.eventsConsumer.subscribe((event) => { + if (!event) { + return EventConsumerResult.NotConsumed; + } + + // Consume run_created - every run has exactly one + if (event.eventType === 'run_created') { + return EventConsumerResult.Consumed; + } + + // Consume run_started - every run has exactly one + if (event.eventType === 'run_started') { + return EventConsumerResult.Consumed; + } + + return EventConsumerResult.NotConsumed; + }); + const useStep = createUseStep(workflowContext); const createHook = createCreateHook(workflowContext); const sleep = createSleep(workflowContext); diff --git a/packages/web-shared/src/api/workflow-server-actions.ts b/packages/web-shared/src/api/workflow-server-actions.ts index ae6814cd1..ae07e86df 100644 --- a/packages/web-shared/src/api/workflow-server-actions.ts +++ b/packages/web-shared/src/api/workflow-server-actions.ts @@ -477,10 +477,12 @@ export async function cancelRun( ): Promise> { try { const world = getWorldFromEnv(worldEnv); - await world.runs.cancel(runId); + await world.events.create(runId, { eventType: 'run_cancelled' }); return createResponse(undefined); } catch (error) { - return createServerActionError(error, 'world.runs.cancel', { runId }); + return createServerActionError(error, 'world.events.create', { + runId, + }); } } diff --git a/packages/web-shared/src/trace-viewer/trace-viewer.module.css b/packages/web-shared/src/trace-viewer/trace-viewer.module.css index 676851713..36d25ffd9 100644 --- a/packages/web-shared/src/trace-viewer/trace-viewer.module.css +++ b/packages/web-shared/src/trace-viewer/trace-viewer.module.css @@ -1193,8 +1193,7 @@ --span-secondary: var(--ds-green-900); } -.spanCancelled, -.spanPaused { +.spanCancelled { --span-background: var(--ds-amber-200); --span-border: var(--ds-amber-500); --span-line: var(--ds-amber-400); diff --git a/packages/web-shared/src/workflow-traces/trace-colors.ts b/packages/web-shared/src/workflow-traces/trace-colors.ts index 05cacf929..aa1f2e017 100644 --- a/packages/web-shared/src/workflow-traces/trace-colors.ts +++ b/packages/web-shared/src/workflow-traces/trace-colors.ts @@ -26,8 +26,6 @@ function getStatusClassName( return styles.spanCompleted; case 'cancelled': return styles.spanCancelled; - case 'paused': - return styles.spanPaused; case 'failed': return styles.spanFailed; default: diff --git a/packages/web/src/components/display-utils/status-badge.tsx b/packages/web/src/components/display-utils/status-badge.tsx index 0c138469b..eaa0ad385 100644 --- a/packages/web/src/components/display-utils/status-badge.tsx +++ b/packages/web/src/components/display-utils/status-badge.tsx @@ -37,8 +37,6 @@ export function StatusBadge({ return 'bg-yellow-500'; case 'pending': return 'bg-gray-400'; - case 'paused': - return 'bg-orange-500'; default: return 'bg-gray-400'; } diff --git a/packages/web/src/components/flow-graph/workflow-graph-execution-viewer.tsx b/packages/web/src/components/flow-graph/workflow-graph-execution-viewer.tsx index 1d8462529..7facc6f26 100644 --- a/packages/web/src/components/flow-graph/workflow-graph-execution-viewer.tsx +++ b/packages/web/src/components/flow-graph/workflow-graph-execution-viewer.tsx @@ -50,8 +50,7 @@ type StatusBadgeStatus = | 'running' | 'completed' | 'failed' - | 'cancelled' - | 'paused'; + | 'cancelled'; function mapToStatusBadgeStatus( status: StepExecution['status'] ): StatusBadgeStatus { diff --git a/packages/web/src/components/runs-table.tsx b/packages/web/src/components/runs-table.tsx index ea4614fdd..fb744c513 100644 --- a/packages/web/src/components/runs-table.tsx +++ b/packages/web/src/components/runs-table.tsx @@ -166,7 +166,6 @@ const statusMap: Record = { running: { label: 'Running', color: 'bg-blue-600 dark:bg-blue-400' }, completed: { label: 'Completed', color: 'bg-green-600 dark:bg-green-400' }, failed: { label: 'Failed', color: 'bg-red-600 dark:bg-red-400' }, - paused: { label: 'Paused', color: 'bg-yellow-600 dark:bg-yellow-400' }, cancelled: { label: 'Cancelled', color: 'bg-gray-600 dark:bg-gray-400' }, }; diff --git a/packages/web/src/lib/flow-graph/graph-execution-mapper.ts b/packages/web/src/lib/flow-graph/graph-execution-mapper.ts index 6f0d33e54..9c05f5f15 100644 --- a/packages/web/src/lib/flow-graph/graph-execution-mapper.ts +++ b/packages/web/src/lib/flow-graph/graph-execution-mapper.ts @@ -188,7 +188,7 @@ function initializeStartNode( /** * Add end node execution based on workflow run status - * Handles all run statuses: pending, running, completed, failed, paused, cancelled + * Handles all run statuses: pending, running, completed, failed, cancelled */ function addEndNodeExecution( run: WorkflowRun, @@ -216,10 +216,6 @@ function addEndNodeExecution( case 'running': endNodeStatus = 'running'; break; - case 'paused': - // Paused is like running but waiting - endNodeStatus = 'pending'; - break; case 'pending': default: // Don't add end node for pending runs diff --git a/packages/web/src/lib/flow-graph/workflow-graph-types.ts b/packages/web/src/lib/flow-graph/workflow-graph-types.ts index 2976ddfc8..d807e4f8c 100644 --- a/packages/web/src/lib/flow-graph/workflow-graph-types.ts +++ b/packages/web/src/lib/flow-graph/workflow-graph-types.ts @@ -117,13 +117,7 @@ export interface EdgeTraversal { export interface WorkflowRunExecution { runId: string; - status: - | 'pending' - | 'running' - | 'completed' - | 'failed' - | 'paused' - | 'cancelled'; + status: 'pending' | 'running' | 'completed' | 'failed' | 'cancelled'; nodeExecutions: Map; // nodeId -> array of executions (for retries) edgeTraversals: Map; // edgeId -> traversal info currentNode?: string; // for running workflows diff --git a/packages/world-local/src/storage.test.ts b/packages/world-local/src/storage.test.ts index c74bb9842..da8067af0 100644 --- a/packages/world-local/src/storage.test.ts +++ b/packages/world-local/src/storage.test.ts @@ -1,7 +1,7 @@ import { promises as fs } from 'node:fs'; import os from 'node:os'; import path from 'node:path'; -import type { Storage } from '@workflow/world'; +import type { Storage, WorkflowRun, Step, Hook } from '@workflow/world'; import { monotonicFactory } from 'ulid'; import { EventSchema, @@ -12,6 +12,111 @@ import { import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { createStorage } from './storage.js'; +// Helper functions to create entities through events.create +async function createRun( + storage: Storage, + data: { + deploymentId: string; + workflowName: string; + input: unknown[]; + executionContext?: Record; + } +): Promise { + const result = await storage.events.create(null, { + eventType: 'run_created', + eventData: data, + }); + if (!result.run) { + throw new Error('Expected run to be created'); + } + return result.run; +} + +async function updateRun( + storage: Storage, + runId: string, + eventType: 'run_started' | 'run_completed' | 'run_failed', + eventData?: Record +): Promise { + const result = await storage.events.create(runId, { + eventType, + eventData, + }); + if (!result.run) { + throw new Error('Expected run to be updated'); + } + return result.run; +} + +async function createStep( + storage: Storage, + runId: string, + data: { + stepId: string; + stepName: string; + input: unknown[]; + } +): Promise { + const result = await storage.events.create(runId, { + eventType: 'step_created', + correlationId: data.stepId, + eventData: { stepName: data.stepName, input: data.input }, + }); + if (!result.step) { + throw new Error('Expected step to be created'); + } + return result.step; +} + +async function updateStep( + storage: Storage, + runId: string, + stepId: string, + eventType: 'step_started' | 'step_completed' | 'step_failed', + eventData?: Record +): Promise { + const result = await storage.events.create(runId, { + eventType, + correlationId: stepId, + eventData, + }); + if (!result.step) { + throw new Error('Expected step to be updated'); + } + return result.step; +} + +async function createHook( + storage: Storage, + runId: string, + data: { + hookId: string; + token: string; + metadata?: unknown; + } +): Promise { + const result = await storage.events.create(runId, { + eventType: 'hook_created', + correlationId: data.hookId, + eventData: { token: data.token, metadata: data.metadata }, + }); + if (!result.hook) { + throw new Error('Expected hook to be created'); + } + return result.hook; +} + +async function disposeHook( + storage: Storage, + runId: string, + hookId: string +): Promise { + await storage.events.create(runId, { + eventType: 'hook_disposed', + correlationId: hookId, + }); +} + describe('Storage', () => { let testDir: string; let storage: Storage; @@ -41,7 +146,7 @@ describe('Storage', () => { input: ['arg1', 'arg2'], }; - const run = await storage.runs.create(runData); + const run = await createRun(storage, runData); expect(run.runId).toMatch(/^wrun_/); expect(run.deploymentId).toBe('deployment-123'); @@ -72,37 +177,16 @@ describe('Storage', () => { input: [], }; - const run = await storage.runs.create(runData); + const run = await createRun(storage, runData); expect(run.executionContext).toBeUndefined(); expect(run.input).toEqual([]); }); - - it('should validate run against schema before writing', async () => { - const parseSpy = vi.spyOn(WorkflowRunSchema, 'parse'); - - await storage.runs.create({ - deploymentId: 'deployment-123', - workflowName: 'test-workflow', - input: [], - }); - - expect(parseSpy).toHaveBeenCalledTimes(1); - expect(parseSpy).toHaveBeenCalledWith( - expect.objectContaining({ - deploymentId: 'deployment-123', - workflowName: 'test-workflow', - status: 'pending', - }) - ); - - parseSpy.mockRestore(); - }); }); describe('get', () => { it('should retrieve an existing run', async () => { - const created = await storage.runs.create({ + const created = await createRun(storage, { deploymentId: 'deployment-123', workflowName: 'test-workflow', input: [], @@ -120,9 +204,9 @@ describe('Storage', () => { }); }); - describe('update', () => { - it('should update run status to running', async () => { - const created = await storage.runs.create({ + describe('update via events', () => { + it('should update run status to running via run_started event', async () => { + const created = await createRun(storage, { deploymentId: 'deployment-123', workflowName: 'test-workflow', input: [], @@ -131,9 +215,7 @@ describe('Storage', () => { // Small delay to ensure different timestamps await new Promise((resolve) => setTimeout(resolve, 1)); - const updated = await storage.runs.update(created.runId, { - status: 'running', - }); + const updated = await updateRun(storage, created.runId, 'run_started'); expect(updated.status).toBe('running'); expect(updated.startedAt).toBeInstanceOf(Date); @@ -142,56 +224,47 @@ describe('Storage', () => { ); }); - it('should update run status to completed', async () => { - const created = await storage.runs.create({ + it('should update run status to completed via run_completed event', async () => { + const created = await createRun(storage, { deploymentId: 'deployment-123', workflowName: 'test-workflow', input: [], }); - const updated = await storage.runs.update(created.runId, { - status: 'completed', - output: { result: 'success' }, - }); + const updated = await updateRun( + storage, + created.runId, + 'run_completed', + { + output: { result: 'success' }, + } + ); expect(updated.status).toBe('completed'); expect(updated.output).toEqual({ result: 'success' }); expect(updated.completedAt).toBeInstanceOf(Date); }); - it('should update run status to failed', async () => { - const created = await storage.runs.create({ + it('should update run status to failed via run_failed event', async () => { + const created = await createRun(storage, { deploymentId: 'deployment-123', workflowName: 'test-workflow', input: [], }); - const updated = await storage.runs.update(created.runId, { - status: 'failed', - error: { - message: 'Something went wrong', - code: 'ERR_001', - }, + const updated = await updateRun(storage, created.runId, 'run_failed', { + error: 'Something went wrong', }); expect(updated.status).toBe('failed'); - expect(updated.error).toEqual({ - message: 'Something went wrong', - code: 'ERR_001', - }); + expect(updated.error?.message).toBe('Something went wrong'); expect(updated.completedAt).toBeInstanceOf(Date); }); - - it('should throw error for non-existent run', async () => { - await expect( - storage.runs.update('wrun_nonexistent', { status: 'running' }) - ).rejects.toThrow('Workflow run "wrun_nonexistent" not found'); - }); }); describe('list', () => { it('should list all runs', async () => { - const run1 = await storage.runs.create({ + const run1 = await createRun(storage, { deploymentId: 'deployment-1', workflowName: 'workflow-1', input: [], @@ -200,7 +273,7 @@ describe('Storage', () => { // Small delay to ensure different timestamps in ULIDs await new Promise((resolve) => setTimeout(resolve, 2)); - const run2 = await storage.runs.create({ + const run2 = await createRun(storage, { deploymentId: 'deployment-2', workflowName: 'workflow-2', input: [], @@ -218,12 +291,12 @@ describe('Storage', () => { }); it('should filter runs by workflowName', async () => { - await storage.runs.create({ + await createRun(storage, { deploymentId: 'deployment-1', workflowName: 'workflow-1', input: [], }); - const run2 = await storage.runs.create({ + const run2 = await createRun(storage, { deploymentId: 'deployment-2', workflowName: 'workflow-2', input: [], @@ -238,7 +311,7 @@ describe('Storage', () => { it('should support pagination', async () => { // Create multiple runs for (let i = 0; i < 5; i++) { - await storage.runs.create({ + await createRun(storage, { deploymentId: `deployment-${i}`, workflowName: `workflow-${i}`, input: [], @@ -260,58 +333,13 @@ describe('Storage', () => { expect(page2.data[0].runId).not.toBe(page1.data[0].runId); }); }); - - describe('cancel', () => { - it('should cancel a run', async () => { - const created = await storage.runs.create({ - deploymentId: 'deployment-123', - workflowName: 'test-workflow', - input: [], - }); - - const cancelled = await storage.runs.cancel(created.runId); - - expect(cancelled.status).toBe('cancelled'); - expect(cancelled.completedAt).toBeInstanceOf(Date); - }); - }); - - describe('pause', () => { - it('should pause a run', async () => { - const created = await storage.runs.create({ - deploymentId: 'deployment-123', - workflowName: 'test-workflow', - input: [], - }); - - const paused = await storage.runs.pause(created.runId); - - expect(paused.status).toBe('paused'); - }); - }); - - describe('resume', () => { - it('should resume a paused run', async () => { - const created = await storage.runs.create({ - deploymentId: 'deployment-123', - workflowName: 'test-workflow', - input: [], - }); - - await storage.runs.pause(created.runId); - const resumed = await storage.runs.resume(created.runId); - - expect(resumed.status).toBe('running'); - expect(resumed.startedAt).toBeInstanceOf(Date); - }); - }); }); describe('steps', () => { let testRunId: string; beforeEach(async () => { - const run = await storage.runs.create({ + const run = await createRun(storage, { deploymentId: 'deployment-123', workflowName: 'test-workflow', input: [], @@ -327,7 +355,7 @@ describe('Storage', () => { input: ['input1', 'input2'], }; - const step = await storage.steps.create(testRunId, stepData); + const step = await createStep(storage, testRunId, stepData); expect(step.runId).toBe(testRunId); expect(step.stepId).toBe('step_123'); @@ -354,33 +382,11 @@ describe('Storage', () => { .catch(() => false); expect(fileExists).toBe(true); }); - - it('should validate step against schema before writing', async () => { - const parseSpy = vi.spyOn(StepSchema, 'parse'); - - await storage.steps.create(testRunId, { - stepId: 'step_validated', - stepName: 'validated-step', - input: ['arg1'], - }); - - expect(parseSpy).toHaveBeenCalledTimes(1); - expect(parseSpy).toHaveBeenCalledWith( - expect.objectContaining({ - runId: testRunId, - stepId: 'step_validated', - stepName: 'validated-step', - status: 'pending', - }) - ); - - parseSpy.mockRestore(); - }); }); describe('get', () => { it('should retrieve a step with runId and stepId', async () => { - const created = await storage.steps.create(testRunId, { + const created = await createStep(storage, testRunId, { stepId: 'step_123', stepName: 'test-step', input: ['input1'], @@ -392,7 +398,7 @@ describe('Storage', () => { }); it('should retrieve a step with only stepId', async () => { - const created = await storage.steps.create(testRunId, { + const created = await createStep(storage, testRunId, { stepId: 'unique_step_123', stepName: 'test-step', input: ['input1'], @@ -410,83 +416,76 @@ describe('Storage', () => { }); }); - describe('update', () => { - it('should update step status to running', async () => { - await storage.steps.create(testRunId, { + describe('update via events', () => { + it('should update step status to running via step_started event', async () => { + await createStep(storage, testRunId, { stepId: 'step_123', stepName: 'test-step', input: ['input1'], }); - const updated = await storage.steps.update(testRunId, 'step_123', { - status: 'running', - }); + const updated = await updateStep( + storage, + testRunId, + 'step_123', + 'step_started', + {} // step_started no longer needs attempt in eventData - World increments it + ); expect(updated.status).toBe('running'); expect(updated.startedAt).toBeInstanceOf(Date); + expect(updated.attempt).toBe(1); // Incremented by step_started }); - it('should update step status to completed', async () => { - await storage.steps.create(testRunId, { + it('should update step status to completed via step_completed event', async () => { + await createStep(storage, testRunId, { stepId: 'step_123', stepName: 'test-step', input: ['input1'], }); - const updated = await storage.steps.update(testRunId, 'step_123', { - status: 'completed', - output: { result: 'done' }, - }); + const updated = await updateStep( + storage, + testRunId, + 'step_123', + 'step_completed', + { result: { result: 'done' } } + ); expect(updated.status).toBe('completed'); expect(updated.output).toEqual({ result: 'done' }); expect(updated.completedAt).toBeInstanceOf(Date); }); - it('should update step status to failed', async () => { - await storage.steps.create(testRunId, { + it('should update step status to failed via step_failed event', async () => { + await createStep(storage, testRunId, { stepId: 'step_123', stepName: 'test-step', input: ['input1'], }); - const updated = await storage.steps.update(testRunId, 'step_123', { - status: 'failed', - error: { - message: 'Step failed', - code: 'STEP_ERR', - }, - }); + const updated = await updateStep( + storage, + testRunId, + 'step_123', + 'step_failed', + { error: 'Step failed' } + ); expect(updated.status).toBe('failed'); expect(updated.error?.message).toBe('Step failed'); - expect(updated.error?.code).toBe('STEP_ERR'); expect(updated.completedAt).toBeInstanceOf(Date); }); - - it('should update attempt count', async () => { - await storage.steps.create(testRunId, { - stepId: 'step_123', - stepName: 'test-step', - input: ['input1'], - }); - - const updated = await storage.steps.update(testRunId, 'step_123', { - attempt: 2, - }); - - expect(updated.attempt).toBe(2); - }); }); describe('list', () => { it('should list all steps for a run', async () => { - const step1 = await storage.steps.create(testRunId, { + const step1 = await createStep(storage, testRunId, { stepId: 'step_1', stepName: 'first-step', input: [], }); - const step2 = await storage.steps.create(testRunId, { + const step2 = await createStep(storage, testRunId, { stepId: 'step_2', stepName: 'second-step', input: [], @@ -508,7 +507,7 @@ describe('Storage', () => { it('should support pagination', async () => { // Create multiple steps for (let i = 0; i < 5; i++) { - await storage.steps.create(testRunId, { + await createStep(storage, testRunId, { stepId: `step_${i}`, stepName: `step-${i}`, input: [], @@ -535,7 +534,7 @@ describe('Storage', () => { it('should handle pagination when new items are created after getting a cursor', async () => { // Create initial set of items (4 items) for (let i = 0; i < 4; i++) { - await storage.steps.create(testRunId, { + await createStep(storage, testRunId, { stepId: `step_${i}`, stepName: `step-${i}`, input: [], @@ -555,7 +554,7 @@ describe('Storage', () => { // Now create 4 more items (total: 8 items) for (let i = 4; i < 8; i++) { - await storage.steps.create(testRunId, { + await createStep(storage, testRunId, { stepId: `step_${i}`, stepName: `step-${i}`, input: [], @@ -595,7 +594,7 @@ describe('Storage', () => { it('should handle pagination with cursor after items are added mid-pagination', async () => { // Create initial 4 items for (let i = 0; i < 4; i++) { - await storage.steps.create(testRunId, { + await createStep(storage, testRunId, { stepId: `step_${i}`, stepName: `step-${i}`, input: [], @@ -627,7 +626,7 @@ describe('Storage', () => { // Now add 4 more items (total: 8) for (let i = 4; i < 8; i++) { - await storage.steps.create(testRunId, { + await createStep(storage, testRunId, { stepId: `step_${i}`, stepName: `step-${i}`, input: [], @@ -668,7 +667,7 @@ describe('Storage', () => { // Start with X items (4 items) for (let i = 0; i < 4; i++) { - await storage.steps.create(testRunId, { + await createStep(storage, testRunId, { stepId: `step_${i}`, stepName: `step-${i}`, input: [], @@ -690,7 +689,7 @@ describe('Storage', () => { // Create new items (total becomes 2X = 8 items) for (let i = 4; i < 8; i++) { - await storage.steps.create(testRunId, { + await createStep(storage, testRunId, { stepId: `step_${i}`, stepName: `step-${i}`, input: [], @@ -742,7 +741,7 @@ describe('Storage', () => { let testRunId: string; beforeEach(async () => { - const run = await storage.runs.create({ + const run = await createRun(storage, { deploymentId: 'deployment-123', workflowName: 'test-workflow', input: [], @@ -752,12 +751,19 @@ describe('Storage', () => { describe('create', () => { it('should create a new event', async () => { + // Create step first (required for step events) + await createStep(storage, testRunId, { + stepId: 'corr_123', + stepName: 'test-step', + input: [], + }); + const eventData = { eventType: 'step_started' as const, correlationId: 'corr_123', }; - const event = await storage.events.create(testRunId, eventData); + const { event } = await storage.events.create(testRunId, eventData); expect(event.runId).toBe(testRunId); expect(event.eventId).toMatch(/^evnt_/); @@ -783,43 +789,33 @@ describe('Storage', () => { eventType: 'workflow_completed' as const, }; - const event = await storage.events.create(testRunId, eventData); + const { event } = await storage.events.create(testRunId, eventData); expect(event.eventType).toBe('workflow_completed'); expect(event.correlationId).toBeUndefined(); }); - - it('should validate event against schema before writing', async () => { - const parseSpy = vi.spyOn(EventSchema, 'parse'); - - await storage.events.create(testRunId, { - eventType: 'step_started' as const, - correlationId: 'corr_validated', - }); - - expect(parseSpy).toHaveBeenCalledTimes(1); - expect(parseSpy).toHaveBeenCalledWith( - expect.objectContaining({ - runId: testRunId, - eventType: 'step_started', - correlationId: 'corr_validated', - }) - ); - - parseSpy.mockRestore(); - }); }); describe('list', () => { it('should list all events for a run', async () => { - const event1 = await storage.events.create(testRunId, { + // Note: testRunId was created via createRun which creates a run_created event + const { event: event1 } = await storage.events.create(testRunId, { eventType: 'workflow_started' as const, }); // Small delay to ensure different timestamps in event IDs await new Promise((resolve) => setTimeout(resolve, 2)); - const event2 = await storage.events.create(testRunId, { + // Create the step first (required for step events) + await createStep(storage, testRunId, { + stepId: 'corr_step_1', + stepName: 'test-step', + input: [], + }); + + await new Promise((resolve) => setTimeout(resolve, 2)); + + const { event: event2 } = await storage.events.create(testRunId, { eventType: 'step_started' as const, correlationId: 'corr_step_1', }); @@ -829,24 +825,37 @@ describe('Storage', () => { pagination: { sortOrder: 'asc' }, // Explicitly request ascending order }); - expect(result.data).toHaveLength(2); + // 4 events: run_created (from createRun), workflow_started, step_created, step_started + expect(result.data).toHaveLength(4); // Should be in chronological order (oldest first) - expect(result.data[0].eventId).toBe(event1.eventId); - expect(result.data[1].eventId).toBe(event2.eventId); - expect(result.data[1].createdAt.getTime()).toBeGreaterThanOrEqual( - result.data[0].createdAt.getTime() + expect(result.data[0].eventType).toBe('run_created'); + expect(result.data[1].eventId).toBe(event1.eventId); + expect(result.data[2].eventType).toBe('step_created'); + expect(result.data[3].eventId).toBe(event2.eventId); + expect(result.data[3].createdAt.getTime()).toBeGreaterThanOrEqual( + result.data[2].createdAt.getTime() ); }); it('should list events in descending order when explicitly requested (newest first)', async () => { - const event1 = await storage.events.create(testRunId, { + // Note: testRunId was created via createRun which creates a run_created event + const { event: event1 } = await storage.events.create(testRunId, { eventType: 'workflow_started' as const, }); // Small delay to ensure different timestamps in event IDs await new Promise((resolve) => setTimeout(resolve, 2)); - const event2 = await storage.events.create(testRunId, { + // Create the step first (required for step events) + await createStep(storage, testRunId, { + stepId: 'corr_step_1', + stepName: 'test-step', + input: [], + }); + + await new Promise((resolve) => setTimeout(resolve, 2)); + + const { event: event2 } = await storage.events.create(testRunId, { eventType: 'step_started' as const, correlationId: 'corr_step_1', }); @@ -856,18 +865,26 @@ describe('Storage', () => { pagination: { sortOrder: 'desc' }, }); - expect(result.data).toHaveLength(2); + // 4 events: run_created (from createRun), workflow_started, step_created, step_started + expect(result.data).toHaveLength(4); // Should be in reverse chronological order (newest first) expect(result.data[0].eventId).toBe(event2.eventId); - expect(result.data[1].eventId).toBe(event1.eventId); + expect(result.data[1].eventType).toBe('step_created'); + expect(result.data[2].eventId).toBe(event1.eventId); + expect(result.data[3].eventType).toBe('run_created'); expect(result.data[0].createdAt.getTime()).toBeGreaterThanOrEqual( result.data[1].createdAt.getTime() ); }); it('should support pagination', async () => { - // Create multiple events + // Create steps first, then create step_completed events for (let i = 0; i < 5; i++) { + await createStep(storage, testRunId, { + stepId: `corr_${i}`, + stepName: `step-${i}`, + input: [], + }); await storage.events.create(testRunId, { eventType: 'step_completed' as const, correlationId: `corr_${i}`, @@ -897,15 +914,29 @@ describe('Storage', () => { it('should list all events with a specific correlation ID', async () => { const correlationId = 'step-abc123'; + // Create the step first (required for step events) + await createStep(storage, testRunId, { + stepId: correlationId, + stepName: 'test-step', + input: [], + }); + + // Create step for the different correlation ID too + await createStep(storage, testRunId, { + stepId: 'different-step', + stepName: 'different-step', + input: [], + }); + // Create events with the target correlation ID - const event1 = await storage.events.create(testRunId, { + const { event: event1 } = await storage.events.create(testRunId, { eventType: 'step_started' as const, correlationId, }); await new Promise((resolve) => setTimeout(resolve, 2)); - const event2 = await storage.events.create(testRunId, { + const { event: event2 } = await storage.events.create(testRunId, { eventType: 'step_completed' as const, correlationId, eventData: { result: 'success' }, @@ -925,32 +956,37 @@ describe('Storage', () => { pagination: {}, }); - expect(result.data).toHaveLength(2); - expect(result.data[0].eventId).toBe(event1.eventId); + // step_created + step_started + step_completed = 3 events + expect(result.data).toHaveLength(3); + // First event is step_created from createStep + expect(result.data[0].eventType).toBe('step_created'); expect(result.data[0].correlationId).toBe(correlationId); - expect(result.data[1].eventId).toBe(event2.eventId); + expect(result.data[1].eventId).toBe(event1.eventId); expect(result.data[1].correlationId).toBe(correlationId); + expect(result.data[2].eventId).toBe(event2.eventId); + expect(result.data[2].correlationId).toBe(correlationId); }); it('should list events across multiple runs with same correlation ID', async () => { const correlationId = 'hook-xyz789'; // Create another run - const run2 = await storage.runs.create({ + const run2 = await createRun(storage, { deploymentId: 'deployment-456', workflowName: 'test-workflow-2', input: [], }); // Create events in both runs with same correlation ID - const event1 = await storage.events.create(testRunId, { + const { event: event1 } = await storage.events.create(testRunId, { eventType: 'hook_created' as const, correlationId, + eventData: { token: `test-token-${correlationId}`, metadata: {} }, }); await new Promise((resolve) => setTimeout(resolve, 2)); - const event2 = await storage.events.create(run2.runId, { + const { event: event2 } = await storage.events.create(run2.runId, { eventType: 'hook_received' as const, correlationId, eventData: { payload: { data: 'test' } }, @@ -958,7 +994,7 @@ describe('Storage', () => { await new Promise((resolve) => setTimeout(resolve, 2)); - const event3 = await storage.events.create(testRunId, { + const { event: event3 } = await storage.events.create(testRunId, { eventType: 'hook_disposed' as const, correlationId, }); @@ -978,6 +1014,13 @@ describe('Storage', () => { }); it('should return empty list for non-existent correlation ID', async () => { + // Create the step first (required for step events) + await createStep(storage, testRunId, { + stepId: 'existing-step', + stepName: 'existing-step', + input: [], + }); + await storage.events.create(testRunId, { eventType: 'step_started' as const, correlationId: 'existing-step', @@ -996,6 +1039,13 @@ describe('Storage', () => { it('should respect pagination parameters', async () => { const correlationId = 'step-paginated'; + // Create the step first (required for step events) + await createStep(storage, testRunId, { + stepId: correlationId, + stepName: 'test-step', + input: [], + }); + // Create multiple events await storage.events.create(testRunId, { eventType: 'step_started' as const, @@ -1007,7 +1057,14 @@ describe('Storage', () => { await storage.events.create(testRunId, { eventType: 'step_retrying' as const, correlationId, - eventData: { attempt: 1 }, + eventData: { error: 'retry error' }, + }); + + await new Promise((resolve) => setTimeout(resolve, 2)); + + await storage.events.create(testRunId, { + eventType: 'step_started' as const, + correlationId, }); await new Promise((resolve) => setTimeout(resolve, 2)); @@ -1018,7 +1075,7 @@ describe('Storage', () => { eventData: { result: 'success' }, }); - // Get first page + // Get first page (step_created + step_started = 2) const page1 = await storage.events.listByCorrelationId({ correlationId, pagination: { limit: 2 }, @@ -1028,19 +1085,26 @@ describe('Storage', () => { expect(page1.hasMore).toBe(true); expect(page1.cursor).toBeDefined(); - // Get second page + // Get second page (step_retrying + step_started + step_completed = 3) const page2 = await storage.events.listByCorrelationId({ correlationId, - pagination: { limit: 2, cursor: page1.cursor || undefined }, + pagination: { limit: 3, cursor: page1.cursor || undefined }, }); - expect(page2.data).toHaveLength(1); + expect(page2.data).toHaveLength(3); expect(page2.hasMore).toBe(false); }); it('should filter event data when resolveData is "none"', async () => { const correlationId = 'step-with-data'; + // Create the step first (required for step events) + await createStep(storage, testRunId, { + stepId: correlationId, + stepName: 'test-step', + input: [], + }); + await storage.events.create(testRunId, { eventType: 'step_completed' as const, correlationId, @@ -1053,23 +1117,34 @@ describe('Storage', () => { resolveData: 'none', }); - expect(result.data).toHaveLength(1); + // step_created + step_completed = 2 events + expect(result.data).toHaveLength(2); expect((result.data[0] as any).eventData).toBeUndefined(); + expect((result.data[1] as any).eventData).toBeUndefined(); expect(result.data[0].correlationId).toBe(correlationId); }); it('should return events in ascending order by default', async () => { const correlationId = 'step-ordering'; + // Create the step first (required for step events) + await createStep(storage, testRunId, { + stepId: correlationId, + stepName: 'test-step', + input: [], + }); + + await new Promise((resolve) => setTimeout(resolve, 2)); + // Create events with slight delays to ensure different timestamps - const event1 = await storage.events.create(testRunId, { + const { event: event1 } = await storage.events.create(testRunId, { eventType: 'step_started' as const, correlationId, }); await new Promise((resolve) => setTimeout(resolve, 2)); - const event2 = await storage.events.create(testRunId, { + const { event: event2 } = await storage.events.create(testRunId, { eventType: 'step_completed' as const, correlationId, eventData: { result: 'success' }, @@ -1080,9 +1155,12 @@ describe('Storage', () => { pagination: {}, }); - expect(result.data).toHaveLength(2); - expect(result.data[0].eventId).toBe(event1.eventId); - expect(result.data[1].eventId).toBe(event2.eventId); + // step_created + step_started + step_completed = 3 events + expect(result.data).toHaveLength(3); + // Verify order: step_created, step_started, step_completed + expect(result.data[0].eventType).toBe('step_created'); + expect(result.data[1].eventId).toBe(event1.eventId); + expect(result.data[2].eventId).toBe(event2.eventId); expect(result.data[0].createdAt.getTime()).toBeLessThanOrEqual( result.data[1].createdAt.getTime() ); @@ -1091,14 +1169,23 @@ describe('Storage', () => { it('should support descending order', async () => { const correlationId = 'step-desc-order'; - const event1 = await storage.events.create(testRunId, { + // Create the step first (required for step events) + await createStep(storage, testRunId, { + stepId: correlationId, + stepName: 'test-step', + input: [], + }); + + await new Promise((resolve) => setTimeout(resolve, 2)); + + const { event: event1 } = await storage.events.create(testRunId, { eventType: 'step_started' as const, correlationId, }); await new Promise((resolve) => setTimeout(resolve, 2)); - const event2 = await storage.events.create(testRunId, { + const { event: event2 } = await storage.events.create(testRunId, { eventType: 'step_completed' as const, correlationId, eventData: { result: 'success' }, @@ -1109,9 +1196,12 @@ describe('Storage', () => { pagination: { sortOrder: 'desc' }, }); - expect(result.data).toHaveLength(2); + // step_created + step_started + step_completed = 3 events + expect(result.data).toHaveLength(3); + // Verify order: step_completed, step_started, step_created (descending) expect(result.data[0].eventId).toBe(event2.eventId); expect(result.data[1].eventId).toBe(event1.eventId); + expect(result.data[2].eventType).toBe('step_created'); expect(result.data[0].createdAt.getTime()).toBeGreaterThanOrEqual( result.data[1].createdAt.getTime() ); @@ -1121,14 +1211,15 @@ describe('Storage', () => { const hookId = 'hook_test123'; // Create a typical hook lifecycle - const created = await storage.events.create(testRunId, { + const { event: created } = await storage.events.create(testRunId, { eventType: 'hook_created' as const, correlationId: hookId, + eventData: { token: `test-token-${hookId}`, metadata: {} }, }); await new Promise((resolve) => setTimeout(resolve, 2)); - const received1 = await storage.events.create(testRunId, { + const { event: received1 } = await storage.events.create(testRunId, { eventType: 'hook_received' as const, correlationId: hookId, eventData: { payload: { request: 1 } }, @@ -1136,7 +1227,7 @@ describe('Storage', () => { await new Promise((resolve) => setTimeout(resolve, 2)); - const received2 = await storage.events.create(testRunId, { + const { event: received2 } = await storage.events.create(testRunId, { eventType: 'hook_received' as const, correlationId: hookId, eventData: { payload: { request: 2 } }, @@ -1144,7 +1235,7 @@ describe('Storage', () => { await new Promise((resolve) => setTimeout(resolve, 2)); - const disposed = await storage.events.create(testRunId, { + const { event: disposed } = await storage.events.create(testRunId, { eventType: 'hook_disposed' as const, correlationId: hookId, }); @@ -1171,7 +1262,7 @@ describe('Storage', () => { let testRunId: string; beforeEach(async () => { - const run = await storage.runs.create({ + const run = await createRun(storage, { deploymentId: 'deployment-123', workflowName: 'test-workflow', input: [], @@ -1186,14 +1277,11 @@ describe('Storage', () => { token: 'my-hook-token', }; - const hook = await storage.hooks.create(testRunId, hookData); + const hook = await createHook(storage, testRunId, hookData); expect(hook.runId).toBe(testRunId); expect(hook.hookId).toBe('hook_123'); expect(hook.token).toBe('my-hook-token'); - expect(hook.ownerId).toBe('local-owner'); - expect(hook.projectId).toBe('local-project'); - expect(hook.environment).toBe('local'); expect(hook.createdAt).toBeInstanceOf(Date); // Verify file was created @@ -1212,7 +1300,7 @@ describe('Storage', () => { token: 'duplicate-test-token', }; - await storage.hooks.create(testRunId, hookData); + await createHook(storage, testRunId, hookData); // Try to create another hook with the same token const duplicateHookData = { @@ -1221,19 +1309,19 @@ describe('Storage', () => { }; await expect( - storage.hooks.create(testRunId, duplicateHookData) + createHook(storage, testRunId, duplicateHookData) ).rejects.toThrow( 'Hook with token duplicate-test-token already exists for this project' ); }); it('should allow multiple hooks with different tokens for the same run', async () => { - const hook1 = await storage.hooks.create(testRunId, { + const hook1 = await createHook(storage, testRunId, { hookId: 'hook_1', token: 'token-1', }); - const hook2 = await storage.hooks.create(testRunId, { + const hook2 = await createHook(storage, testRunId, { hookId: 'hook_2', token: 'token-2', }); @@ -1246,7 +1334,7 @@ describe('Storage', () => { const token = 'reusable-token'; // Create first hook - const hook1 = await storage.hooks.create(testRunId, { + const hook1 = await createHook(storage, testRunId, { hookId: 'hook_1', token, }); @@ -1255,7 +1343,7 @@ describe('Storage', () => { // Try to create another hook with the same token - should fail await expect( - storage.hooks.create(testRunId, { + createHook(storage, testRunId, { hookId: 'hook_2', token, }) @@ -1263,11 +1351,11 @@ describe('Storage', () => { `Hook with token ${token} already exists for this project` ); - // Dispose the first hook - await storage.hooks.dispose('hook_1'); + // Dispose the first hook via hook_disposed event + await disposeHook(storage, testRunId, 'hook_1'); // Now we should be able to create a new hook with the same token - const hook2 = await storage.hooks.create(testRunId, { + const hook2 = await createHook(storage, testRunId, { hookId: 'hook_2', token, }); @@ -1278,7 +1366,7 @@ describe('Storage', () => { it('should enforce token uniqueness across different runs within the same project', async () => { // Create a second run - const run2 = await storage.runs.create({ + const run2 = await createRun(storage, { deploymentId: 'deployment-456', workflowName: 'another-workflow', input: [], @@ -1287,7 +1375,7 @@ describe('Storage', () => { const token = 'shared-token-across-runs'; // Create hook in first run - const hook1 = await storage.hooks.create(testRunId, { + const hook1 = await createHook(storage, testRunId, { hookId: 'hook_1', token, }); @@ -1296,7 +1384,7 @@ describe('Storage', () => { // Try to create hook with same token in second run - should fail await expect( - storage.hooks.create(run2.runId, { + createHook(storage, run2.runId, { hookId: 'hook_2', token, }) @@ -1304,31 +1392,11 @@ describe('Storage', () => { `Hook with token ${token} already exists for this project` ); }); - - it('should validate hook against schema before writing', async () => { - const parseSpy = vi.spyOn(HookSchema, 'parse'); - - await storage.hooks.create(testRunId, { - hookId: 'hook_validated', - token: 'validated-token', - }); - - expect(parseSpy).toHaveBeenCalledTimes(1); - expect(parseSpy).toHaveBeenCalledWith( - expect.objectContaining({ - runId: testRunId, - hookId: 'hook_validated', - token: 'validated-token', - }) - ); - - parseSpy.mockRestore(); - }); }); describe('get', () => { it('should retrieve an existing hook by hookId', async () => { - const created = await storage.hooks.create(testRunId, { + const created = await createHook(storage, testRunId, { hookId: 'hook_123', token: 'test-token-123', }); @@ -1345,7 +1413,7 @@ describe('Storage', () => { }); it('should respect resolveData option', async () => { - const created = await storage.hooks.create(testRunId, { + const created = await createHook(storage, testRunId, { hookId: 'hook_with_response', token: 'test-token', }); @@ -1367,7 +1435,7 @@ describe('Storage', () => { describe('getByToken', () => { it('should retrieve an existing hook by token', async () => { - const created = await storage.hooks.create(testRunId, { + const created = await createHook(storage, testRunId, { hookId: 'hook_123', token: 'test-token-123', }); @@ -1384,15 +1452,15 @@ describe('Storage', () => { }); it('should find the correct hook when multiple hooks exist', async () => { - const hook1 = await storage.hooks.create(testRunId, { + const hook1 = await createHook(storage, testRunId, { hookId: 'hook_1', token: 'token-1', }); - await storage.hooks.create(testRunId, { + await createHook(storage, testRunId, { hookId: 'hook_2', token: 'token-2', }); - await storage.hooks.create(testRunId, { + await createHook(storage, testRunId, { hookId: 'hook_3', token: 'token-3', }); @@ -1406,7 +1474,7 @@ describe('Storage', () => { describe('list', () => { it('should list all hooks', async () => { - const hook1 = await storage.hooks.create(testRunId, { + const hook1 = await createHook(storage, testRunId, { hookId: 'hook_1', token: 'token-1', }); @@ -1414,7 +1482,7 @@ describe('Storage', () => { // Small delay to ensure different timestamps await new Promise((resolve) => setTimeout(resolve, 2)); - const hook2 = await storage.hooks.create(testRunId, { + const hook2 = await createHook(storage, testRunId, { hookId: 'hook_2', token: 'token-2', }); @@ -1432,17 +1500,17 @@ describe('Storage', () => { it('should filter hooks by runId', async () => { // Create a second run - const run2 = await storage.runs.create({ + const run2 = await createRun(storage, { deploymentId: 'deployment-456', workflowName: 'test-workflow-2', input: [], }); - await storage.hooks.create(testRunId, { + await createHook(storage, testRunId, { hookId: 'hook_run1', token: 'token-run1', }); - const hook2 = await storage.hooks.create(run2.runId, { + const hook2 = await createHook(storage, run2.runId, { hookId: 'hook_run2', token: 'token-run2', }); @@ -1457,7 +1525,7 @@ describe('Storage', () => { it('should support pagination', async () => { // Create multiple hooks for (let i = 0; i < 5; i++) { - await storage.hooks.create(testRunId, { + await createHook(storage, testRunId, { hookId: `hook_${i}`, token: `token-${i}`, }); @@ -1480,14 +1548,14 @@ describe('Storage', () => { }); it('should support ascending sort order', async () => { - const hook1 = await storage.hooks.create(testRunId, { + const hook1 = await createHook(storage, testRunId, { hookId: 'hook_1', token: 'token-1', }); await new Promise((resolve) => setTimeout(resolve, 2)); - const hook2 = await storage.hooks.create(testRunId, { + const hook2 = await createHook(storage, testRunId, { hookId: 'hook_2', token: 'token-2', }); @@ -1503,7 +1571,7 @@ describe('Storage', () => { }); it('should respect resolveData option', async () => { - await storage.hooks.create(testRunId, { + await createHook(storage, testRunId, { hookId: 'hook_with_response', token: 'token-with-response', }); @@ -1531,29 +1599,825 @@ describe('Storage', () => { expect(result.hasMore).toBe(false); }); }); + }); + + describe('step terminal state validation', () => { + let testRunId: string; + + beforeEach(async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + testRunId = run.runId; + }); + + describe('completed step', () => { + it('should reject step_started on completed step', async () => { + await createStep(storage, testRunId, { + stepId: 'step_terminal_1', + stepName: 'test-step', + input: [], + }); + await updateStep( + storage, + testRunId, + 'step_terminal_1', + 'step_completed', + { + result: 'done', + } + ); + + await expect( + updateStep(storage, testRunId, 'step_terminal_1', 'step_started') + ).rejects.toThrow(/terminal/i); + }); + + it('should reject step_completed on already completed step', async () => { + await createStep(storage, testRunId, { + stepId: 'step_terminal_2', + stepName: 'test-step', + input: [], + }); + await updateStep( + storage, + testRunId, + 'step_terminal_2', + 'step_completed', + { + result: 'done', + } + ); + + await expect( + updateStep(storage, testRunId, 'step_terminal_2', 'step_completed', { + result: 'done again', + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject step_failed on completed step', async () => { + await createStep(storage, testRunId, { + stepId: 'step_terminal_3', + stepName: 'test-step', + input: [], + }); + await updateStep( + storage, + testRunId, + 'step_terminal_3', + 'step_completed', + { + result: 'done', + } + ); + + await expect( + updateStep(storage, testRunId, 'step_terminal_3', 'step_failed', { + error: 'Should not work', + }) + ).rejects.toThrow(/terminal/i); + }); + }); - describe('dispose', () => { - it('should delete an existing hook', async () => { - const created = await storage.hooks.create(testRunId, { - hookId: 'hook_to_delete', - token: 'token-to-delete', + describe('failed step', () => { + it('should reject step_started on failed step', async () => { + await createStep(storage, testRunId, { + stepId: 'step_failed_1', + stepName: 'test-step', + input: [], + }); + await updateStep(storage, testRunId, 'step_failed_1', 'step_failed', { + error: 'Failed permanently', }); - const disposed = await storage.hooks.dispose('hook_to_delete'); + await expect( + updateStep(storage, testRunId, 'step_failed_1', 'step_started') + ).rejects.toThrow(/terminal/i); + }); - expect(disposed).toEqual(created); + it('should reject step_completed on failed step', async () => { + await createStep(storage, testRunId, { + stepId: 'step_failed_2', + stepName: 'test-step', + input: [], + }); + await updateStep(storage, testRunId, 'step_failed_2', 'step_failed', { + error: 'Failed permanently', + }); - // Verify file was deleted await expect( - storage.hooks.getByToken('token-to-delete') - ).rejects.toThrow('Hook with token token-to-delete not found'); + updateStep(storage, testRunId, 'step_failed_2', 'step_completed', { + result: 'Should not work', + }) + ).rejects.toThrow(/terminal/i); }); - it('should throw error for non-existent hook', async () => { - await expect(storage.hooks.dispose('hook_nonexistent')).rejects.toThrow( - 'Hook hook_nonexistent not found' + it('should reject step_failed on already failed step', async () => { + await createStep(storage, testRunId, { + stepId: 'step_failed_3', + stepName: 'test-step', + input: [], + }); + await updateStep(storage, testRunId, 'step_failed_3', 'step_failed', { + error: 'Failed once', + }); + + await expect( + updateStep(storage, testRunId, 'step_failed_3', 'step_failed', { + error: 'Failed again', + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject step_retrying on failed step', async () => { + await createStep(storage, testRunId, { + stepId: 'step_failed_retry', + stepName: 'test-step', + input: [], + }); + await updateStep( + storage, + testRunId, + 'step_failed_retry', + 'step_failed', + { + error: 'Failed permanently', + } + ); + + await expect( + updateStep(storage, testRunId, 'step_failed_retry', 'step_retrying', { + error: 'Retry attempt', + }) + ).rejects.toThrow(/terminal/i); + }); + }); + + describe('step_retrying validation', () => { + it('should reject step_retrying on completed step', async () => { + await createStep(storage, testRunId, { + stepId: 'step_completed_retry', + stepName: 'test-step', + input: [], + }); + await updateStep( + storage, + testRunId, + 'step_completed_retry', + 'step_completed', + { + result: 'done', + } ); + + await expect( + updateStep( + storage, + testRunId, + 'step_completed_retry', + 'step_retrying', + { + error: 'Retry attempt', + } + ) + ).rejects.toThrow(/terminal/i); }); }); }); + + describe('run terminal state validation', () => { + describe('completed run', () => { + it('should reject run_started on completed run', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await updateRun(storage, run.runId, 'run_completed', { + output: 'done', + }); + + await expect( + updateRun(storage, run.runId, 'run_started') + ).rejects.toThrow(/terminal/i); + }); + + it('should reject run_failed on completed run', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await updateRun(storage, run.runId, 'run_completed', { + output: 'done', + }); + + await expect( + updateRun(storage, run.runId, 'run_failed', { + error: 'Should not work', + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject run_cancelled on completed run', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await updateRun(storage, run.runId, 'run_completed', { + output: 'done', + }); + + await expect( + storage.events.create(run.runId, { eventType: 'run_cancelled' }) + ).rejects.toThrow(/terminal/i); + }); + }); + + describe('failed run', () => { + it('should reject run_started on failed run', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await updateRun(storage, run.runId, 'run_failed', { error: 'Failed' }); + + await expect( + updateRun(storage, run.runId, 'run_started') + ).rejects.toThrow(/terminal/i); + }); + + it('should reject run_completed on failed run', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await updateRun(storage, run.runId, 'run_failed', { error: 'Failed' }); + + await expect( + updateRun(storage, run.runId, 'run_completed', { + output: 'Should not work', + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject run_cancelled on failed run', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await updateRun(storage, run.runId, 'run_failed', { error: 'Failed' }); + + await expect( + storage.events.create(run.runId, { eventType: 'run_cancelled' }) + ).rejects.toThrow(/terminal/i); + }); + }); + + describe('cancelled run', () => { + it('should reject run_started on cancelled run', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await storage.events.create(run.runId, { eventType: 'run_cancelled' }); + + await expect( + updateRun(storage, run.runId, 'run_started') + ).rejects.toThrow(/terminal/i); + }); + + it('should reject run_completed on cancelled run', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await storage.events.create(run.runId, { eventType: 'run_cancelled' }); + + await expect( + updateRun(storage, run.runId, 'run_completed', { + output: 'Should not work', + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject run_failed on cancelled run', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await storage.events.create(run.runId, { eventType: 'run_cancelled' }); + + await expect( + updateRun(storage, run.runId, 'run_failed', { + error: 'Should not work', + }) + ).rejects.toThrow(/terminal/i); + }); + }); + }); + + describe('allowed operations on terminal runs', () => { + it('should allow step_completed on completed run for in-progress step', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + + // Create and start a step (making it in-progress) + await createStep(storage, run.runId, { + stepId: 'step_in_progress', + stepName: 'test-step', + input: [], + }); + await updateStep(storage, run.runId, 'step_in_progress', 'step_started'); + + // Complete the run while step is still running + await updateRun(storage, run.runId, 'run_completed', { output: 'done' }); + + // Should succeed - completing an in-progress step on a terminal run is allowed + const result = await updateStep( + storage, + run.runId, + 'step_in_progress', + 'step_completed', + { result: 'step done' } + ); + expect(result.status).toBe('completed'); + }); + + it('should allow step_failed on completed run for in-progress step', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + + // Create and start a step + await createStep(storage, run.runId, { + stepId: 'step_in_progress_fail', + stepName: 'test-step', + input: [], + }); + await updateStep( + storage, + run.runId, + 'step_in_progress_fail', + 'step_started' + ); + + // Complete the run + await updateRun(storage, run.runId, 'run_completed', { output: 'done' }); + + // Should succeed - failing an in-progress step on a terminal run is allowed + const result = await updateStep( + storage, + run.runId, + 'step_in_progress_fail', + 'step_failed', + { error: 'step failed' } + ); + expect(result.status).toBe('failed'); + }); + + it('should auto-delete hooks when run completes (world-local specific behavior)', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + + // Create a hook + await createHook(storage, run.runId, { + hookId: 'hook_auto_delete', + token: 'test-token-auto-delete', + }); + + // Verify hook exists before completion + const hookBefore = await storage.hooks.get('hook_auto_delete'); + expect(hookBefore).toBeDefined(); + + // Complete the run - this auto-deletes hooks in world-local + await updateRun(storage, run.runId, 'run_completed', { output: 'done' }); + + // Hook should be auto-deleted + await expect(storage.hooks.get('hook_auto_delete')).rejects.toThrow( + /not found/i + ); + }); + }); + + describe('disallowed operations on terminal runs', () => { + it('should reject step_created on completed run', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await updateRun(storage, run.runId, 'run_completed', { output: 'done' }); + + await expect( + createStep(storage, run.runId, { + stepId: 'new_step', + stepName: 'test-step', + input: [], + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject step_started on completed run for pending step', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + + // Create a step but don't start it + await createStep(storage, run.runId, { + stepId: 'pending_step', + stepName: 'test-step', + input: [], + }); + + // Complete the run + await updateRun(storage, run.runId, 'run_completed', { output: 'done' }); + + // Should reject - cannot start a pending step on a terminal run + await expect( + updateStep(storage, run.runId, 'pending_step', 'step_started') + ).rejects.toThrow(/terminal/i); + }); + + it('should reject hook_created on completed run', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await updateRun(storage, run.runId, 'run_completed', { output: 'done' }); + + await expect( + createHook(storage, run.runId, { + hookId: 'new_hook', + token: 'new-token', + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject step_created on failed run', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await updateRun(storage, run.runId, 'run_failed', { error: 'Failed' }); + + await expect( + createStep(storage, run.runId, { + stepId: 'new_step_failed', + stepName: 'test-step', + input: [], + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject step_created on cancelled run', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await storage.events.create(run.runId, { eventType: 'run_cancelled' }); + + await expect( + createStep(storage, run.runId, { + stepId: 'new_step_cancelled', + stepName: 'test-step', + input: [], + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject hook_created on failed run', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await updateRun(storage, run.runId, 'run_failed', { error: 'Failed' }); + + await expect( + createHook(storage, run.runId, { + hookId: 'new_hook_failed', + token: 'new-token-failed', + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject hook_created on cancelled run', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await storage.events.create(run.runId, { eventType: 'run_cancelled' }); + + await expect( + createHook(storage, run.runId, { + hookId: 'new_hook_cancelled', + token: 'new-token-cancelled', + }) + ).rejects.toThrow(/terminal/i); + }); + }); + + describe('idempotent operations', () => { + it('should allow run_cancelled on already cancelled run (idempotent)', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await storage.events.create(run.runId, { eventType: 'run_cancelled' }); + + // Should succeed - idempotent operation + const result = await storage.events.create(run.runId, { + eventType: 'run_cancelled', + }); + expect(result.run?.status).toBe('cancelled'); + }); + }); + + describe('step_retrying event handling', () => { + let testRunId: string; + + beforeEach(async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + testRunId = run.runId; + }); + + it('should set step status to pending and record error', async () => { + await createStep(storage, testRunId, { + stepId: 'step_retry_1', + stepName: 'test-step', + input: [], + }); + await updateStep(storage, testRunId, 'step_retry_1', 'step_started'); + + const result = await storage.events.create(testRunId, { + eventType: 'step_retrying', + correlationId: 'step_retry_1', + eventData: { + error: 'Temporary failure', + retryAfter: new Date(Date.now() + 5000), + }, + }); + + expect(result.step?.status).toBe('pending'); + expect(result.step?.error?.message).toBe('Temporary failure'); + expect(result.step?.retryAfter).toBeInstanceOf(Date); + }); + + it('should increment attempt when step_started is called after step_retrying', async () => { + await createStep(storage, testRunId, { + stepId: 'step_retry_2', + stepName: 'test-step', + input: [], + }); + + // First attempt + const started1 = await updateStep( + storage, + testRunId, + 'step_retry_2', + 'step_started' + ); + expect(started1.attempt).toBe(1); + + // Retry + await storage.events.create(testRunId, { + eventType: 'step_retrying', + correlationId: 'step_retry_2', + eventData: { error: 'Temporary failure' }, + }); + + // Second attempt + const started2 = await updateStep( + storage, + testRunId, + 'step_retry_2', + 'step_started' + ); + expect(started2.attempt).toBe(2); + }); + + it('should reject step_retrying on completed step', async () => { + await createStep(storage, testRunId, { + stepId: 'step_retry_completed', + stepName: 'test-step', + input: [], + }); + await updateStep( + storage, + testRunId, + 'step_retry_completed', + 'step_completed', + { + result: 'done', + } + ); + + await expect( + storage.events.create(testRunId, { + eventType: 'step_retrying', + correlationId: 'step_retry_completed', + eventData: { error: 'Should not work' }, + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject step_retrying on failed step', async () => { + await createStep(storage, testRunId, { + stepId: 'step_retry_failed', + stepName: 'test-step', + input: [], + }); + await updateStep(storage, testRunId, 'step_retry_failed', 'step_failed', { + error: 'Permanent failure', + }); + + await expect( + storage.events.create(testRunId, { + eventType: 'step_retrying', + correlationId: 'step_retry_failed', + eventData: { error: 'Should not work' }, + }) + ).rejects.toThrow(/terminal/i); + }); + }); + + describe('run cancellation with in-flight entities', () => { + it('should allow in-progress step to complete after run cancelled', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + + // Create and start a step + await createStep(storage, run.runId, { + stepId: 'step_in_flight', + stepName: 'test-step', + input: [], + }); + await updateStep(storage, run.runId, 'step_in_flight', 'step_started'); + + // Cancel the run + await storage.events.create(run.runId, { eventType: 'run_cancelled' }); + + // Should succeed - completing an in-progress step is allowed + const result = await updateStep( + storage, + run.runId, + 'step_in_flight', + 'step_completed', + { result: 'done' } + ); + expect(result.status).toBe('completed'); + }); + + it('should reject step_created after run cancelled', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await storage.events.create(run.runId, { eventType: 'run_cancelled' }); + + await expect( + createStep(storage, run.runId, { + stepId: 'new_step_after_cancel', + stepName: 'test-step', + input: [], + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject step_started for pending step after run cancelled', async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + + // Create a step but don't start it + await createStep(storage, run.runId, { + stepId: 'pending_after_cancel', + stepName: 'test-step', + input: [], + }); + + // Cancel the run + await storage.events.create(run.runId, { eventType: 'run_cancelled' }); + + // Should reject - cannot start a pending step on a cancelled run + await expect( + updateStep(storage, run.runId, 'pending_after_cancel', 'step_started') + ).rejects.toThrow(/terminal/i); + }); + }); + + describe('event ordering validation', () => { + let testRunId: string; + + beforeEach(async () => { + const run = await createRun(storage, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + testRunId = run.runId; + }); + + it('should reject step_completed before step_created', async () => { + await expect( + storage.events.create(testRunId, { + eventType: 'step_completed', + correlationId: 'nonexistent_step', + eventData: { result: 'done' }, + }) + ).rejects.toThrow(/not found/i); + }); + + it('should reject step_started before step_created', async () => { + await expect( + storage.events.create(testRunId, { + eventType: 'step_started', + correlationId: 'nonexistent_step_started', + }) + ).rejects.toThrow(/not found/i); + }); + + it('should reject step_failed before step_created', async () => { + await expect( + storage.events.create(testRunId, { + eventType: 'step_failed', + correlationId: 'nonexistent_step_failed', + eventData: { error: 'Failed' }, + }) + ).rejects.toThrow(/not found/i); + }); + + it('should allow step_completed without step_started (instant completion)', async () => { + await createStep(storage, testRunId, { + stepId: 'instant_complete', + stepName: 'test-step', + input: [], + }); + + // Should succeed - instant completion without starting + const result = await updateStep( + storage, + testRunId, + 'instant_complete', + 'step_completed', + { result: 'instant' } + ); + expect(result.status).toBe('completed'); + }); + + it('should reject hook_disposed before hook_created', async () => { + await expect( + storage.events.create(testRunId, { + eventType: 'hook_disposed', + correlationId: 'nonexistent_hook', + }) + ).rejects.toThrow(/not found/i); + }); + + it('should reject hook_received before hook_created', async () => { + await expect( + storage.events.create(testRunId, { + eventType: 'hook_received', + correlationId: 'nonexistent_hook_received', + eventData: { payload: {} }, + }) + ).rejects.toThrow(/not found/i); + }); + }); }); diff --git a/packages/world-local/src/storage.ts b/packages/world-local/src/storage.ts index 46485e7d8..1a2ead142 100644 --- a/packages/world-local/src/storage.ts +++ b/packages/world-local/src/storage.ts @@ -1,8 +1,8 @@ import path from 'node:path'; -import { WorkflowRunNotFoundError } from '@workflow/errors'; +import { WorkflowAPIError, WorkflowRunNotFoundError } from '@workflow/errors'; import { - type CreateHookRequest, type Event, + type EventResult, EventSchema, type GetHookParams, type Hook, @@ -102,7 +102,7 @@ const getObjectCreatedAt = * Implements the Storage['hooks'] interface with hook CRUD operations. */ function createHooksStorage(basedir: string): Storage['hooks'] { - // Helper function to find a hook by token (shared between create and getByToken) + // Helper function to find a hook by token (shared between getByToken) async function findHookByToken(token: string): Promise { const hooksDir = path.join(basedir, 'hooks'); const files = await listJSONFiles(hooksDir); @@ -118,35 +118,6 @@ function createHooksStorage(basedir: string): Storage['hooks'] { return null; } - async function create(runId: string, data: CreateHookRequest): Promise { - // Check if a hook with the same token already exists - // Token uniqueness is enforced globally per local environment - const existingHook = await findHookByToken(data.token); - if (existingHook) { - throw new Error( - `Hook with token ${data.token} already exists for this project` - ); - } - - const now = new Date(); - - const result = { - runId, - hookId: data.hookId, - token: data.token, - metadata: data.metadata, - ownerId: 'local-owner', - projectId: 'local-project', - environment: 'local', - createdAt: now, - } as Hook; - - const hookPath = path.join(basedir, 'hooks', `${data.hookId}.json`); - HookSchema.parse(result); - await writeJSON(hookPath, result); - return result; - } - async function get(hookId: string, params?: GetHookParams): Promise { const hookPath = path.join(basedir, 'hooks', `${hookId}.json`); const hook = await readJSON(hookPath, HookSchema); @@ -202,17 +173,7 @@ function createHooksStorage(basedir: string): Storage['hooks'] { }; } - async function dispose(hookId: string): Promise { - const hookPath = path.join(basedir, 'hooks', `${hookId}.json`); - const hook = await readJSON(hookPath, HookSchema); - if (!hook) { - throw new Error(`Hook ${hookId} not found`); - } - await deleteJSON(hookPath); - return hook; - } - - return { create, get, getByToken, list, dispose }; + return { get, getByToken, list }; } /** @@ -237,33 +198,6 @@ async function deleteAllHooksForRun( export function createStorage(basedir: string): Storage { return { runs: { - async create(data) { - const runId = `wrun_${monotonicUlid()}`; - const now = new Date(); - - const result: WorkflowRun = { - runId, - deploymentId: data.deploymentId, - status: 'pending', - workflowName: data.workflowName, - executionContext: data.executionContext as - | Record - | undefined, - input: (data.input as any[]) || [], - output: undefined, - error: undefined, - startedAt: undefined, - completedAt: undefined, - createdAt: now, - updatedAt: now, - }; - - const runPath = path.join(basedir, 'runs', `${runId}.json`); - WorkflowRunSchema.parse(result); - await writeJSON(runPath, result); - return result; - }, - async get(id, params) { const runPath = path.join(basedir, 'runs', `${id}.json`); const run = await readJSON(runPath, WorkflowRunSchema); @@ -274,54 +208,6 @@ export function createStorage(basedir: string): Storage { return filterRunData(run, resolveData); }, - /** - * Updates a workflow run. - * - * Note: This operation is not atomic. Concurrent updates from multiple - * processes may result in lost updates (last writer wins). This is an - * inherent limitation of filesystem-based storage without locking. - * For the local world, this is acceptable as it's typically - * used in single-process scenarios. - */ - async update(id, data) { - const runPath = path.join(basedir, 'runs', `${id}.json`); - const run = await readJSON(runPath, WorkflowRunSchema); - if (!run) { - throw new WorkflowRunNotFoundError(id); - } - - const now = new Date(); - const updatedRun = { - ...run, - ...data, - updatedAt: now, - } as WorkflowRun; - - // Only set startedAt the first time the run transitions to 'running' - if (data.status === 'running' && !updatedRun.startedAt) { - updatedRun.startedAt = now; - } - - const isBecomingTerminal = - data.status === 'completed' || - data.status === 'failed' || - data.status === 'cancelled'; - - if (isBecomingTerminal) { - updatedRun.completedAt = now; - } - - WorkflowRunSchema.parse(updatedRun); - await writeJSON(runPath, updatedRun, { overwrite: true }); - - // If transitioning to a terminal status, clean up all hooks for this run - if (isBecomingTerminal) { - await deleteAllHooksForRun(basedir, id); - } - - return updatedRun; - }, - async list(params) { const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION; const result = await paginatedFileSystemQuery({ @@ -360,54 +246,9 @@ export function createStorage(basedir: string): Storage { return result; }, - - async cancel(id, params) { - // This will call update which triggers hook cleanup automatically - const run = await this.update(id, { status: 'cancelled' }); - const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION; - return filterRunData(run, resolveData); - }, - - async pause(id, params) { - const run = await this.update(id, { status: 'paused' }); - const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION; - return filterRunData(run, resolveData); - }, - - async resume(id, params) { - const run = await this.update(id, { status: 'running' }); - const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION; - return filterRunData(run, resolveData); - }, }, steps: { - async create(runId, data) { - const now = new Date(); - - const result: Step = { - runId, - stepId: data.stepId, - stepName: data.stepName, - status: 'pending', - input: data.input as any[], - output: undefined, - error: undefined, - attempt: 0, - startedAt: undefined, - completedAt: undefined, - createdAt: now, - updatedAt: now, - }; - - const compositeKey = `${runId}-${data.stepId}`; - const stepPath = path.join(basedir, 'steps', `${compositeKey}.json`); - StepSchema.parse(result); - await writeJSON(stepPath, result); - - return result; - }, - async get( runId: string | undefined, stepId: string, @@ -433,41 +274,6 @@ export function createStorage(basedir: string): Storage { return filterStepData(step, resolveData); }, - /** - * Updates a step. - * - * Note: This operation is not atomic. Concurrent updates from multiple - * processes may result in lost updates (last writer wins). This is an - * inherent limitation of filesystem-based storage without locking. - */ - async update(runId, stepId, data) { - const compositeKey = `${runId}-${stepId}`; - const stepPath = path.join(basedir, 'steps', `${compositeKey}.json`); - const step = await readJSON(stepPath, StepSchema); - if (!step) { - throw new Error(`Step ${stepId} in run ${runId} not found`); - } - - const now = new Date(); - const updatedStep: Step = { - ...step, - ...data, - updatedAt: now, - }; - - // Only set startedAt the first time the step transitions to 'running' - if (data.status === 'running' && !updatedStep.startedAt) { - updatedStep.startedAt = now; - } - if (data.status === 'completed' || data.status === 'failed') { - updatedStep.completedAt = now; - } - - StepSchema.parse(updatedStep); - await writeJSON(stepPath, updatedStep, { overwrite: true }); - return updatedStep; - }, - async list(params) { const resolveData = params.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION; const result = await paginatedFileSystemQuery({ @@ -499,25 +305,522 @@ export function createStorage(basedir: string): Storage { // Events - filesystem-backed storage events: { - async create(runId, data, params) { + async create(runId, data, params): Promise { const eventId = `evnt_${monotonicUlid()}`; const now = new Date(); - const result: Event = { + // For run_created events, generate runId server-side if null or empty + let effectiveRunId: string; + if (data.eventType === 'run_created' && (!runId || runId === '')) { + effectiveRunId = `wrun_${monotonicUlid()}`; + } else if (!runId) { + throw new Error('runId is required for non-run_created events'); + } else { + effectiveRunId = runId; + } + + // Helper to check if run is in terminal state + const isRunTerminal = (status: string) => + ['completed', 'failed', 'cancelled'].includes(status); + + // Helper to check if step is in terminal state + const isStepTerminal = (status: string) => + ['completed', 'failed'].includes(status); + + // Get current run state for validation (if not creating a new run) + // Skip run validation for step_completed and step_retrying - they only operate + // on running steps, and running steps are always allowed to modify regardless + // of run state. This optimization saves filesystem reads per step event. + let currentRun: WorkflowRun | null = null; + const skipRunValidationEvents = ['step_completed', 'step_retrying']; + if ( + data.eventType !== 'run_created' && + !skipRunValidationEvents.includes(data.eventType) + ) { + const runPath = path.join(basedir, 'runs', `${effectiveRunId}.json`); + currentRun = await readJSON(runPath, WorkflowRunSchema); + } + + // ============================================================ + // VALIDATION: Terminal state and event ordering checks + // ============================================================ + + // Run terminal state validation + if (currentRun && isRunTerminal(currentRun.status)) { + const runTerminalEvents = [ + 'run_started', + 'run_completed', + 'run_failed', + ]; + + // Idempotent operation: run_cancelled on already cancelled run is allowed + if ( + data.eventType === 'run_cancelled' && + currentRun.status === 'cancelled' + ) { + // Return existing state (idempotent) + const event: Event = { + ...data, + runId: effectiveRunId, + eventId, + createdAt: now, + }; + const compositeKey = `${effectiveRunId}-${eventId}`; + const eventPath = path.join( + basedir, + 'events', + `${compositeKey}.json` + ); + await writeJSON(eventPath, event); + const resolveData = + params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION; + return { + event: filterEventData(event, resolveData), + run: currentRun, + }; + } + + // Run state transitions are not allowed on terminal runs + if ( + runTerminalEvents.includes(data.eventType) || + data.eventType === 'run_cancelled' + ) { + throw new WorkflowAPIError( + `Cannot transition run from terminal state "${currentRun.status}"`, + { status: 409 } + ); + } + + // Creating new entities on terminal runs is not allowed + if ( + data.eventType === 'step_created' || + data.eventType === 'hook_created' + ) { + throw new WorkflowAPIError( + `Cannot create new entities on run in terminal state "${currentRun.status}"`, + { status: 409 } + ); + } + } + + // Step-related event validation (ordering and terminal state) + // Store existingStep so we can reuse it later (avoid double read) + let validatedStep: Step | null = null; + const stepEvents = [ + 'step_started', + 'step_completed', + 'step_failed', + 'step_retrying', + ]; + if (stepEvents.includes(data.eventType) && data.correlationId) { + const stepCompositeKey = `${effectiveRunId}-${data.correlationId}`; + const stepPath = path.join( + basedir, + 'steps', + `${stepCompositeKey}.json` + ); + validatedStep = await readJSON(stepPath, StepSchema); + + // Event ordering: step must exist before these events + if (!validatedStep) { + throw new WorkflowAPIError( + `Step "${data.correlationId}" not found`, + { status: 404 } + ); + } + + // Step terminal state validation + if (isStepTerminal(validatedStep.status)) { + throw new WorkflowAPIError( + `Cannot modify step in terminal state "${validatedStep.status}"`, + { status: 409 } + ); + } + + // On terminal runs: only allow completing/failing in-progress steps + if (currentRun && isRunTerminal(currentRun.status)) { + if (validatedStep.status !== 'running') { + throw new WorkflowAPIError( + `Cannot modify non-running step on run in terminal state "${currentRun.status}"`, + { status: 409 } + ); + } + } + } + + // Hook-related event validation (ordering) + const hookEventsRequiringExistence = ['hook_disposed', 'hook_received']; + if ( + hookEventsRequiringExistence.includes(data.eventType) && + data.correlationId + ) { + const hookPath = path.join( + basedir, + 'hooks', + `${data.correlationId}.json` + ); + const existingHook = await readJSON(hookPath, HookSchema); + + if (!existingHook) { + throw new WorkflowAPIError( + `Hook "${data.correlationId}" not found`, + { status: 404 } + ); + } + } + + const event: Event = { ...data, - runId, + runId: effectiveRunId, eventId, createdAt: now, }; + // Track entity created/updated for EventResult + let run: WorkflowRun | undefined; + let step: Step | undefined; + let hook: Hook | undefined; + + // Create/update entity based on event type (event-sourced architecture) + // Run lifecycle events + if (data.eventType === 'run_created' && 'eventData' in data) { + const runData = data.eventData as { + deploymentId: string; + workflowName: string; + input: any[]; + executionContext?: Record; + }; + run = { + runId: effectiveRunId, + deploymentId: runData.deploymentId, + status: 'pending', + workflowName: runData.workflowName, + executionContext: runData.executionContext, + input: runData.input || [], + output: undefined, + error: undefined, + startedAt: undefined, + completedAt: undefined, + createdAt: now, + updatedAt: now, + }; + const runPath = path.join(basedir, 'runs', `${effectiveRunId}.json`); + await writeJSON(runPath, run); + } else if (data.eventType === 'run_started') { + // Reuse currentRun from validation (already read above) + if (currentRun) { + const runPath = path.join( + basedir, + 'runs', + `${effectiveRunId}.json` + ); + run = { + runId: currentRun.runId, + deploymentId: currentRun.deploymentId, + workflowName: currentRun.workflowName, + executionContext: currentRun.executionContext, + input: currentRun.input, + createdAt: currentRun.createdAt, + expiredAt: currentRun.expiredAt, + status: 'running', + output: undefined, + error: undefined, + completedAt: undefined, + startedAt: currentRun.startedAt ?? now, + updatedAt: now, + }; + await writeJSON(runPath, run, { overwrite: true }); + } + } else if (data.eventType === 'run_completed' && 'eventData' in data) { + const completedData = data.eventData as { output?: any }; + // Reuse currentRun from validation (already read above) + if (currentRun) { + const runPath = path.join( + basedir, + 'runs', + `${effectiveRunId}.json` + ); + run = { + runId: currentRun.runId, + deploymentId: currentRun.deploymentId, + workflowName: currentRun.workflowName, + executionContext: currentRun.executionContext, + input: currentRun.input, + createdAt: currentRun.createdAt, + expiredAt: currentRun.expiredAt, + startedAt: currentRun.startedAt, + status: 'completed', + output: completedData.output, + error: undefined, + completedAt: now, + updatedAt: now, + }; + await writeJSON(runPath, run, { overwrite: true }); + await deleteAllHooksForRun(basedir, effectiveRunId); + } + } else if (data.eventType === 'run_failed' && 'eventData' in data) { + const failedData = data.eventData as { + error: any; + errorCode?: string; + }; + // Reuse currentRun from validation (already read above) + if (currentRun) { + const runPath = path.join( + basedir, + 'runs', + `${effectiveRunId}.json` + ); + run = { + runId: currentRun.runId, + deploymentId: currentRun.deploymentId, + workflowName: currentRun.workflowName, + executionContext: currentRun.executionContext, + input: currentRun.input, + createdAt: currentRun.createdAt, + expiredAt: currentRun.expiredAt, + startedAt: currentRun.startedAt, + status: 'failed', + output: undefined, + error: { + message: + typeof failedData.error === 'string' + ? failedData.error + : (failedData.error?.message ?? 'Unknown error'), + stack: failedData.error?.stack, + code: failedData.errorCode, + }, + completedAt: now, + updatedAt: now, + }; + await writeJSON(runPath, run, { overwrite: true }); + await deleteAllHooksForRun(basedir, effectiveRunId); + } + } else if (data.eventType === 'run_cancelled') { + // Reuse currentRun from validation (already read above) + if (currentRun) { + const runPath = path.join( + basedir, + 'runs', + `${effectiveRunId}.json` + ); + run = { + runId: currentRun.runId, + deploymentId: currentRun.deploymentId, + workflowName: currentRun.workflowName, + executionContext: currentRun.executionContext, + input: currentRun.input, + createdAt: currentRun.createdAt, + expiredAt: currentRun.expiredAt, + startedAt: currentRun.startedAt, + status: 'cancelled', + output: undefined, + error: undefined, + completedAt: now, + updatedAt: now, + }; + await writeJSON(runPath, run, { overwrite: true }); + await deleteAllHooksForRun(basedir, effectiveRunId); + } + } else if ( + // Step lifecycle events + data.eventType === 'step_created' && + 'eventData' in data + ) { + // step_created: Creates step entity with status 'pending', attempt=0, createdAt set + const stepData = data.eventData as { + stepName: string; + input: any; + }; + step = { + runId: effectiveRunId, + stepId: data.correlationId, + stepName: stepData.stepName, + status: 'pending', + input: stepData.input, + output: undefined, + error: undefined, + attempt: 0, + startedAt: undefined, + completedAt: undefined, + createdAt: now, + updatedAt: now, + }; + const stepCompositeKey = `${effectiveRunId}-${data.correlationId}`; + const stepPath = path.join( + basedir, + 'steps', + `${stepCompositeKey}.json` + ); + await writeJSON(stepPath, step); + } else if (data.eventType === 'step_started') { + // step_started: Increments attempt, sets status to 'running' + // Sets startedAt only on the first start (not updated on retries) + // Reuse validatedStep from validation (already read above) + if (validatedStep) { + const stepCompositeKey = `${effectiveRunId}-${data.correlationId}`; + const stepPath = path.join( + basedir, + 'steps', + `${stepCompositeKey}.json` + ); + step = { + ...validatedStep, + status: 'running', + // Only set startedAt on the first start + startedAt: validatedStep.startedAt ?? now, + // Increment attempt counter on every start + attempt: validatedStep.attempt + 1, + updatedAt: now, + }; + await writeJSON(stepPath, step, { overwrite: true }); + } + } else if (data.eventType === 'step_completed' && 'eventData' in data) { + // step_completed: Terminal state with output + // Reuse validatedStep from validation (already read above) + const completedData = data.eventData as { result: any }; + if (validatedStep) { + const stepCompositeKey = `${effectiveRunId}-${data.correlationId}`; + const stepPath = path.join( + basedir, + 'steps', + `${stepCompositeKey}.json` + ); + step = { + ...validatedStep, + status: 'completed', + output: completedData.result, + completedAt: now, + updatedAt: now, + }; + await writeJSON(stepPath, step, { overwrite: true }); + } + } else if (data.eventType === 'step_failed' && 'eventData' in data) { + // step_failed: Terminal state with error + // Reuse validatedStep from validation (already read above) + const failedData = data.eventData as { + error: any; + stack?: string; + }; + if (validatedStep) { + const stepCompositeKey = `${effectiveRunId}-${data.correlationId}`; + const stepPath = path.join( + basedir, + 'steps', + `${stepCompositeKey}.json` + ); + const error = { + message: + typeof failedData.error === 'string' + ? failedData.error + : (failedData.error?.message ?? 'Unknown error'), + stack: failedData.stack, + }; + step = { + ...validatedStep, + status: 'failed', + error, + completedAt: now, + updatedAt: now, + }; + await writeJSON(stepPath, step, { overwrite: true }); + } + } else if (data.eventType === 'step_retrying' && 'eventData' in data) { + // step_retrying: Sets status back to 'pending', records error + // Reuse validatedStep from validation (already read above) + const retryData = data.eventData as { + error: any; + stack?: string; + retryAfter?: Date; + }; + if (validatedStep) { + const stepCompositeKey = `${effectiveRunId}-${data.correlationId}`; + const stepPath = path.join( + basedir, + 'steps', + `${stepCompositeKey}.json` + ); + step = { + ...validatedStep, + status: 'pending', + error: { + message: + typeof retryData.error === 'string' + ? retryData.error + : (retryData.error?.message ?? 'Unknown error'), + stack: retryData.stack, + }, + retryAfter: retryData.retryAfter, + updatedAt: now, + }; + await writeJSON(stepPath, step, { overwrite: true }); + } + } else if ( + // Hook lifecycle events + data.eventType === 'hook_created' && + 'eventData' in data + ) { + const hookData = data.eventData as { + token: string; + metadata?: any; + }; + + // Check for duplicate token before creating hook + const hooksDir = path.join(basedir, 'hooks'); + const hookFiles = await listJSONFiles(hooksDir); + for (const file of hookFiles) { + const existingHookPath = path.join(hooksDir, `${file}.json`); + const existingHook = await readJSON(existingHookPath, HookSchema); + if (existingHook && existingHook.token === hookData.token) { + throw new WorkflowAPIError( + `Hook with token ${hookData.token} already exists for this project`, + { status: 409 } + ); + } + } + + hook = { + runId: effectiveRunId, + hookId: data.correlationId, + token: hookData.token, + metadata: hookData.metadata, + ownerId: 'local-owner', + projectId: 'local-project', + environment: 'local', + createdAt: now, + }; + const hookPath = path.join( + basedir, + 'hooks', + `${data.correlationId}.json` + ); + await writeJSON(hookPath, hook); + } else if (data.eventType === 'hook_disposed') { + // Delete the hook when disposed + const hookPath = path.join( + basedir, + 'hooks', + `${data.correlationId}.json` + ); + await deleteJSON(hookPath); + } + // Note: hook_received events are stored in the event log but don't + // modify the Hook entity (which doesn't have a payload field) + // Store event using composite key {runId}-{eventId} - const compositeKey = `${runId}-${eventId}`; + const compositeKey = `${effectiveRunId}-${eventId}`; const eventPath = path.join(basedir, 'events', `${compositeKey}.json`); - EventSchema.parse(result); - await writeJSON(eventPath, result); + await writeJSON(eventPath, event); const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION; - return filterEventData(result, resolveData); + const filteredEvent = filterEventData(event, resolveData); + + // Return EventResult with event and any created/updated entity + return { + event: filteredEvent, + run, + step, + hook, + }; }, async list(params) { diff --git a/packages/world-postgres/src/drizzle/schema.ts b/packages/world-postgres/src/drizzle/schema.ts index 8a6bce004..0da98e369 100644 --- a/packages/world-postgres/src/drizzle/schema.ts +++ b/packages/world-postgres/src/drizzle/schema.ts @@ -105,6 +105,12 @@ export const events = schema.table( (tb) => [index().on(tb.runId), index().on(tb.correlationId)] ); +/** + * Database schema for steps. Note: DB column names differ from Step interface: + * - error (DB) → error (Step interface, parsed from JSON string) + * - startedAt (DB) → startedAt (Step interface) + * The mapping is done in storage.ts deserializeStepError() + */ export const steps = schema.table( 'workflow_steps', { @@ -118,8 +124,10 @@ export const steps = schema.table( /** @deprecated we stream binary data */ outputJson: jsonb('output').$type(), output: Cbor()('output_cbor'), + /** JSON-stringified StructuredError - parsed and set as error in Step interface */ error: text('error'), attempt: integer('attempt').notNull(), + /** Maps to startedAt in Step interface */ startedAt: timestamp('started_at'), completedAt: timestamp('completed_at'), createdAt: timestamp('created_at').defaultNow().notNull(), @@ -129,7 +137,14 @@ export const steps = schema.table( .notNull(), retryAfter: timestamp('retry_after'), } satisfies DrizzlishOfType< - Cborized & { input?: unknown }, 'output' | 'input'> + Cborized< + Omit & { + input?: unknown; + error?: string; + startedAt?: Date; + }, + 'output' | 'input' + > >, (tb) => [index().on(tb.runId), index().on(tb.status)] ); diff --git a/packages/world-postgres/src/storage.ts b/packages/world-postgres/src/storage.ts index a36e1514c..026e9e29c 100644 --- a/packages/world-postgres/src/storage.ts +++ b/packages/world-postgres/src/storage.ts @@ -1,6 +1,7 @@ import { WorkflowAPIError } from '@workflow/errors'; import type { Event, + EventResult, Hook, ListEventsParams, ListHooksParams, @@ -8,8 +9,6 @@ import type { ResolveData, Step, Storage, - UpdateStepRequest, - UpdateWorkflowRunRequest, WorkflowRun, } from '@workflow/world'; import { @@ -18,31 +17,12 @@ import { StepSchema, WorkflowRunSchema, } from '@workflow/world'; -import { and, desc, eq, gt, lt, sql } from 'drizzle-orm'; +import { and, desc, eq, gt, lt, notInArray, sql } from 'drizzle-orm'; import { monotonicFactory } from 'ulid'; import { type Drizzle, Schema } from './drizzle/index.js'; import type { SerializedContent } from './drizzle/schema.js'; import { compact } from './util.js'; -/** - * Serialize a StructuredError object into a JSON string - */ -function serializeRunError(data: UpdateWorkflowRunRequest): any { - if (!data.error) { - return data; - } - - const { error, ...rest } = data; - return { - ...rest, - error: JSON.stringify({ - message: error.message, - stack: error.stack, - code: error.code, - }), - }; -} - /** * Deserialize error JSON string (or legacy flat fields) into a StructuredError object * Handles backwards compatibility: @@ -88,64 +68,46 @@ function deserializeRunError(run: any): WorkflowRun { } /** - * Serialize a StructuredError object into a JSON string for steps + * Deserialize step data, mapping DB columns to interface fields: + * - `error` (DB column) → `error` (Step interface, parsed from JSON) + * - `startedAt` (DB column) → `startedAt` (Step interface) */ -function serializeStepError(data: UpdateStepRequest): any { - if (!data.error) { - return data; - } +function deserializeStepError(step: any): Step { + const { error, startedAt, ...rest } = step; - const { error, ...rest } = data; - return { + const result: any = { ...rest, - error: JSON.stringify({ - message: error.message, - stack: error.stack, - code: error.code, - }), + // Map startedAt to startedAt + startedAt: startedAt, }; -} - -/** - * Deserialize error JSON string (or legacy flat fields) into a StructuredError object for steps - */ -function deserializeStepError(step: any): Step { - const { error, ...rest } = step; if (!error) { - return step as Step; + return result as Step; } // Try to parse as structured error JSON - if (error) { - try { - const parsed = JSON.parse(error); - if (typeof parsed === 'object' && parsed.message !== undefined) { - return { - ...rest, - error: { - message: parsed.message, - stack: parsed.stack, - code: parsed.code, - }, - } as Step; - } - } catch { - // Not JSON, treat as plain string + try { + const parsed = JSON.parse(error); + if (typeof parsed === 'object' && parsed.message !== undefined) { + result.error = { + message: parsed.message, + stack: parsed.stack, + code: parsed.code, + }; + return result as Step; } + } catch { + // Not JSON, treat as plain string } // Backwards compatibility: handle legacy separate fields or plain string error - return { - ...rest, - error: { - message: error || '', - }, - } as Step; + result.error = { + message: error || '', + }; + return result as Step; } export function createRunsStorage(drizzle: Drizzle): Storage['runs'] { - const ulid = monotonicFactory(); const { runs } = Schema; const get = drizzle .select() @@ -168,76 +130,6 @@ export function createRunsStorage(drizzle: Drizzle): Storage['runs'] { const resolveData = params?.resolveData ?? 'all'; return filterRunData(parsed, resolveData); }, - async cancel(id, params) { - // TODO: we might want to guard this for only specific statuses - const [value] = await drizzle - .update(Schema.runs) - .set({ status: 'cancelled', completedAt: sql`now()` }) - .where(eq(runs.runId, id)) - .returning(); - if (!value) { - throw new WorkflowAPIError(`Run not found: ${id}`, { status: 404 }); - } - - // Clean up all hooks for this run when cancelling - await drizzle.delete(Schema.hooks).where(eq(Schema.hooks.runId, id)); - - const deserialized = deserializeRunError(compact(value)); - const parsed = WorkflowRunSchema.parse(deserialized); - const resolveData = params?.resolveData ?? 'all'; - return filterRunData(parsed, resolveData); - }, - async pause(id, params) { - // TODO: we might want to guard this for only specific statuses - const [value] = await drizzle - .update(Schema.runs) - .set({ status: 'paused' }) - .where(eq(runs.runId, id)) - .returning(); - if (!value) { - throw new WorkflowAPIError(`Run not found: ${id}`, { status: 404 }); - } - const deserialized = deserializeRunError(compact(value)); - const parsed = WorkflowRunSchema.parse(deserialized); - const resolveData = params?.resolveData ?? 'all'; - return filterRunData(parsed, resolveData); - }, - async resume(id, params) { - // Fetch current run to check if startedAt is already set - const [currentRun] = await drizzle - .select() - .from(runs) - .where(eq(runs.runId, id)) - .limit(1); - - if (!currentRun) { - throw new WorkflowAPIError(`Run not found: ${id}`, { status: 404 }); - } - - const updates: Partial = { - status: 'running', - }; - - // Only set startedAt the first time the run transitions to 'running' - if (!currentRun.startedAt) { - updates.startedAt = new Date(); - } - - const [value] = await drizzle - .update(Schema.runs) - .set(updates) - .where(and(eq(runs.runId, id), eq(runs.status, 'paused'))) - .returning(); - if (!value) { - throw new WorkflowAPIError(`Paused run not found: ${id}`, { - status: 404, - }); - } - const deserialized = deserializeRunError(compact(value)); - const parsed = WorkflowRunSchema.parse(deserialized); - const resolveData = params?.resolveData ?? 'all'; - return filterRunData(parsed, resolveData); - }, async list(params) { const limit = params?.pagination?.limit ?? 20; const fromCursor = params?.pagination?.cursor; @@ -268,98 +160,581 @@ export function createRunsStorage(drizzle: Drizzle): Storage['runs'] { cursor: values.at(-1)?.runId ?? null, }; }, - async create(data) { - const runId = `wrun_${ulid()}`; - const [value] = await drizzle - .insert(runs) - .values({ - runId, - input: data.input, - executionContext: data.executionContext as Record< - string, - unknown - > | null, - deploymentId: data.deploymentId, - status: 'pending', - workflowName: data.workflowName, - }) - .onConflictDoNothing() - .returning(); - if (!value) { - throw new WorkflowAPIError(`Run ${runId} already exists`, { - status: 409, + }; +} + +function map(obj: T | null | undefined, fn: (v: T) => R): undefined | R { + return obj ? fn(obj) : undefined; +} + +export function createEventsStorage(drizzle: Drizzle): Storage['events'] { + const ulid = monotonicFactory(); + const { events } = Schema; + + // Prepared statements for validation queries (performance optimization) + const getRunStatus = drizzle + .select({ status: Schema.runs.status }) + .from(Schema.runs) + .where(eq(Schema.runs.runId, sql.placeholder('runId'))) + .limit(1) + .prepare('events_get_run_status'); + + const getStepForValidation = drizzle + .select({ + status: Schema.steps.status, + startedAt: Schema.steps.startedAt, + }) + .from(Schema.steps) + .where( + and( + eq(Schema.steps.runId, sql.placeholder('runId')), + eq(Schema.steps.stepId, sql.placeholder('stepId')) + ) + ) + .limit(1) + .prepare('events_get_step_for_validation'); + + const getHookByToken = drizzle + .select({ hookId: Schema.hooks.hookId }) + .from(Schema.hooks) + .where(eq(Schema.hooks.token, sql.placeholder('token'))) + .limit(1) + .prepare('events_get_hook_by_token'); + + return { + async create(runId, data, params): Promise { + const eventId = `wevt_${ulid()}`; + + // For run_created events, generate runId server-side if null or empty + let effectiveRunId: string; + if (data.eventType === 'run_created' && (!runId || runId === '')) { + effectiveRunId = `wrun_${ulid()}`; + } else if (!runId) { + throw new Error('runId is required for non-run_created events'); + } else { + effectiveRunId = runId; + } + + // Track entity created/updated for EventResult + let run: WorkflowRun | undefined; + let step: Step | undefined; + let hook: Hook | undefined; + const now = new Date(); + + // Helper to check if run is in terminal state + const isRunTerminal = (status: string) => + ['completed', 'failed', 'cancelled'].includes(status); + + // Helper to check if step is in terminal state + const isStepTerminal = (status: string) => + ['completed', 'failed'].includes(status); + + // ============================================================ + // VALIDATION: Terminal state and event ordering checks + // ============================================================ + + // Get current run state for validation (if not creating a new run) + // Skip run validation for step_completed and step_retrying - they only operate + // on running steps, and running steps are always allowed to modify regardless + // of run state. This optimization saves database queries per step event. + let currentRun: { status: string } | null = null; + const skipRunValidationEvents = ['step_completed', 'step_retrying']; + if ( + data.eventType !== 'run_created' && + !skipRunValidationEvents.includes(data.eventType) + ) { + // Use prepared statement for better performance + const [runValue] = await getRunStatus.execute({ + runId: effectiveRunId, }); + currentRun = runValue ?? null; } - return deserializeRunError(compact(value)); - }, - async update(id, data) { - // Fetch current run to check if startedAt is already set - const [currentRun] = await drizzle - .select() - .from(runs) - .where(eq(runs.runId, id)) - .limit(1); - if (!currentRun) { - throw new WorkflowAPIError(`Run not found: ${id}`, { status: 404 }); + // Run terminal state validation + if (currentRun && isRunTerminal(currentRun.status)) { + const runTerminalEvents = [ + 'run_started', + 'run_completed', + 'run_failed', + ]; + + // Idempotent operation: run_cancelled on already cancelled run is allowed + if ( + data.eventType === 'run_cancelled' && + currentRun.status === 'cancelled' + ) { + // Get full run for return value + const [fullRun] = await drizzle + .select() + .from(Schema.runs) + .where(eq(Schema.runs.runId, effectiveRunId)) + .limit(1); + + // Create the event (still record it) + const [value] = await drizzle + .insert(Schema.events) + .values({ + runId: effectiveRunId, + eventId, + correlationId: data.correlationId, + eventType: data.eventType, + eventData: 'eventData' in data ? data.eventData : undefined, + }) + .returning({ createdAt: Schema.events.createdAt }); + + const result = { ...data, ...value, runId: effectiveRunId, eventId }; + const parsed = EventSchema.parse(result); + const resolveData = params?.resolveData ?? 'all'; + return { + event: filterEventData(parsed, resolveData), + run: fullRun ? deserializeRunError(compact(fullRun)) : undefined, + }; + } + + // Run state transitions are not allowed on terminal runs + if ( + runTerminalEvents.includes(data.eventType) || + data.eventType === 'run_cancelled' + ) { + throw new WorkflowAPIError( + `Cannot transition run from terminal state "${currentRun.status}"`, + { status: 409 } + ); + } + + // Creating new entities on terminal runs is not allowed + if ( + data.eventType === 'step_created' || + data.eventType === 'hook_created' + ) { + throw new WorkflowAPIError( + `Cannot create new entities on run in terminal state "${currentRun.status}"`, + { status: 409 } + ); + } } - // Serialize the error field if present - const serialized = serializeRunError(data); + // Step-related event validation (ordering and terminal state) + // Fetch status + startedAt so we can reuse for step_started (avoid double read) + // Skip validation for step_completed/step_failed - use conditional UPDATE instead + let validatedStep: { status: string; startedAt: Date | null } | null = + null; + const stepEventsNeedingValidation = ['step_started', 'step_retrying']; + if ( + stepEventsNeedingValidation.includes(data.eventType) && + data.correlationId + ) { + // Use prepared statement for better performance + const [existingStep] = await getStepForValidation.execute({ + runId: effectiveRunId, + stepId: data.correlationId, + }); - const updates: Partial = { - ...serialized, - output: data.output as SerializedContent, - }; + validatedStep = existingStep ?? null; + + // Event ordering: step must exist before these events + if (!validatedStep) { + throw new WorkflowAPIError(`Step "${data.correlationId}" not found`, { + status: 404, + }); + } + + // Step terminal state validation + if (isStepTerminal(validatedStep.status)) { + throw new WorkflowAPIError( + `Cannot modify step in terminal state "${validatedStep.status}"`, + { status: 409 } + ); + } + + // On terminal runs: only allow completing/failing in-progress steps + if (currentRun && isRunTerminal(currentRun.status)) { + if (validatedStep.status !== 'running') { + throw new WorkflowAPIError( + `Cannot modify non-running step on run in terminal state "${currentRun.status}"`, + { status: 409 } + ); + } + } + } - // Only set startedAt the first time transitioning to 'running' - if (data.status === 'running' && !currentRun.startedAt) { - updates.startedAt = new Date(); + // Hook-related event validation (ordering) + const hookEventsRequiringExistence = ['hook_disposed', 'hook_received']; + if ( + hookEventsRequiringExistence.includes(data.eventType) && + data.correlationId + ) { + const [existingHook] = await drizzle + .select({ hookId: Schema.hooks.hookId }) + .from(Schema.hooks) + .where(eq(Schema.hooks.hookId, data.correlationId)) + .limit(1); + + if (!existingHook) { + throw new WorkflowAPIError(`Hook "${data.correlationId}" not found`, { + status: 404, + }); + } } - const isBecomingTerminal = - data.status === 'completed' || - data.status === 'failed' || - data.status === 'cancelled'; + // ============================================================ + // Entity creation/updates based on event type + // ============================================================ + + // Handle run_created event: create the run entity atomically + if (data.eventType === 'run_created') { + const eventData = (data as any).eventData as { + deploymentId: string; + workflowName: string; + input: any[]; + executionContext?: Record; + }; + const [runValue] = await drizzle + .insert(Schema.runs) + .values({ + runId: effectiveRunId, + deploymentId: eventData.deploymentId, + workflowName: eventData.workflowName, + input: eventData.input as SerializedContent, + executionContext: eventData.executionContext as + | SerializedContent + | undefined, + status: 'pending', + }) + .onConflictDoNothing() + .returning(); + if (runValue) { + run = deserializeRunError(compact(runValue)); + } + } - if (isBecomingTerminal) { - updates.completedAt = new Date(); + // Handle run_started event: update run status + if (data.eventType === 'run_started') { + const [runValue] = await drizzle + .update(Schema.runs) + .set({ + status: 'running', + startedAt: now, + updatedAt: now, + }) + .where(eq(Schema.runs.runId, effectiveRunId)) + .returning(); + if (runValue) { + run = deserializeRunError(compact(runValue)); + } } - const [value] = await drizzle - .update(runs) - .set(updates) - .where(eq(runs.runId, id)) - .returning(); - if (!value) { - throw new WorkflowAPIError(`Run not found: ${id}`, { status: 404 }); + // Handle run_completed event: update run status and cleanup hooks + if (data.eventType === 'run_completed') { + const eventData = (data as any).eventData as { output?: any }; + const [runValue] = await drizzle + .update(Schema.runs) + .set({ + status: 'completed', + output: eventData.output as SerializedContent | undefined, + completedAt: now, + updatedAt: now, + }) + .where(eq(Schema.runs.runId, effectiveRunId)) + .returning(); + if (runValue) { + run = deserializeRunError(compact(runValue)); + } + // Delete all hooks for this run to allow token reuse + await drizzle + .delete(Schema.hooks) + .where(eq(Schema.hooks.runId, effectiveRunId)); } - // If transitioning to a terminal status, clean up all hooks for this run - if (isBecomingTerminal) { - await drizzle.delete(Schema.hooks).where(eq(Schema.hooks.runId, id)); + // Handle run_failed event: update run status and cleanup hooks + if (data.eventType === 'run_failed') { + const eventData = (data as any).eventData as { + error: any; + errorCode?: string; + }; + const errorMessage = + typeof eventData.error === 'string' + ? eventData.error + : (eventData.error?.message ?? 'Unknown error'); + // Store structured error as JSON for deserializeRunError to parse + const errorJson = JSON.stringify({ + message: errorMessage, + stack: eventData.error?.stack, + code: eventData.errorCode, + }); + const [runValue] = await drizzle + .update(Schema.runs) + .set({ + status: 'failed', + error: errorJson, + completedAt: now, + updatedAt: now, + }) + .where(eq(Schema.runs.runId, effectiveRunId)) + .returning(); + if (runValue) { + run = deserializeRunError(compact(runValue)); + } + // Delete all hooks for this run to allow token reuse + await drizzle + .delete(Schema.hooks) + .where(eq(Schema.hooks.runId, effectiveRunId)); } - return deserializeRunError(compact(value)); - }, - }; -} + // Handle run_cancelled event: update run status and cleanup hooks + if (data.eventType === 'run_cancelled') { + const [runValue] = await drizzle + .update(Schema.runs) + .set({ + status: 'cancelled', + completedAt: now, + updatedAt: now, + }) + .where(eq(Schema.runs.runId, effectiveRunId)) + .returning(); + if (runValue) { + run = deserializeRunError(compact(runValue)); + } + // Delete all hooks for this run to allow token reuse + await drizzle + .delete(Schema.hooks) + .where(eq(Schema.hooks.runId, effectiveRunId)); + } -function map(obj: T | null | undefined, fn: (v: T) => R): undefined | R { - return obj ? fn(obj) : undefined; -} + // Handle step_created event: create step entity + if (data.eventType === 'step_created') { + const eventData = (data as any).eventData as { + stepName: string; + input: any; + }; + const [stepValue] = await drizzle + .insert(Schema.steps) + .values({ + runId: effectiveRunId, + stepId: data.correlationId!, + stepName: eventData.stepName, + input: eventData.input as SerializedContent, + status: 'pending', + attempt: 0, + }) + .onConflictDoNothing() + .returning(); + if (stepValue) { + step = deserializeStepError(compact(stepValue)); + } + } -export function createEventsStorage(drizzle: Drizzle): Storage['events'] { - const ulid = monotonicFactory(); - const { events } = Schema; + // Handle step_started event: increment attempt, set status to 'running' + // Sets startedAt (maps to startedAt) only on first start + // Reuse validatedStep from validation (already read above) + if (data.eventType === 'step_started') { + const isFirstStart = !validatedStep?.startedAt; + + const [stepValue] = await drizzle + .update(Schema.steps) + .set({ + status: 'running', + // Increment attempt counter using SQL + attempt: sql`${Schema.steps.attempt} + 1`, + // Only set startedAt on first start (not updated on retries) + ...(isFirstStart ? { startedAt: now } : {}), + }) + .where( + and( + eq(Schema.steps.runId, effectiveRunId), + eq(Schema.steps.stepId, data.correlationId!) + ) + ) + .returning(); + if (stepValue) { + step = deserializeStepError(compact(stepValue)); + } + } + + // Handle step_completed event: update step status + // Uses conditional UPDATE to skip validation query (performance optimization) + if (data.eventType === 'step_completed') { + const eventData = (data as any).eventData as { result?: any }; + const [stepValue] = await drizzle + .update(Schema.steps) + .set({ + status: 'completed', + output: eventData.result as SerializedContent | undefined, + completedAt: now, + }) + .where( + and( + eq(Schema.steps.runId, effectiveRunId), + eq(Schema.steps.stepId, data.correlationId!), + // Only update if not already in terminal state (validation in WHERE clause) + notInArray(Schema.steps.status, ['completed', 'failed']) + ) + ) + .returning(); + if (stepValue) { + step = deserializeStepError(compact(stepValue)); + } else { + // Step not updated - check if it exists and why + const [existing] = await getStepForValidation.execute({ + runId: effectiveRunId, + stepId: data.correlationId!, + }); + if (!existing) { + throw new WorkflowAPIError( + `Step "${data.correlationId}" not found`, + { status: 404 } + ); + } + if (['completed', 'failed'].includes(existing.status)) { + throw new WorkflowAPIError( + `Cannot modify step in terminal state "${existing.status}"`, + { status: 409 } + ); + } + } + } + + // Handle step_failed event: terminal state with error + // Uses conditional UPDATE to skip validation query (performance optimization) + if (data.eventType === 'step_failed') { + const eventData = (data as any).eventData as { + error?: any; + stack?: string; + }; + // Store structured error as JSON for deserializeStepError to parse + const errorMessage = + typeof eventData.error === 'string' + ? eventData.error + : (eventData.error?.message ?? 'Unknown error'); + const errorJson = JSON.stringify({ + message: errorMessage, + stack: eventData.stack, + }); + + const [stepValue] = await drizzle + .update(Schema.steps) + .set({ + status: 'failed', + error: errorJson, + completedAt: now, + }) + .where( + and( + eq(Schema.steps.runId, effectiveRunId), + eq(Schema.steps.stepId, data.correlationId!), + // Only update if not already in terminal state (validation in WHERE clause) + notInArray(Schema.steps.status, ['completed', 'failed']) + ) + ) + .returning(); + if (stepValue) { + step = deserializeStepError(compact(stepValue)); + } else { + // Step not updated - check if it exists and why + const [existing] = await getStepForValidation.execute({ + runId: effectiveRunId, + stepId: data.correlationId!, + }); + if (!existing) { + throw new WorkflowAPIError( + `Step "${data.correlationId}" not found`, + { status: 404 } + ); + } + if (['completed', 'failed'].includes(existing.status)) { + throw new WorkflowAPIError( + `Cannot modify step in terminal state "${existing.status}"`, + { status: 409 } + ); + } + } + } + + // Handle step_retrying event: sets status back to 'pending', records error + if (data.eventType === 'step_retrying') { + const eventData = (data as any).eventData as { + error?: any; + stack?: string; + retryAfter?: Date; + }; + // Store error as JSON in 'error' column + const errorMessage = + typeof eventData.error === 'string' + ? eventData.error + : (eventData.error?.message ?? 'Unknown error'); + const errorJson = JSON.stringify({ + message: errorMessage, + stack: eventData.stack, + }); + + const [stepValue] = await drizzle + .update(Schema.steps) + .set({ + status: 'pending', + error: errorJson, + retryAfter: eventData.retryAfter, + }) + .where( + and( + eq(Schema.steps.runId, effectiveRunId), + eq(Schema.steps.stepId, data.correlationId!) + ) + ) + .returning(); + if (stepValue) { + step = deserializeStepError(compact(stepValue)); + } + } + + // Handle hook_created event: create hook entity + // Uses prepared statement for token uniqueness check (performance optimization) + if (data.eventType === 'hook_created') { + const eventData = (data as any).eventData as { + token: string; + metadata?: any; + }; + + // Check for duplicate token using prepared statement + const [existingHook] = await getHookByToken.execute({ + token: eventData.token, + }); + if (existingHook) { + throw new WorkflowAPIError( + `Hook with token ${eventData.token} already exists for this project`, + { status: 409 } + ); + } + + const [hookValue] = await drizzle + .insert(Schema.hooks) + .values({ + runId: effectiveRunId, + hookId: data.correlationId!, + token: eventData.token, + metadata: eventData.metadata as SerializedContent, + ownerId: '', // TODO: get from context + projectId: '', // TODO: get from context + environment: '', // TODO: get from context + }) + .onConflictDoNothing() + .returning(); + if (hookValue) { + hookValue.metadata ||= hookValue.metadataJson; + hook = HookSchema.parse(compact(hookValue)); + } + } + + // Handle hook_disposed event: delete hook entity + if (data.eventType === 'hook_disposed' && data.correlationId) { + await drizzle + .delete(Schema.hooks) + .where(eq(Schema.hooks.hookId, data.correlationId)); + } - return { - async create(runId, data, params) { - const eventId = `wevt_${ulid()}`; const [value] = await drizzle .insert(events) .values({ - runId, + runId: effectiveRunId, eventId, correlationId: data.correlationId, eventType: data.eventType, @@ -371,10 +746,10 @@ export function createEventsStorage(drizzle: Drizzle): Storage['events'] { status: 409, }); } - const result = { ...data, ...value, runId, eventId }; + const result = { ...data, ...value, runId: effectiveRunId, eventId }; const parsed = EventSchema.parse(result); const resolveData = params?.resolveData ?? 'all'; - return filterEventData(parsed, resolveData); + return { event: filterEventData(parsed, resolveData), run, step, hook }; }, async list(params: ListEventsParams): Promise> { const limit = params?.pagination?.limit ?? 100; @@ -468,30 +843,6 @@ export function createHooksStorage(drizzle: Drizzle): Storage['hooks'] { const resolveData = params?.resolveData ?? 'all'; return filterHookData(parsed, resolveData); }, - async create(runId, data, params) { - const [value] = await drizzle - .insert(hooks) - .values({ - runId, - hookId: data.hookId, - token: data.token, - metadata: data.metadata as SerializedContent, - ownerId: '', // TODO: get from context - projectId: '', // TODO: get from context - environment: '', // TODO: get from context - }) - .onConflictDoNothing() - .returning(); - if (!value) { - throw new WorkflowAPIError(`Hook ${data.hookId} already exists`, { - status: 409, - }); - } - value.metadata ||= value.metadataJson; - const parsed = HookSchema.parse(compact(value)); - const resolveData = params?.resolveData ?? 'all'; - return filterHookData(parsed, resolveData); - }, async getByToken(token, params) { const [value] = await getByToken.execute({ token }); if (!value) { @@ -532,20 +883,6 @@ export function createHooksStorage(drizzle: Drizzle): Storage['hooks'] { hasMore, }; }, - async dispose(hookId, params) { - const [value] = await drizzle - .delete(hooks) - .where(eq(hooks.hookId, hookId)) - .returning(); - if (!value) { - throw new WorkflowAPIError(`Hook not found: ${hookId}`, { - status: 404, - }); - } - const parsed = HookSchema.parse(compact(value)); - const resolveData = params?.resolveData ?? 'all'; - return filterHookData(parsed, resolveData); - }, }; } @@ -553,28 +890,6 @@ export function createStepsStorage(drizzle: Drizzle): Storage['steps'] { const { steps } = Schema; return { - async create(runId, data) { - const [value] = await drizzle - .insert(steps) - .values({ - runId, - stepId: data.stepId, - stepName: data.stepName, - input: data.input as SerializedContent, - status: 'pending', - attempt: 0, - }) - .onConflictDoNothing() - .returning(); - - if (!value) { - throw new WorkflowAPIError(`Step ${data.stepId} already exists`, { - status: 409, - }); - } - return deserializeStepError(compact(value)); - }, - async get(runId, stepId, params) { // If runId is not provided, query only by stepId const whereClause = runId @@ -598,47 +913,6 @@ export function createStepsStorage(drizzle: Drizzle): Storage['steps'] { const resolveData = params?.resolveData ?? 'all'; return filterStepData(parsed, resolveData); }, - async update(runId, stepId, data) { - // Fetch current step to check if startedAt is already set - const [currentStep] = await drizzle - .select() - .from(steps) - .where(and(eq(steps.stepId, stepId), eq(steps.runId, runId))) - .limit(1); - - if (!currentStep) { - throw new WorkflowAPIError(`Step not found: ${stepId}`, { - status: 404, - }); - } - - // Serialize the error field if present - const serialized = serializeStepError(data); - - const updates: Partial = { - ...serialized, - output: data.output as SerializedContent, - }; - const now = new Date(); - // Only set startedAt the first time the step transitions to 'running' - if (data.status === 'running' && !currentStep.startedAt) { - updates.startedAt = now; - } - if (data.status === 'completed' || data.status === 'failed') { - updates.completedAt = now; - } - const [value] = await drizzle - .update(steps) - .set(updates) - .where(and(eq(steps.stepId, stepId), eq(steps.runId, runId))) - .returning(); - if (!value) { - throw new WorkflowAPIError(`Step not found: ${stepId}`, { - status: 404, - }); - } - return deserializeStepError(compact(value)); - }, async list(params) { const limit = params?.pagination?.limit ?? 20; const fromCursor = params?.pagination?.cursor; diff --git a/packages/world-postgres/test/storage.test.ts b/packages/world-postgres/test/storage.test.ts index b28ab5a83..7812205e1 100644 --- a/packages/world-postgres/test/storage.test.ts +++ b/packages/world-postgres/test/storage.test.ts @@ -1,5 +1,6 @@ import { execSync } from 'node:child_process'; import { PostgreSqlContainer } from '@testcontainers/postgresql'; +import type { Hook, Step, WorkflowRun } from '@workflow/world'; import postgres from 'postgres'; import { afterAll, @@ -17,6 +18,103 @@ import { createStepsStorage, } from '../src/storage.js'; +// Helper types for events storage +type EventsStorage = ReturnType; + +// Helper functions to create entities through events.create +async function createRun( + events: EventsStorage, + data: { + deploymentId: string; + workflowName: string; + input: unknown[]; + executionContext?: Record; + } +): Promise { + const result = await events.create(null, { + eventType: 'run_created', + eventData: data, + }); + if (!result.run) { + throw new Error('Expected run to be created'); + } + return result.run; +} + +async function updateRun( + events: EventsStorage, + runId: string, + eventType: 'run_started' | 'run_completed' | 'run_failed', + eventData?: Record +): Promise { + const result = await events.create(runId, { + eventType, + eventData, + }); + if (!result.run) { + throw new Error('Expected run to be updated'); + } + return result.run; +} + +async function createStep( + events: EventsStorage, + runId: string, + data: { + stepId: string; + stepName: string; + input: unknown[]; + } +): Promise { + const result = await events.create(runId, { + eventType: 'step_created', + correlationId: data.stepId, + eventData: { stepName: data.stepName, input: data.input }, + }); + if (!result.step) { + throw new Error('Expected step to be created'); + } + return result.step; +} + +async function updateStep( + events: EventsStorage, + runId: string, + stepId: string, + eventType: 'step_started' | 'step_completed' | 'step_failed', + eventData?: Record +): Promise { + const result = await events.create(runId, { + eventType, + correlationId: stepId, + eventData, + }); + if (!result.step) { + throw new Error('Expected step to be updated'); + } + return result.step; +} + +async function createHook( + events: EventsStorage, + runId: string, + data: { + hookId: string; + token: string; + metadata?: unknown; + } +): Promise { + const result = await events.create(runId, { + eventType: 'hook_created', + correlationId: data.hookId, + eventData: { token: data.token, metadata: data.metadata }, + }); + if (!result.hook) { + throw new Error('Expected hook to be created'); + } + return result.hook; +} + describe('Storage (Postgres integration)', () => { if (process.platform === 'win32') { test.skip('skipped on Windows since it relies on a docker container', () => {}); @@ -75,7 +173,7 @@ describe('Storage (Postgres integration)', () => { input: ['arg1', 'arg2'], }; - const run = await runs.create(runData); + const run = await createRun(events, runData); expect(run.runId).toMatch(/^wrun_/); expect(run.deploymentId).toBe('deployment-123'); @@ -98,7 +196,7 @@ describe('Storage (Postgres integration)', () => { input: [], }; - const run = await runs.create(runData); + const run = await createRun(events, runData); expect(run.executionContext).toBeUndefined(); expect(run.input).toEqual([]); @@ -107,7 +205,7 @@ describe('Storage (Postgres integration)', () => { describe('get', () => { it('should retrieve an existing run', async () => { - const created = await runs.create({ + const created = await createRun(events, { deploymentId: 'deployment-123', workflowName: 'test-workflow', input: ['arg'], @@ -126,72 +224,59 @@ describe('Storage (Postgres integration)', () => { }); }); - describe('update', () => { - it('should update run status to running', async () => { - const created = await runs.create({ + describe('update via events', () => { + it('should update run status to running via run_started event', async () => { + const created = await createRun(events, { deploymentId: 'deployment-123', workflowName: 'test-workflow', input: [], }); - const updated = await runs.update(created.runId, { - status: 'running', - }); + const updated = await updateRun(events, created.runId, 'run_started'); expect(updated.status).toBe('running'); expect(updated.startedAt).toBeInstanceOf(Date); }); - it('should update run status to completed', async () => { - const created = await runs.create({ + it('should update run status to completed via run_completed event', async () => { + const created = await createRun(events, { deploymentId: 'deployment-123', workflowName: 'test-workflow', input: [], }); - const updated = await runs.update(created.runId, { - status: 'completed', - output: [{ result: 42 }], - }); + const updated = await updateRun( + events, + created.runId, + 'run_completed', + { + output: [{ result: 42 }], + } + ); expect(updated.status).toBe('completed'); expect(updated.completedAt).toBeInstanceOf(Date); expect(updated.output).toEqual([{ result: 42 }]); }); - it('should update run status to failed', async () => { - const created = await runs.create({ + it('should update run status to failed via run_failed event', async () => { + const created = await createRun(events, { deploymentId: 'deployment-123', workflowName: 'test-workflow', input: [], }); - const updated = await runs.update(created.runId, { - status: 'failed', - error: { - message: 'Something went wrong', - code: 'ERR_001', - }, + const updated = await updateRun(events, created.runId, 'run_failed', { + error: 'Something went wrong', }); expect(updated.status).toBe('failed'); - expect(updated.error).toEqual({ - message: 'Something went wrong', - code: 'ERR_001', - }); + expect(updated.error?.message).toBe('Something went wrong'); expect(updated.completedAt).toBeInstanceOf(Date); }); - - it('should throw error for non-existent run', async () => { - await expect( - runs.update('missing', { status: 'running' }) - ).rejects.toMatchObject({ - status: 404, - }); - }); }); describe('list', () => { it('should list all runs', async () => { - const run1 = await runs.create({ + const run1 = await createRun(events, { deploymentId: 'deployment-1', workflowName: 'workflow-1', input: [], @@ -200,7 +285,7 @@ describe('Storage (Postgres integration)', () => { // Small delay to ensure different timestamps in createdAt await new Promise((resolve) => setTimeout(resolve, 2)); - const run2 = await runs.create({ + const run2 = await createRun(events, { deploymentId: 'deployment-2', workflowName: 'workflow-2', input: [], @@ -218,12 +303,12 @@ describe('Storage (Postgres integration)', () => { }); it('should filter runs by workflowName', async () => { - await runs.create({ + await createRun(events, { deploymentId: 'deployment-1', workflowName: 'workflow-1', input: [], }); - const run2 = await runs.create({ + const run2 = await createRun(events, { deploymentId: 'deployment-2', workflowName: 'workflow-2', input: [], @@ -238,7 +323,7 @@ describe('Storage (Postgres integration)', () => { it('should support pagination', async () => { // Create multiple runs for (let i = 0; i < 5; i++) { - await runs.create({ + await createRun(events, { deploymentId: `deployment-${i}`, workflowName: `workflow-${i}`, input: [], @@ -260,58 +345,13 @@ describe('Storage (Postgres integration)', () => { expect(page2.data[0].runId).not.toBe(page1.data[0].runId); }); }); - - describe('cancel', () => { - it('should cancel a run', async () => { - const created = await runs.create({ - deploymentId: 'deployment-123', - workflowName: 'test-workflow', - input: [], - }); - - const cancelled = await runs.cancel(created.runId); - - expect(cancelled.status).toBe('cancelled'); - expect(cancelled.completedAt).toBeInstanceOf(Date); - }); - }); - - describe('pause', () => { - it('should pause a run', async () => { - const created = await runs.create({ - deploymentId: 'deployment-123', - workflowName: 'test-workflow', - input: [], - }); - - const paused = await runs.pause(created.runId); - - expect(paused.status).toBe('paused'); - }); - }); - - describe('resume', () => { - it('should resume a paused run', async () => { - const created = await runs.create({ - deploymentId: 'deployment-123', - workflowName: 'test-workflow', - input: [], - }); - - await runs.pause(created.runId); - const resumed = await runs.resume(created.runId); - - expect(resumed.status).toBe('running'); - expect(resumed.startedAt).toBeInstanceOf(Date); - }); - }); }); describe('steps', () => { let testRunId: string; beforeEach(async () => { - const run = await runs.create({ + const run = await createRun(events, { deploymentId: 'deployment-123', workflowName: 'test-workflow', input: [], @@ -327,7 +367,7 @@ describe('Storage (Postgres integration)', () => { input: ['input1', 'input2'], }; - const step = await steps.create(testRunId, stepData); + const step = await createStep(events, testRunId, stepData); expect(step).toEqual({ runId: testRunId, @@ -348,7 +388,7 @@ describe('Storage (Postgres integration)', () => { describe('get', () => { it('should retrieve a step with runId and stepId', async () => { - const created = await steps.create(testRunId, { + const created = await createStep(events, testRunId, { stepId: 'step-123', stepName: 'test-step', input: ['input1'], @@ -360,7 +400,7 @@ describe('Storage (Postgres integration)', () => { }); it('should retrieve a step with only stepId', async () => { - const created = await steps.create(testRunId, { + const created = await createStep(events, testRunId, { stepId: 'unique-step-123', stepName: 'test-step', input: ['input1'], @@ -378,83 +418,76 @@ describe('Storage (Postgres integration)', () => { }); }); - describe('update', () => { - it('should update step status to running', async () => { - await steps.create(testRunId, { + describe('update via events', () => { + it('should update step status to running via step_started event', async () => { + await createStep(events, testRunId, { stepId: 'step-123', stepName: 'test-step', input: ['input1'], }); - const updated = await steps.update(testRunId, 'step-123', { - status: 'running', - }); + const updated = await updateStep( + events, + testRunId, + 'step-123', + 'step_started', + {} // step_started no longer needs attempt in eventData - World increments it + ); expect(updated.status).toBe('running'); expect(updated.startedAt).toBeInstanceOf(Date); + expect(updated.attempt).toBe(1); // Incremented by step_started }); - it('should update step status to completed', async () => { - await steps.create(testRunId, { + it('should update step status to completed via step_completed event', async () => { + await createStep(events, testRunId, { stepId: 'step-123', stepName: 'test-step', input: ['input1'], }); - const updated = await steps.update(testRunId, 'step-123', { - status: 'completed', - output: ['ok'], - }); + const updated = await updateStep( + events, + testRunId, + 'step-123', + 'step_completed', + { result: ['ok'] } + ); expect(updated.status).toBe('completed'); expect(updated.completedAt).toBeInstanceOf(Date); expect(updated.output).toEqual(['ok']); }); - it('should update step status to failed', async () => { - await steps.create(testRunId, { + it('should update step status to failed via step_failed event', async () => { + await createStep(events, testRunId, { stepId: 'step-123', stepName: 'test-step', input: ['input1'], }); - const updated = await steps.update(testRunId, 'step-123', { - status: 'failed', - error: { - message: 'Step failed', - code: 'STEP_ERR', - }, - }); + const updated = await updateStep( + events, + testRunId, + 'step-123', + 'step_failed', + { error: 'Step failed' } + ); expect(updated.status).toBe('failed'); expect(updated.error?.message).toBe('Step failed'); - expect(updated.error?.code).toBe('STEP_ERR'); expect(updated.completedAt).toBeInstanceOf(Date); }); - - it('should update attempt count', async () => { - await steps.create(testRunId, { - stepId: 'step-123', - stepName: 'test-step', - input: ['input1'], - }); - - const updated = await steps.update(testRunId, 'step-123', { - attempt: 2, - }); - - expect(updated.attempt).toBe(2); - }); }); describe('list', () => { it('should list all steps for a run', async () => { - const step1 = await steps.create(testRunId, { + const step1 = await createStep(events, testRunId, { stepId: 'step-1', stepName: 'first-step', input: [], }); - const step2 = await steps.create(testRunId, { + const step2 = await createStep(events, testRunId, { stepId: 'step-2', stepName: 'second-step', input: [], @@ -476,7 +509,7 @@ describe('Storage (Postgres integration)', () => { it('should support pagination', async () => { // Create multiple steps for (let i = 0; i < 5; i++) { - await steps.create(testRunId, { + await createStep(events, testRunId, { stepId: `step-${i}`, stepName: `step-name-${i}`, input: [], @@ -506,7 +539,7 @@ describe('Storage (Postgres integration)', () => { let testRunId: string; beforeEach(async () => { - const run = await runs.create({ + const run = await createRun(events, { deploymentId: 'deployment-123', workflowName: 'test-workflow', input: [], @@ -516,32 +549,50 @@ describe('Storage (Postgres integration)', () => { describe('create', () => { it('should create a new event', async () => { + // Create step before step_started event + await createStep(events, testRunId, { + stepId: 'corr_123', + stepName: 'test-step', + input: [], + }); + const eventData = { eventType: 'step_started' as const, correlationId: 'corr_123', }; - const event = await events.create(testRunId, eventData); + const result = await events.create(testRunId, eventData); - expect(event.runId).toBe(testRunId); - expect(event.eventId).toMatch(/^wevt_/); - expect(event.eventType).toBe('step_started'); - expect(event.correlationId).toBe('corr_123'); - expect(event.createdAt).toBeInstanceOf(Date); + expect(result.event.runId).toBe(testRunId); + expect(result.event.eventId).toMatch(/^wevt_/); + expect(result.event.eventType).toBe('step_started'); + expect(result.event.correlationId).toBe('corr_123'); + expect(result.event.createdAt).toBeInstanceOf(Date); }); it('should create a new event with null byte in payload', async () => { - const event = await events.create(testRunId, { + // Create step before step_failed event + await createStep(events, testRunId, { + stepId: 'corr_123_null', + stepName: 'test-step-null', + input: [], + }); + await events.create(testRunId, { + eventType: 'step_started', + correlationId: 'corr_123_null', + }); + + const result = await events.create(testRunId, { eventType: 'step_failed', - correlationId: 'corr_123', + correlationId: 'corr_123_null', eventData: { error: 'Error with null byte \u0000 in message' }, }); - expect(event.runId).toBe(testRunId); - expect(event.eventId).toMatch(/^wevt_/); - expect(event.eventType).toBe('step_failed'); - expect(event.correlationId).toBe('corr_123'); - expect(event.createdAt).toBeInstanceOf(Date); + expect(result.event.runId).toBe(testRunId); + expect(result.event.eventId).toMatch(/^wevt_/); + expect(result.event.eventType).toBe('step_failed'); + expect(result.event.correlationId).toBe('corr_123_null'); + expect(result.event.createdAt).toBeInstanceOf(Date); }); it('should handle workflow completed events', async () => { @@ -549,23 +600,30 @@ describe('Storage (Postgres integration)', () => { eventType: 'workflow_completed' as const, }; - const event = await events.create(testRunId, eventData); + const result = await events.create(testRunId, eventData); - expect(event.eventType).toBe('workflow_completed'); - expect(event.correlationId).toBeUndefined(); + expect(result.event.eventType).toBe('workflow_completed'); + expect(result.event.correlationId).toBeUndefined(); }); }); describe('list', () => { it('should list all events for a run', async () => { - const event1 = await events.create(testRunId, { + const result1 = await events.create(testRunId, { eventType: 'workflow_started' as const, }); // Small delay to ensure different timestamps in event IDs await new Promise((resolve) => setTimeout(resolve, 2)); - const event2 = await events.create(testRunId, { + // Create step before step_started event + await createStep(events, testRunId, { + stepId: 'corr-step-1', + stepName: 'test-step', + input: [], + }); + + const result2 = await events.create(testRunId, { eventType: 'step_started' as const, correlationId: 'corr-step-1', }); @@ -575,24 +633,33 @@ describe('Storage (Postgres integration)', () => { pagination: { sortOrder: 'asc' }, // Explicitly request ascending order }); - expect(result.data).toHaveLength(2); + // 4 events: run_created (from createRun), workflow_started, step_created, step_started + expect(result.data).toHaveLength(4); // Should be in chronological order (oldest first) - expect(result.data[0].eventId).toBe(event1.eventId); - expect(result.data[1].eventId).toBe(event2.eventId); - expect(result.data[1].createdAt.getTime()).toBeGreaterThanOrEqual( - result.data[0].createdAt.getTime() + expect(result.data[0].eventType).toBe('run_created'); + expect(result.data[1].eventId).toBe(result1.event.eventId); + expect(result.data[3].eventId).toBe(result2.event.eventId); + expect(result.data[3].createdAt.getTime()).toBeGreaterThanOrEqual( + result.data[1].createdAt.getTime() ); }); it('should list events in descending order when explicitly requested (newest first)', async () => { - const event1 = await events.create(testRunId, { + const result1 = await events.create(testRunId, { eventType: 'workflow_started' as const, }); // Small delay to ensure different timestamps in event IDs await new Promise((resolve) => setTimeout(resolve, 2)); - const event2 = await events.create(testRunId, { + // Create step before step_started event + await createStep(events, testRunId, { + stepId: 'corr-step-1', + stepName: 'test-step', + input: [], + }); + + const result2 = await events.create(testRunId, { eventType: 'step_started' as const, correlationId: 'corr-step-1', }); @@ -602,18 +669,31 @@ describe('Storage (Postgres integration)', () => { pagination: { sortOrder: 'desc' }, }); - expect(result.data).toHaveLength(2); + // 4 events: run_created (from createRun), workflow_started, step_created, step_started + expect(result.data).toHaveLength(4); // Should be in reverse chronological order (newest first) - expect(result.data[0].eventId).toBe(event2.eventId); - expect(result.data[1].eventId).toBe(event1.eventId); + expect(result.data[0].eventId).toBe(result2.event.eventId); + expect(result.data[1].eventType).toBe('step_created'); + expect(result.data[2].eventId).toBe(result1.event.eventId); + expect(result.data[3].eventType).toBe('run_created'); expect(result.data[0].createdAt.getTime()).toBeGreaterThanOrEqual( - result.data[1].createdAt.getTime() + result.data[2].createdAt.getTime() ); }); it('should support pagination', async () => { - // Create multiple events + // Create multiple events - must create steps first for (let i = 0; i < 5; i++) { + await createStep(events, testRunId, { + stepId: `corr_${i}`, + stepName: `test-step-${i}`, + input: [], + }); + // Start the step before completing + await events.create(testRunId, { + eventType: 'step_started', + correlationId: `corr_${i}`, + }); await events.create(testRunId, { eventType: 'step_completed', correlationId: `corr_${i}`, @@ -643,21 +723,33 @@ describe('Storage (Postgres integration)', () => { it('should list all events with a specific correlation ID', async () => { const correlationId = 'step-abc123'; + // Create step before step events + await createStep(events, testRunId, { + stepId: correlationId, + stepName: 'test-step', + input: [], + }); + // Create events with the target correlation ID - const event1 = await events.create(testRunId, { + const result1 = await events.create(testRunId, { eventType: 'step_started', correlationId, }); await new Promise((resolve) => setTimeout(resolve, 2)); - const event2 = await events.create(testRunId, { + const result2 = await events.create(testRunId, { eventType: 'step_completed', correlationId, eventData: { result: 'success' }, }); // Create events with different correlation IDs (should be filtered out) + await createStep(events, testRunId, { + stepId: 'different-step', + stepName: 'different-step', + input: [], + }); await events.create(testRunId, { eventType: 'step_started', correlationId: 'different-step', @@ -671,32 +763,35 @@ describe('Storage (Postgres integration)', () => { pagination: {}, }); - expect(result.data).toHaveLength(2); - expect(result.data[0].eventId).toBe(event1.eventId); - expect(result.data[0].correlationId).toBe(correlationId); - expect(result.data[1].eventId).toBe(event2.eventId); + // 3 events: step_created, step_started, step_completed + expect(result.data).toHaveLength(3); + expect(result.data[0].eventType).toBe('step_created'); + expect(result.data[1].eventId).toBe(result1.event.eventId); expect(result.data[1].correlationId).toBe(correlationId); + expect(result.data[2].eventId).toBe(result2.event.eventId); + expect(result.data[2].correlationId).toBe(correlationId); }); it('should list events across multiple runs with same correlation ID', async () => { const correlationId = 'hook-xyz789'; // Create another run - const run2 = await runs.create({ + const run2 = await createRun(events, { deploymentId: 'deployment-456', workflowName: 'test-workflow-2', input: [], }); // Create events in both runs with same correlation ID - const event1 = await events.create(testRunId, { + const result1 = await events.create(testRunId, { eventType: 'hook_created', correlationId, + eventData: { token: 'test-token-1' }, }); await new Promise((resolve) => setTimeout(resolve, 2)); - const event2 = await events.create(run2.runId, { + const result2 = await events.create(run2.runId, { eventType: 'hook_received', correlationId, eventData: { payload: { data: 'test' } }, @@ -704,7 +799,7 @@ describe('Storage (Postgres integration)', () => { await new Promise((resolve) => setTimeout(resolve, 2)); - const event3 = await events.create(testRunId, { + const result3 = await events.create(testRunId, { eventType: 'hook_disposed', correlationId, }); @@ -715,15 +810,21 @@ describe('Storage (Postgres integration)', () => { }); expect(result.data).toHaveLength(3); - expect(result.data[0].eventId).toBe(event1.eventId); + expect(result.data[0].eventId).toBe(result1.event.eventId); expect(result.data[0].runId).toBe(testRunId); - expect(result.data[1].eventId).toBe(event2.eventId); + expect(result.data[1].eventId).toBe(result2.event.eventId); expect(result.data[1].runId).toBe(run2.runId); - expect(result.data[2].eventId).toBe(event3.eventId); + expect(result.data[2].eventId).toBe(result3.event.eventId); expect(result.data[2].runId).toBe(testRunId); }); it('should return empty list for non-existent correlation ID', async () => { + // Create a step and start it + await createStep(events, testRunId, { + stepId: 'existing-step', + stepName: 'existing-step', + input: [], + }); await events.create(testRunId, { eventType: 'step_started', correlationId: 'existing-step', @@ -742,6 +843,13 @@ describe('Storage (Postgres integration)', () => { it('should respect pagination parameters', async () => { const correlationId = 'step_paginated'; + // Create step first + await createStep(events, testRunId, { + stepId: correlationId, + stepName: 'test-step', + input: [], + }); + // Create multiple events await events.create(testRunId, { eventType: 'step_started', @@ -753,7 +861,15 @@ describe('Storage (Postgres integration)', () => { await events.create(testRunId, { eventType: 'step_retrying', correlationId, - eventData: { attempt: 1 }, + eventData: { error: 'retry error' }, + }); + + await new Promise((resolve) => setTimeout(resolve, 2)); + + // Start again after retry + await events.create(testRunId, { + eventType: 'step_started', + correlationId, }); await new Promise((resolve) => setTimeout(resolve, 2)); @@ -764,27 +880,38 @@ describe('Storage (Postgres integration)', () => { eventData: { result: 'success' }, }); - // Get first page + // Get first page (step_created, step_started, step_retrying) const page1 = await events.listByCorrelationId({ correlationId, - pagination: { limit: 2 }, + pagination: { limit: 3 }, }); - expect(page1.data).toHaveLength(2); + expect(page1.data).toHaveLength(3); expect(page1.hasMore).toBe(true); expect(page1.cursor).toBeDefined(); - // Get second page + // Get second page (step_started, step_completed) const page2 = await events.listByCorrelationId({ correlationId, - pagination: { limit: 2, cursor: page1.cursor || undefined }, + pagination: { limit: 3, cursor: page1.cursor || undefined }, }); - expect(page2.data).toHaveLength(1); + expect(page2.data).toHaveLength(2); expect(page2.hasMore).toBe(false); }); it('should always return full event data', async () => { + // Create step first + await createStep(events, testRunId, { + stepId: 'step-with-data', + stepName: 'step-with-data', + input: [], + }); + // Start the step before completing + await events.create(testRunId, { + eventType: 'step_started', + correlationId: 'step-with-data', + }); await events.create(testRunId, { eventType: 'step_completed', correlationId: 'step-with-data', @@ -797,22 +924,30 @@ describe('Storage (Postgres integration)', () => { pagination: {}, }); - expect(result.data).toHaveLength(1); - expect(result.data[0].correlationId).toBe('step-with-data'); + // 3 events: step_created, step_started, step_completed + expect(result.data).toHaveLength(3); + expect(result.data[2].correlationId).toBe('step-with-data'); }); it('should return events in ascending order by default', async () => { const correlationId = 'step-ordering'; + // Create step first + await createStep(events, testRunId, { + stepId: correlationId, + stepName: 'test-step', + input: [], + }); + // Create events with slight delays to ensure different timestamps - const event1 = await events.create(testRunId, { + const result1 = await events.create(testRunId, { eventType: 'step_started', correlationId, }); await new Promise((resolve) => setTimeout(resolve, 2)); - const event2 = await events.create(testRunId, { + const result2 = await events.create(testRunId, { eventType: 'step_completed', correlationId, eventData: { result: 'success' }, @@ -823,25 +958,33 @@ describe('Storage (Postgres integration)', () => { pagination: {}, }); - expect(result.data).toHaveLength(2); - expect(result.data[0].eventId).toBe(event1.eventId); - expect(result.data[1].eventId).toBe(event2.eventId); - expect(result.data[0].createdAt.getTime()).toBeLessThanOrEqual( - result.data[1].createdAt.getTime() + // 3 events: step_created, step_started, step_completed + expect(result.data).toHaveLength(3); + expect(result.data[1].eventId).toBe(result1.event.eventId); + expect(result.data[2].eventId).toBe(result2.event.eventId); + expect(result.data[1].createdAt.getTime()).toBeLessThanOrEqual( + result.data[2].createdAt.getTime() ); }); it('should support descending order', async () => { const correlationId = 'step-desc-order'; - const event1 = await events.create(testRunId, { + // Create step first + await createStep(events, testRunId, { + stepId: correlationId, + stepName: 'test-step', + input: [], + }); + + const result1 = await events.create(testRunId, { eventType: 'step_started', correlationId, }); await new Promise((resolve) => setTimeout(resolve, 2)); - const event2 = await events.create(testRunId, { + const result2 = await events.create(testRunId, { eventType: 'step_completed', correlationId, eventData: { result: 'success' }, @@ -852,9 +995,10 @@ describe('Storage (Postgres integration)', () => { pagination: { sortOrder: 'desc' }, }); - expect(result.data).toHaveLength(2); - expect(result.data[0].eventId).toBe(event2.eventId); - expect(result.data[1].eventId).toBe(event1.eventId); + // 3 events in descending order: step_completed, step_started, step_created + expect(result.data).toHaveLength(3); + expect(result.data[0].eventId).toBe(result2.event.eventId); + expect(result.data[1].eventId).toBe(result1.event.eventId); expect(result.data[0].createdAt.getTime()).toBeGreaterThanOrEqual( result.data[1].createdAt.getTime() ); @@ -864,14 +1008,15 @@ describe('Storage (Postgres integration)', () => { const hookId = 'hook_test123'; // Create a typical hook lifecycle - const created = await events.create(testRunId, { + const createdResult = await events.create(testRunId, { eventType: 'hook_created' as const, correlationId: hookId, + eventData: { token: 'lifecycle-test-token' }, }); await new Promise((resolve) => setTimeout(resolve, 2)); - const received1 = await events.create(testRunId, { + const received1Result = await events.create(testRunId, { eventType: 'hook_received' as const, correlationId: hookId, eventData: { payload: { request: 1 } }, @@ -879,7 +1024,7 @@ describe('Storage (Postgres integration)', () => { await new Promise((resolve) => setTimeout(resolve, 2)); - const received2 = await events.create(testRunId, { + const received2Result = await events.create(testRunId, { eventType: 'hook_received' as const, correlationId: hookId, eventData: { payload: { request: 2 } }, @@ -887,7 +1032,7 @@ describe('Storage (Postgres integration)', () => { await new Promise((resolve) => setTimeout(resolve, 2)); - const disposed = await events.create(testRunId, { + const disposedResult = await events.create(testRunId, { eventType: 'hook_disposed' as const, correlationId: hookId, }); @@ -898,15 +1043,892 @@ describe('Storage (Postgres integration)', () => { }); expect(result.data).toHaveLength(4); - expect(result.data[0].eventId).toBe(created.eventId); + expect(result.data[0].eventId).toBe(createdResult.event.eventId); expect(result.data[0].eventType).toBe('hook_created'); - expect(result.data[1].eventId).toBe(received1.eventId); + expect(result.data[1].eventId).toBe(received1Result.event.eventId); expect(result.data[1].eventType).toBe('hook_received'); - expect(result.data[2].eventId).toBe(received2.eventId); + expect(result.data[2].eventId).toBe(received2Result.event.eventId); expect(result.data[2].eventType).toBe('hook_received'); - expect(result.data[3].eventId).toBe(disposed.eventId); + expect(result.data[3].eventId).toBe(disposedResult.event.eventId); expect(result.data[3].eventType).toBe('hook_disposed'); }); + + it('should enforce token uniqueness across different runs', async () => { + const token = 'unique-token-test'; + + // Create first hook with the token + await events.create(testRunId, { + eventType: 'hook_created' as const, + correlationId: 'hook_1', + eventData: { token }, + }); + + // Create another run + const run2 = await createRun(events, { + deploymentId: 'deployment-456', + workflowName: 'test-workflow-2', + input: [], + }); + + // Try to create another hook with the same token - should fail + await expect( + events.create(run2.runId, { + eventType: 'hook_created' as const, + correlationId: 'hook_2', + eventData: { token }, + }) + ).rejects.toThrow( + `Hook with token ${token} already exists for this project` + ); + }); + + it('should allow token reuse after hook is disposed', async () => { + const token = 'reusable-token-test'; + + // Create first hook with the token + await events.create(testRunId, { + eventType: 'hook_created' as const, + correlationId: 'hook_reuse_1', + eventData: { token }, + }); + + // Dispose the first hook + await events.create(testRunId, { + eventType: 'hook_disposed' as const, + correlationId: 'hook_reuse_1', + }); + + // Create another run + const run2 = await createRun(events, { + deploymentId: 'deployment-789', + workflowName: 'test-workflow-3', + input: [], + }); + + // Now creating a hook with the same token should succeed + const result = await events.create(run2.runId, { + eventType: 'hook_created' as const, + correlationId: 'hook_reuse_2', + eventData: { token }, + }); + + expect(result.hook).toBeDefined(); + expect(result.hook!.token).toBe(token); + }); + }); + }); + + describe('step terminal state validation', () => { + let testRunId: string; + + beforeEach(async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + testRunId = run.runId; + }); + + describe('completed step', () => { + it('should reject step_started on completed step', async () => { + await createStep(events, testRunId, { + stepId: 'step_terminal_1', + stepName: 'test-step', + input: [], + }); + await updateStep( + events, + testRunId, + 'step_terminal_1', + 'step_completed', + { + result: 'done', + } + ); + + await expect( + updateStep(events, testRunId, 'step_terminal_1', 'step_started') + ).rejects.toThrow(/terminal/i); + }); + + it('should reject step_completed on already completed step', async () => { + await createStep(events, testRunId, { + stepId: 'step_terminal_2', + stepName: 'test-step', + input: [], + }); + await updateStep( + events, + testRunId, + 'step_terminal_2', + 'step_completed', + { + result: 'done', + } + ); + + await expect( + updateStep(events, testRunId, 'step_terminal_2', 'step_completed', { + result: 'done again', + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject step_failed on completed step', async () => { + await createStep(events, testRunId, { + stepId: 'step_terminal_3', + stepName: 'test-step', + input: [], + }); + await updateStep( + events, + testRunId, + 'step_terminal_3', + 'step_completed', + { + result: 'done', + } + ); + + await expect( + updateStep(events, testRunId, 'step_terminal_3', 'step_failed', { + error: 'Should not work', + }) + ).rejects.toThrow(/terminal/i); + }); + }); + + describe('failed step', () => { + it('should reject step_started on failed step', async () => { + await createStep(events, testRunId, { + stepId: 'step_failed_1', + stepName: 'test-step', + input: [], + }); + await updateStep(events, testRunId, 'step_failed_1', 'step_failed', { + error: 'Failed permanently', + }); + + await expect( + updateStep(events, testRunId, 'step_failed_1', 'step_started') + ).rejects.toThrow(/terminal/i); + }); + + it('should reject step_completed on failed step', async () => { + await createStep(events, testRunId, { + stepId: 'step_failed_2', + stepName: 'test-step', + input: [], + }); + await updateStep(events, testRunId, 'step_failed_2', 'step_failed', { + error: 'Failed permanently', + }); + + await expect( + updateStep(events, testRunId, 'step_failed_2', 'step_completed', { + result: 'Should not work', + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject step_failed on already failed step', async () => { + await createStep(events, testRunId, { + stepId: 'step_failed_3', + stepName: 'test-step', + input: [], + }); + await updateStep(events, testRunId, 'step_failed_3', 'step_failed', { + error: 'Failed once', + }); + + await expect( + updateStep(events, testRunId, 'step_failed_3', 'step_failed', { + error: 'Failed again', + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject step_retrying on failed step', async () => { + await createStep(events, testRunId, { + stepId: 'step_failed_retry', + stepName: 'test-step', + input: [], + }); + await updateStep( + events, + testRunId, + 'step_failed_retry', + 'step_failed', + { + error: 'Failed permanently', + } + ); + + await expect( + updateStep(events, testRunId, 'step_failed_retry', 'step_retrying', { + error: 'Retry attempt', + }) + ).rejects.toThrow(/terminal/i); + }); + }); + + describe('step_retrying validation', () => { + it('should reject step_retrying on completed step', async () => { + await createStep(events, testRunId, { + stepId: 'step_completed_retry', + stepName: 'test-step', + input: [], + }); + await updateStep( + events, + testRunId, + 'step_completed_retry', + 'step_completed', + { + result: 'done', + } + ); + + await expect( + updateStep( + events, + testRunId, + 'step_completed_retry', + 'step_retrying', + { + error: 'Retry attempt', + } + ) + ).rejects.toThrow(/terminal/i); + }); + }); + }); + + describe('run terminal state validation', () => { + describe('completed run', () => { + it('should reject run_started on completed run', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await updateRun(events, run.runId, 'run_completed', { output: 'done' }); + + await expect( + updateRun(events, run.runId, 'run_started') + ).rejects.toThrow(/terminal/i); + }); + + it('should reject run_failed on completed run', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await updateRun(events, run.runId, 'run_completed', { output: 'done' }); + + await expect( + updateRun(events, run.runId, 'run_failed', { + error: 'Should not work', + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject run_cancelled on completed run', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await updateRun(events, run.runId, 'run_completed', { output: 'done' }); + + await expect( + events.create(run.runId, { eventType: 'run_cancelled' }) + ).rejects.toThrow(/terminal/i); + }); + }); + + describe('failed run', () => { + it('should reject run_started on failed run', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await updateRun(events, run.runId, 'run_failed', { error: 'Failed' }); + + await expect( + updateRun(events, run.runId, 'run_started') + ).rejects.toThrow(/terminal/i); + }); + + it('should reject run_completed on failed run', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await updateRun(events, run.runId, 'run_failed', { error: 'Failed' }); + + await expect( + updateRun(events, run.runId, 'run_completed', { + output: 'Should not work', + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject run_cancelled on failed run', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await updateRun(events, run.runId, 'run_failed', { error: 'Failed' }); + + await expect( + events.create(run.runId, { eventType: 'run_cancelled' }) + ).rejects.toThrow(/terminal/i); + }); + }); + + describe('cancelled run', () => { + it('should reject run_started on cancelled run', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await events.create(run.runId, { eventType: 'run_cancelled' }); + + await expect( + updateRun(events, run.runId, 'run_started') + ).rejects.toThrow(/terminal/i); + }); + + it('should reject run_completed on cancelled run', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await events.create(run.runId, { eventType: 'run_cancelled' }); + + await expect( + updateRun(events, run.runId, 'run_completed', { + output: 'Should not work', + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject run_failed on cancelled run', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await events.create(run.runId, { eventType: 'run_cancelled' }); + + await expect( + updateRun(events, run.runId, 'run_failed', { + error: 'Should not work', + }) + ).rejects.toThrow(/terminal/i); + }); + }); + }); + + describe('allowed operations on terminal runs', () => { + it('should allow step_completed on completed run for in-progress step', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + + // Create and start a step (making it in-progress) + await createStep(events, run.runId, { + stepId: 'step_in_progress', + stepName: 'test-step', + input: [], + }); + await updateStep(events, run.runId, 'step_in_progress', 'step_started'); + + // Complete the run while step is still running + await updateRun(events, run.runId, 'run_completed', { output: 'done' }); + + // Should succeed - completing an in-progress step on a terminal run is allowed + const result = await updateStep( + events, + run.runId, + 'step_in_progress', + 'step_completed', + { result: 'step done' } + ); + expect(result.status).toBe('completed'); + }); + + it('should allow step_failed on completed run for in-progress step', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + + // Create and start a step + await createStep(events, run.runId, { + stepId: 'step_in_progress_fail', + stepName: 'test-step', + input: [], + }); + await updateStep( + events, + run.runId, + 'step_in_progress_fail', + 'step_started' + ); + + // Complete the run + await updateRun(events, run.runId, 'run_completed', { output: 'done' }); + + // Should succeed - failing an in-progress step on a terminal run is allowed + const result = await updateStep( + events, + run.runId, + 'step_in_progress_fail', + 'step_failed', + { error: 'step failed' } + ); + expect(result.status).toBe('failed'); + }); + + it('should auto-delete hooks when run completes (postgres-specific behavior)', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + + // Create a hook + await createHook(events, run.runId, { + hookId: 'hook_auto_deleted', + token: 'test-token-dispose', + }); + + // Complete the run - this auto-deletes the hook + await updateRun(events, run.runId, 'run_completed', { output: 'done' }); + + // The hook should no longer exist because run completion auto-deletes hooks + // This is intentional behavior to allow token reuse across runs + await expect( + events.create(run.runId, { + eventType: 'hook_disposed', + correlationId: 'hook_auto_deleted', + }) + ).rejects.toThrow(/not found/i); + }); + }); + + describe('disallowed operations on terminal runs', () => { + it('should reject step_created on completed run', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await updateRun(events, run.runId, 'run_completed', { output: 'done' }); + + await expect( + createStep(events, run.runId, { + stepId: 'new_step', + stepName: 'test-step', + input: [], + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject step_started on completed run for pending step', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + + // Create a step but don't start it + await createStep(events, run.runId, { + stepId: 'pending_step', + stepName: 'test-step', + input: [], + }); + + // Complete the run + await updateRun(events, run.runId, 'run_completed', { output: 'done' }); + + // Should reject - cannot start a pending step on a terminal run + await expect( + updateStep(events, run.runId, 'pending_step', 'step_started') + ).rejects.toThrow(/terminal/i); + }); + + it('should reject hook_created on completed run', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await updateRun(events, run.runId, 'run_completed', { output: 'done' }); + + await expect( + createHook(events, run.runId, { + hookId: 'new_hook', + token: 'new-token', + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject step_created on failed run', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await updateRun(events, run.runId, 'run_failed', { error: 'Failed' }); + + await expect( + createStep(events, run.runId, { + stepId: 'new_step_failed', + stepName: 'test-step', + input: [], + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject step_created on cancelled run', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await events.create(run.runId, { eventType: 'run_cancelled' }); + + await expect( + createStep(events, run.runId, { + stepId: 'new_step_cancelled', + stepName: 'test-step', + input: [], + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject hook_created on failed run', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await updateRun(events, run.runId, 'run_failed', { error: 'Failed' }); + + await expect( + createHook(events, run.runId, { + hookId: 'new_hook_failed', + token: 'new-token-failed', + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject hook_created on cancelled run', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await events.create(run.runId, { eventType: 'run_cancelled' }); + + await expect( + createHook(events, run.runId, { + hookId: 'new_hook_cancelled', + token: 'new-token-cancelled', + }) + ).rejects.toThrow(/terminal/i); + }); + }); + + describe('idempotent operations', () => { + it('should allow run_cancelled on already cancelled run (idempotent)', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await events.create(run.runId, { eventType: 'run_cancelled' }); + + // Should succeed - idempotent operation + const result = await events.create(run.runId, { + eventType: 'run_cancelled', + }); + expect(result.run?.status).toBe('cancelled'); + }); + }); + + describe('step_retrying event handling', () => { + let testRunId: string; + + beforeEach(async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + testRunId = run.runId; + }); + + it('should set step status to pending and record error', async () => { + await createStep(events, testRunId, { + stepId: 'step_retry_1', + stepName: 'test-step', + input: [], + }); + await updateStep(events, testRunId, 'step_retry_1', 'step_started'); + + const result = await events.create(testRunId, { + eventType: 'step_retrying', + correlationId: 'step_retry_1', + eventData: { + error: 'Temporary failure', + retryAfter: new Date(Date.now() + 5000), + }, + }); + + expect(result.step?.status).toBe('pending'); + expect(result.step?.error?.message).toBe('Temporary failure'); + expect(result.step?.retryAfter).toBeInstanceOf(Date); + }); + + it('should increment attempt when step_started is called after step_retrying', async () => { + await createStep(events, testRunId, { + stepId: 'step_retry_2', + stepName: 'test-step', + input: [], + }); + + // First attempt + const started1 = await updateStep( + events, + testRunId, + 'step_retry_2', + 'step_started' + ); + expect(started1.attempt).toBe(1); + + // Retry + await events.create(testRunId, { + eventType: 'step_retrying', + correlationId: 'step_retry_2', + eventData: { error: 'Temporary failure' }, + }); + + // Second attempt + const started2 = await updateStep( + events, + testRunId, + 'step_retry_2', + 'step_started' + ); + expect(started2.attempt).toBe(2); + }); + + it('should reject step_retrying on completed step', async () => { + await createStep(events, testRunId, { + stepId: 'step_retry_completed', + stepName: 'test-step', + input: [], + }); + await updateStep( + events, + testRunId, + 'step_retry_completed', + 'step_completed', + { + result: 'done', + } + ); + + await expect( + events.create(testRunId, { + eventType: 'step_retrying', + correlationId: 'step_retry_completed', + eventData: { error: 'Should not work' }, + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject step_retrying on failed step', async () => { + await createStep(events, testRunId, { + stepId: 'step_retry_failed', + stepName: 'test-step', + input: [], + }); + await updateStep(events, testRunId, 'step_retry_failed', 'step_failed', { + error: 'Permanent failure', + }); + + await expect( + events.create(testRunId, { + eventType: 'step_retrying', + correlationId: 'step_retry_failed', + eventData: { error: 'Should not work' }, + }) + ).rejects.toThrow(/terminal/i); + }); + }); + + describe('run cancellation with in-flight entities', () => { + it('should allow in-progress step to complete after run cancelled', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + + // Create and start a step + await createStep(events, run.runId, { + stepId: 'step_in_flight', + stepName: 'test-step', + input: [], + }); + await updateStep(events, run.runId, 'step_in_flight', 'step_started'); + + // Cancel the run + await events.create(run.runId, { eventType: 'run_cancelled' }); + + // Should succeed - completing an in-progress step is allowed + const result = await updateStep( + events, + run.runId, + 'step_in_flight', + 'step_completed', + { result: 'done' } + ); + expect(result.status).toBe('completed'); + }); + + it('should reject step_created after run cancelled', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + await events.create(run.runId, { eventType: 'run_cancelled' }); + + await expect( + createStep(events, run.runId, { + stepId: 'new_step_after_cancel', + stepName: 'test-step', + input: [], + }) + ).rejects.toThrow(/terminal/i); + }); + + it('should reject step_started for pending step after run cancelled', async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + + // Create a step but don't start it + await createStep(events, run.runId, { + stepId: 'pending_after_cancel', + stepName: 'test-step', + input: [], + }); + + // Cancel the run + await events.create(run.runId, { eventType: 'run_cancelled' }); + + // Should reject - cannot start a pending step on a cancelled run + await expect( + updateStep(events, run.runId, 'pending_after_cancel', 'step_started') + ).rejects.toThrow(/terminal/i); + }); + }); + + describe('event ordering validation', () => { + let testRunId: string; + + beforeEach(async () => { + const run = await createRun(events, { + deploymentId: 'deployment-123', + workflowName: 'test-workflow', + input: [], + }); + testRunId = run.runId; + }); + + it('should reject step_completed before step_created', async () => { + await expect( + events.create(testRunId, { + eventType: 'step_completed', + correlationId: 'nonexistent_step', + eventData: { result: 'done' }, + }) + ).rejects.toThrow(/not found/i); + }); + + it('should reject step_started before step_created', async () => { + await expect( + events.create(testRunId, { + eventType: 'step_started', + correlationId: 'nonexistent_step_started', + }) + ).rejects.toThrow(/not found/i); + }); + + it('should reject step_failed before step_created', async () => { + await expect( + events.create(testRunId, { + eventType: 'step_failed', + correlationId: 'nonexistent_step_failed', + eventData: { error: 'Failed' }, + }) + ).rejects.toThrow(/not found/i); + }); + + it('should allow step_completed without step_started (instant completion)', async () => { + await createStep(events, testRunId, { + stepId: 'instant_complete', + stepName: 'test-step', + input: [], + }); + + // Should succeed - instant completion without starting + const result = await updateStep( + events, + testRunId, + 'instant_complete', + 'step_completed', + { result: 'instant' } + ); + expect(result.status).toBe('completed'); + }); + + it('should reject hook_disposed before hook_created', async () => { + await expect( + events.create(testRunId, { + eventType: 'hook_disposed', + correlationId: 'nonexistent_hook', + }) + ).rejects.toThrow(/not found/i); + }); + + it('should reject hook_received before hook_created', async () => { + await expect( + events.create(testRunId, { + eventType: 'hook_received', + correlationId: 'nonexistent_hook_received', + eventData: { payload: {} }, + }) + ).rejects.toThrow(/not found/i); }); }); }); diff --git a/packages/world-vercel/src/events.ts b/packages/world-vercel/src/events.ts index 8b94d03bf..55ee5ac31 100644 --- a/packages/world-vercel/src/events.ts +++ b/packages/world-vercel/src/events.ts @@ -1,13 +1,18 @@ import { + type AnyEventRequest, type CreateEventParams, - type CreateEventRequest, type Event, + type EventResult, EventSchema, EventTypeSchema, + HookSchema, type ListEventsByCorrelationIdParams, type ListEventsParams, type PaginatedResponse, PaginatedResponseSchema, + type Step, + StepSchema, + WorkflowRunSchema, } from '@workflow/world'; import z from 'zod'; import type { APIConfig } from './utils.js'; @@ -17,6 +22,69 @@ import { makeRequest, } from './utils.js'; +/** + * Wire format schema for step in event results. + * Handles error deserialization from wire format. + */ +const StepWireSchema = StepSchema.omit({ + error: true, +}).extend({ + // Backend returns error either as: + // - A JSON string (legacy/lazy mode) + // - An object {message, stack} (when errorRef is resolved) + error: z + .union([ + z.string(), + z.object({ + message: z.string(), + stack: z.string().optional(), + code: z.string().optional(), + }), + ]) + .optional(), + errorRef: z.any().optional(), +}); + +/** + * Deserialize step from wire format to Step interface format. + */ +function deserializeStep(wireStep: z.infer): Step { + const { error, errorRef, ...rest } = wireStep; + + const result: any = { + ...rest, + }; + + // Deserialize error to StructuredError + const errorSource = errorRef ?? error; + if (errorSource) { + if (typeof errorSource === 'string') { + try { + const parsed = JSON.parse(errorSource); + if (typeof parsed === 'object' && parsed.message !== undefined) { + result.error = { + message: parsed.message, + stack: parsed.stack, + code: parsed.code, + }; + } else { + result.error = { message: String(parsed) }; + } + } catch { + result.error = { message: errorSource }; + } + } else if (typeof errorSource === 'object' && errorSource !== null) { + result.error = { + message: errorSource.message ?? 'Unknown error', + stack: errorSource.stack, + code: errorSource.code, + }; + } + } + + return result as Step; +} + // Helper to filter event data based on resolveData setting function filterEventData(event: any, resolveData: 'none' | 'all'): Event { if (resolveData === 'none') { @@ -26,6 +94,15 @@ function filterEventData(event: any, resolveData: 'none' | 'all'): Event { return event; } +// Schema for EventResult wire format returned by events.create +// Uses wire format schemas for step to handle field name mapping +const EventResultWireSchema = z.object({ + event: EventSchema, + run: WorkflowRunSchema.optional(), + step: StepWireSchema.optional(), + hook: HookSchema.optional(), +}); + // Would usually "EventSchema.omit({ eventData: true })" but that doesn't work // on zod unions. Re-creating the schema manually. const EventWithRefsSchema = z.object({ @@ -68,8 +145,8 @@ export async function getWorkflowRunEvents( const queryString = searchParams.toString(); const query = queryString ? `?${queryString}` : ''; const endpoint = correlationId - ? `/v1/events${query}` - : `/v1/runs/${runId}/events${query}`; + ? `/v2/events${query}` + : `/v2/runs/${runId}/events${query}`; const response = (await makeRequest({ endpoint, @@ -89,22 +166,31 @@ export async function getWorkflowRunEvents( } export async function createWorkflowRunEvent( - id: string, - data: CreateEventRequest, + id: string | null, + data: AnyEventRequest, params?: CreateEventParams, config?: APIConfig -): Promise { +): Promise { const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION; - const event = await makeRequest({ - endpoint: `/v1/runs/${id}/events`, + // For run_created events, runId is null - use "null" string in the URL path + const runIdPath = id === null ? 'null' : id; + + const wireResult = await makeRequest({ + endpoint: `/v2/runs/${runIdPath}/events`, options: { method: 'POST', body: JSON.stringify(data, dateToStringReplacer), }, config, - schema: EventSchema, + schema: EventResultWireSchema, }); - return filterEventData(event, resolveData); + // Transform wire format to interface format + return { + event: filterEventData(wireResult.event, resolveData), + run: wireResult.run, + step: wireResult.step ? deserializeStep(wireResult.step) : undefined, + hook: wireResult.hook, + }; } diff --git a/packages/world-vercel/src/hooks.ts b/packages/world-vercel/src/hooks.ts index dd5b8190a..87aa7281b 100644 --- a/packages/world-vercel/src/hooks.ts +++ b/packages/world-vercel/src/hooks.ts @@ -52,7 +52,7 @@ export async function listHooks( if (runId) searchParams.set('runId', runId); const queryString = searchParams.toString(); - const endpoint = `/v1/hooks${queryString ? `?${queryString}` : ''}`; + const endpoint = `/v2/hooks${queryString ? `?${queryString}` : ''}`; const response = (await makeRequest({ endpoint, @@ -75,7 +75,7 @@ export async function getHook( config?: APIConfig ): Promise { const resolveData = params?.resolveData || 'all'; - const endpoint = `/v1/hooks/${hookId}`; + const endpoint = `/v2/hooks/${hookId}`; const hook = await makeRequest({ endpoint, @@ -93,7 +93,7 @@ export async function createHook( config?: APIConfig ): Promise { return makeRequest({ - endpoint: `/v1/hooks/create`, + endpoint: `/v2/hooks/create`, options: { method: 'POST', body: JSON.stringify( @@ -114,7 +114,7 @@ export async function getHookByToken( config?: APIConfig ): Promise { return makeRequest({ - endpoint: `/v1/hooks/by-token?token=${encodeURIComponent(token)}`, + endpoint: `/v2/hooks/by-token?token=${encodeURIComponent(token)}`, options: { method: 'GET', }, @@ -128,7 +128,7 @@ export async function disposeHook( config?: APIConfig ): Promise { return makeRequest({ - endpoint: `/v1/hooks/${hookId}`, + endpoint: `/v2/hooks/${hookId}`, options: { method: 'DELETE' }, config, schema: HookSchema, diff --git a/packages/world-vercel/src/runs.ts b/packages/world-vercel/src/runs.ts index 1a2647cac..93e13f945 100644 --- a/packages/world-vercel/src/runs.ts +++ b/packages/world-vercel/src/runs.ts @@ -6,8 +6,6 @@ import { type ListWorkflowRunsParams, type PaginatedResponse, PaginatedResponseSchema, - type PauseWorkflowRunParams, - type ResumeWorkflowRunParams, type UpdateWorkflowRunRequest, type WorkflowRun, WorkflowRunBaseSchema, @@ -99,7 +97,7 @@ export async function listWorkflowRuns( searchParams.set('remoteRefBehavior', remoteRefBehavior); const queryString = searchParams.toString(); - const endpoint = `/v1/runs${queryString ? `?${queryString}` : ''}`; + const endpoint = `/v2/runs${queryString ? `?${queryString}` : ''}`; const response = (await makeRequest({ endpoint, @@ -123,7 +121,7 @@ export async function createWorkflowRun( config?: APIConfig ): Promise { const run = await makeRequest({ - endpoint: '/v1/runs/create', + endpoint: '/v2/runs/create', options: { method: 'POST', body: JSON.stringify(data, dateToStringReplacer), @@ -146,7 +144,7 @@ export async function getWorkflowRun( searchParams.set('remoteRefBehavior', remoteRefBehavior); const queryString = searchParams.toString(); - const endpoint = `/v1/runs/${id}${queryString ? `?${queryString}` : ''}`; + const endpoint = `/v2/runs/${id}${queryString ? `?${queryString}` : ''}`; try { const run = await makeRequest({ @@ -175,7 +173,7 @@ export async function updateWorkflowRun( try { const serialized = serializeError(data); const run = await makeRequest({ - endpoint: `/v1/runs/${id}`, + endpoint: `/v2/runs/${id}`, options: { method: 'PUT', body: JSON.stringify(serialized, dateToStringReplacer), @@ -204,73 +202,7 @@ export async function cancelWorkflowRun( searchParams.set('remoteRefBehavior', remoteRefBehavior); const queryString = searchParams.toString(); - const endpoint = `/v1/runs/${id}/cancel${queryString ? `?${queryString}` : ''}`; - - try { - const run = await makeRequest({ - endpoint, - options: { method: 'PUT' }, - config, - schema: (remoteRefBehavior === 'lazy' - ? WorkflowRunWireWithRefsSchema - : WorkflowRunWireSchema) as any, - }); - - return filterRunData(run, resolveData); - } catch (error) { - if (error instanceof WorkflowAPIError && error.status === 404) { - throw new WorkflowRunNotFoundError(id); - } - throw error; - } -} - -export async function pauseWorkflowRun( - id: string, - params?: PauseWorkflowRunParams, - config?: APIConfig -): Promise { - const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION; - const remoteRefBehavior = resolveData === 'none' ? 'lazy' : 'resolve'; - - const searchParams = new URLSearchParams(); - searchParams.set('remoteRefBehavior', remoteRefBehavior); - - const queryString = searchParams.toString(); - const endpoint = `/v1/runs/${id}/pause${queryString ? `?${queryString}` : ''}`; - - try { - const run = await makeRequest({ - endpoint, - options: { method: 'PUT' }, - config, - schema: (remoteRefBehavior === 'lazy' - ? WorkflowRunWireWithRefsSchema - : WorkflowRunWireSchema) as any, - }); - - return filterRunData(run, resolveData); - } catch (error) { - if (error instanceof WorkflowAPIError && error.status === 404) { - throw new WorkflowRunNotFoundError(id); - } - throw error; - } -} - -export async function resumeWorkflowRun( - id: string, - params?: ResumeWorkflowRunParams, - config?: APIConfig -): Promise { - const resolveData = params?.resolveData ?? DEFAULT_RESOLVE_DATA_OPTION; - const remoteRefBehavior = resolveData === 'none' ? 'lazy' : 'resolve'; - - const searchParams = new URLSearchParams(); - searchParams.set('remoteRefBehavior', remoteRefBehavior); - - const queryString = searchParams.toString(); - const endpoint = `/v1/runs/${id}/resume${queryString ? `?${queryString}` : ''}`; + const endpoint = `/v2/runs/${id}/cancel${queryString ? `?${queryString}` : ''}`; try { const run = await makeRequest({ diff --git a/packages/world-vercel/src/steps.ts b/packages/world-vercel/src/steps.ts index 83233446d..f97014567 100644 --- a/packages/world-vercel/src/steps.ts +++ b/packages/world-vercel/src/steps.ts @@ -13,24 +13,31 @@ import type { APIConfig } from './utils.js'; import { DEFAULT_RESOLVE_DATA_OPTION, dateToStringReplacer, - deserializeError, makeRequest, - serializeError, } from './utils.js'; /** * Wire format schema for steps coming from the backend. - * The backend returns error as a JSON string, not an object, so we need - * a schema that accepts the wire format before deserialization. - * - * This is used for validation in makeRequest(), then deserializeStepError() - * transforms the string into the expected StructuredError object. + * Handles error deserialization from wire format. */ const StepWireSchema = StepSchema.omit({ error: true, }).extend({ - // Backend returns error as a JSON string, not an object - error: z.string().optional(), + // Backend returns error either as: + // - A JSON string (legacy/lazy mode) + // - An object {message, stack} (when errorRef is resolved) + // This will be deserialized and mapped to error + error: z + .union([ + z.string(), + z.object({ + message: z.string(), + stack: z.string().optional(), + code: z.string().optional(), + }), + ]) + .optional(), + errorRef: z.any().optional(), }); // Wire schema for lazy mode with refs instead of data @@ -45,18 +52,66 @@ const StepWireWithRefsSchema = StepWireSchema.omit({ output: z.any().optional(), }); +/** + * Transform step from wire format to Step interface format. + * Maps: + * - error/errorRef → error (deserializing JSON string to StructuredError) + */ +function deserializeStep(wireStep: any): Step { + const { error, errorRef, ...rest } = wireStep; + + const result: any = { + ...rest, + }; + + // Deserialize error to StructuredError + // The backend returns error as: + // - errorRef: resolved object {message, stack} when remoteRefBehavior=resolve + // - error: JSON string (legacy) or object (when resolved) + const errorSource = errorRef ?? error; + if (errorSource) { + if (typeof errorSource === 'string') { + try { + const parsed = JSON.parse(errorSource); + if (typeof parsed === 'object' && parsed.message !== undefined) { + result.error = { + message: parsed.message, + stack: parsed.stack, + code: parsed.code, + }; + } else { + // Parsed but not an object with message + result.error = { message: String(parsed) }; + } + } catch { + // Not JSON, treat as plain string + result.error = { message: errorSource }; + } + } else if (typeof errorSource === 'object' && errorSource !== null) { + // Already an object (from resolved ref) + result.error = { + message: errorSource.message ?? 'Unknown error', + stack: errorSource.stack, + code: errorSource.code, + }; + } + } + + return result as Step; +} + // Helper to filter step data based on resolveData setting function filterStepData(step: any, resolveData: 'none' | 'all'): Step { if (resolveData === 'none') { const { inputRef: _inputRef, outputRef: _outputRef, ...rest } = step; - const deserialized = deserializeError(rest); + const deserialized = deserializeStep(rest); return { ...deserialized, input: [], output: undefined, }; } - return deserializeError(step); + return deserializeStep(step); } // Functions @@ -82,7 +137,7 @@ export async function listWorkflowRunSteps( searchParams.set('remoteRefBehavior', remoteRefBehavior); const queryString = searchParams.toString(); - const endpoint = `/v1/runs/${runId}/steps${queryString ? `?${queryString}` : ''}`; + const endpoint = `/v2/runs/${runId}/steps${queryString ? `?${queryString}` : ''}`; const response = (await makeRequest({ endpoint, @@ -105,7 +160,7 @@ export async function createStep( config?: APIConfig ): Promise { const step = await makeRequest({ - endpoint: `/v1/runs/${runId}/steps`, + endpoint: `/v2/runs/${runId}/steps`, options: { method: 'POST', body: JSON.stringify(data, dateToStringReplacer), @@ -113,7 +168,7 @@ export async function createStep( config, schema: StepWireSchema, }); - return deserializeError(step); + return deserializeStep(step); } export async function updateStep( @@ -122,17 +177,22 @@ export async function updateStep( data: UpdateStepRequest, config?: APIConfig ): Promise { - const serialized = serializeError(data); + // Map interface field names to wire format field names + const { error: stepError, ...rest } = data; + const wireData: any = { ...rest }; + if (stepError) { + wireData.error = JSON.stringify(stepError); + } const step = await makeRequest({ - endpoint: `/v1/runs/${runId}/steps/${stepId}`, + endpoint: `/v2/runs/${runId}/steps/${stepId}`, options: { method: 'PUT', - body: JSON.stringify(serialized, dateToStringReplacer), + body: JSON.stringify(wireData, dateToStringReplacer), }, config, schema: StepWireSchema, }); - return deserializeError(step); + return deserializeStep(step); } export async function getStep( @@ -149,8 +209,8 @@ export async function getStep( const queryString = searchParams.toString(); const endpoint = runId - ? `/v1/runs/${runId}/steps/${stepId}${queryString ? `?${queryString}` : ''}` - : `/v1/steps/${stepId}${queryString ? `?${queryString}` : ''}`; + ? `/v2/runs/${runId}/steps/${stepId}${queryString ? `?${queryString}` : ''}` + : `/v2/steps/${stepId}${queryString ? `?${queryString}` : ''}`; const step = await makeRequest({ endpoint, diff --git a/packages/world-vercel/src/storage.ts b/packages/world-vercel/src/storage.ts index 51c5a8eaf..da874bbf7 100644 --- a/packages/world-vercel/src/storage.ts +++ b/packages/world-vercel/src/storage.ts @@ -1,45 +1,19 @@ import type { Storage } from '@workflow/world'; import { createWorkflowRunEvent, getWorkflowRunEvents } from './events.js'; -import { - createHook, - disposeHook, - getHook, - getHookByToken, - listHooks, -} from './hooks.js'; -import { - cancelWorkflowRun, - createWorkflowRun, - getWorkflowRun, - listWorkflowRuns, - pauseWorkflowRun, - resumeWorkflowRun, - updateWorkflowRun, -} from './runs.js'; -import { - createStep, - getStep, - listWorkflowRunSteps, - updateStep, -} from './steps.js'; +import { getHook, getHookByToken, listHooks } from './hooks.js'; +import { getWorkflowRun, listWorkflowRuns } from './runs.js'; +import { getStep, listWorkflowRunSteps } from './steps.js'; import type { APIConfig } from './utils.js'; export function createStorage(config?: APIConfig): Storage { return { // Storage interface with namespaced methods runs: { - create: (data) => createWorkflowRun(data, config), get: (id, params) => getWorkflowRun(id, params, config), - update: (id, data) => updateWorkflowRun(id, data, config), list: (params) => listWorkflowRuns(params, config), - cancel: (id, params) => cancelWorkflowRun(id, params, config), - pause: (id, params) => pauseWorkflowRun(id, params, config), - resume: (id, params) => resumeWorkflowRun(id, params, config), }, steps: { - create: (runId, data) => createStep(runId, data, config), get: (runId, stepId, params) => getStep(runId, stepId, params, config), - update: (runId, stepId, data) => updateStep(runId, stepId, data, config), list: (params) => listWorkflowRunSteps(params, config), }, events: { @@ -49,11 +23,9 @@ export function createStorage(config?: APIConfig): Storage { listByCorrelationId: (params) => getWorkflowRunEvents(params, config), }, hooks: { - create: (runId, data) => createHook(runId, data, config), get: (hookId, params) => getHook(hookId, params, config), getByToken: (token) => getHookByToken(token, config), list: (params) => listHooks(params, config), - dispose: (hookId) => disposeHook(hookId, config), }, }; } diff --git a/packages/world-vercel/src/streamer.ts b/packages/world-vercel/src/streamer.ts index 24e1b00ac..017b3eaed 100644 --- a/packages/world-vercel/src/streamer.ts +++ b/packages/world-vercel/src/streamer.ts @@ -8,10 +8,10 @@ function getStreamUrl( ) { if (runId) { return new URL( - `${httpConfig.baseUrl}/v1/runs/${runId}/stream/${encodeURIComponent(name)}` + `${httpConfig.baseUrl}/v2/runs/${runId}/stream/${encodeURIComponent(name)}` ); } - return new URL(`${httpConfig.baseUrl}/v1/stream/${encodeURIComponent(name)}`); + return new URL(`${httpConfig.baseUrl}/v2/stream/${encodeURIComponent(name)}`); } export function createStreamer(config?: APIConfig): Streamer { @@ -58,7 +58,7 @@ export function createStreamer(config?: APIConfig): Streamer { async listStreamsByRunId(runId: string) { const httpConfig = await getHttpConfig(config); - const url = new URL(`${httpConfig.baseUrl}/v1/runs/${runId}/streams`); + const url = new URL(`${httpConfig.baseUrl}/v2/runs/${runId}/streams`); const res = await fetch(url, { headers: httpConfig.headers }); if (!res.ok) throw new Error(`Failed to list streams: ${res.status}`); return (await res.json()) as string[]; diff --git a/packages/world/src/events.ts b/packages/world/src/events.ts index 56a9082e3..a923cb3e7 100644 --- a/packages/world/src/events.ts +++ b/packages/world/src/events.ts @@ -3,15 +3,26 @@ import type { PaginationOptions, ResolveData } from './shared.js'; // Event type enum export const EventTypeSchema = z.enum([ + // Run lifecycle events + 'run_created', + 'run_started', + 'run_completed', + 'run_failed', + 'run_cancelled', + // Step lifecycle events + 'step_created', 'step_completed', 'step_failed', 'step_retrying', 'step_started', + // Hook lifecycle events 'hook_created', 'hook_received', 'hook_disposed', + // Wait lifecycle events 'wait_created', 'wait_completed', + // Legacy workflow events (deprecated, use run_* instead) 'workflow_completed', 'workflow_failed', 'workflow_started', @@ -41,28 +52,58 @@ const StepFailedEventSchema = BaseEventSchema.extend({ eventData: z.object({ error: z.any(), stack: z.string().optional(), - fatal: z.boolean().optional(), }), }); -// TODO: this is not actually used anywhere yet, we could remove it -// on client and server if needed +/** + * Event created when a step fails and will be retried. + * Sets the step status back to 'pending' and records the error. + * The error is stored in step.error for debugging. + */ const StepRetryingEventSchema = BaseEventSchema.extend({ eventType: z.literal('step_retrying'), correlationId: z.string(), eventData: z.object({ - attempt: z.number().min(1), + error: z.any(), + stack: z.string().optional(), + retryAfter: z.coerce.date().optional(), }), }); const StepStartedEventSchema = BaseEventSchema.extend({ eventType: z.literal('step_started'), correlationId: z.string(), + eventData: z + .object({ + attempt: z.number().optional(), + }) + .optional(), }); +/** + * Event created when a step is first invoked. The World implementation + * atomically creates both the event and the step entity. + */ +const StepCreatedEventSchema = BaseEventSchema.extend({ + eventType: z.literal('step_created'), + correlationId: z.string(), + eventData: z.object({ + stepName: z.string(), + input: z.any(), // SerializedData + }), +}); + +/** + * Event created when a hook is first invoked. The World implementation + * atomically creates both the event and the hook entity. + */ const HookCreatedEventSchema = BaseEventSchema.extend({ eventType: z.literal('hook_created'), correlationId: z.string(), + eventData: z.object({ + token: z.string(), + metadata: z.any().optional(), // SerializedData + }), }); const HookReceivedEventSchema = BaseEventSchema.extend({ @@ -91,12 +132,73 @@ const WaitCompletedEventSchema = BaseEventSchema.extend({ correlationId: z.string(), }); -// TODO: not used yet +// ============================================================================= +// Run lifecycle events +// ============================================================================= + +/** + * Event created when a workflow run is first created. The World implementation + * atomically creates both the event and the run entity with status 'pending'. + */ +const RunCreatedEventSchema = BaseEventSchema.extend({ + eventType: z.literal('run_created'), + eventData: z.object({ + deploymentId: z.string(), + workflowName: z.string(), + input: z.array(z.any()), // SerializedData[] + executionContext: z.record(z.string(), z.any()).optional(), + }), +}); + +/** + * Event created when a workflow run starts executing. + * Updates the run entity to status 'running'. + */ +const RunStartedEventSchema = BaseEventSchema.extend({ + eventType: z.literal('run_started'), +}); + +/** + * Event created when a workflow run completes successfully. + * Updates the run entity to status 'completed' with output. + */ +const RunCompletedEventSchema = BaseEventSchema.extend({ + eventType: z.literal('run_completed'), + eventData: z.object({ + output: z.any().optional(), // SerializedData + }), +}); + +/** + * Event created when a workflow run fails. + * Updates the run entity to status 'failed' with error. + */ +const RunFailedEventSchema = BaseEventSchema.extend({ + eventType: z.literal('run_failed'), + eventData: z.object({ + error: z.any(), + errorCode: z.string().optional(), + }), +}); + +/** + * Event created when a workflow run is cancelled. + * Updates the run entity to status 'cancelled'. + */ +const RunCancelledEventSchema = BaseEventSchema.extend({ + eventType: z.literal('run_cancelled'), +}); + +// ============================================================================= +// Legacy workflow events (deprecated, use run_* events instead) +// ============================================================================= + +/** @deprecated Use run_completed instead */ const WorkflowCompletedEventSchema = BaseEventSchema.extend({ eventType: z.literal('workflow_completed'), }); -// TODO: not used yet +/** @deprecated Use run_failed instead */ const WorkflowFailedEventSchema = BaseEventSchema.extend({ eventType: z.literal('workflow_failed'), eventData: z.object({ @@ -104,22 +206,33 @@ const WorkflowFailedEventSchema = BaseEventSchema.extend({ }), }); -// TODO: not used yet +/** @deprecated Use run_started instead */ const WorkflowStartedEventSchema = BaseEventSchema.extend({ eventType: z.literal('workflow_started'), }); // Discriminated union (used for both creation requests and server responses) export const CreateEventSchema = z.discriminatedUnion('eventType', [ + // Run lifecycle events + RunCreatedEventSchema, + RunStartedEventSchema, + RunCompletedEventSchema, + RunFailedEventSchema, + RunCancelledEventSchema, + // Step lifecycle events + StepCreatedEventSchema, StepCompletedEventSchema, StepFailedEventSchema, StepRetryingEventSchema, StepStartedEventSchema, + // Hook lifecycle events HookCreatedEventSchema, HookReceivedEventSchema, HookDisposedEventSchema, + // Wait lifecycle events WaitCreatedEventSchema, WaitCompletedEventSchema, + // Legacy workflow events (deprecated) WorkflowCompletedEventSchema, WorkflowFailedEventSchema, WorkflowStartedEventSchema, @@ -136,13 +249,49 @@ export const EventSchema = CreateEventSchema.and( // Inferred types export type Event = z.infer; -export type CreateEventRequest = z.infer; export type HookReceivedEvent = z.infer; +/** + * Union of all possible event request types. + * @internal Use CreateEventRequest or RunCreatedEventRequest instead. + */ +export type AnyEventRequest = z.infer; + +/** + * Event request for creating a new workflow run. + * Must be used with runId: null since the server generates the runId. + */ +export type RunCreatedEventRequest = z.infer; + +/** + * Event request types that require an existing runId. + * This is the common case for all events except run_created. + */ +export type CreateEventRequest = Exclude< + AnyEventRequest, + RunCreatedEventRequest +>; + export interface CreateEventParams { resolveData?: ResolveData; } +/** + * Result of creating an event. Includes the created event and optionally + * the entity that was created or updated as a result of the event. + * This reduces round-trips by returning entity data along with the event. + */ +export interface EventResult { + /** The created event */ + event: Event; + /** The workflow run entity (for run_* events) */ + run?: import('./runs.js').WorkflowRun; + /** The step entity (for step_* events) */ + step?: import('./steps.js').Step; + /** The hook entity (for hook_created events) */ + hook?: import('./hooks.js').Hook; +} + export interface ListEventsParams { runId: string; pagination?: PaginationOptions; diff --git a/packages/world/src/interfaces.ts b/packages/world/src/interfaces.ts index 0c22703f9..b4ee231a3 100644 --- a/packages/world/src/interfaces.ts +++ b/packages/world/src/interfaces.ts @@ -2,33 +2,23 @@ import type { CreateEventParams, CreateEventRequest, Event, + EventResult, ListEventsByCorrelationIdParams, ListEventsParams, + RunCreatedEventRequest, } from './events.js'; -import type { - CreateHookRequest, - GetHookParams, - Hook, - ListHooksParams, -} from './hooks.js'; +import type { GetHookParams, Hook, ListHooksParams } from './hooks.js'; import type { Queue } from './queue.js'; import type { - CancelWorkflowRunParams, - CreateWorkflowRunRequest, GetWorkflowRunParams, ListWorkflowRunsParams, - PauseWorkflowRunParams, - ResumeWorkflowRunParams, - UpdateWorkflowRunRequest, WorkflowRun, } from './runs.js'; import type { PaginatedResponse } from './shared.js'; import type { - CreateStepRequest, GetStepParams, ListWorkflowRunStepsParams, Step, - UpdateStepRequest, } from './steps.js'; export interface Streamer { @@ -45,40 +35,74 @@ export interface Streamer { listStreamsByRunId(runId: string): Promise; } +/** + * Storage interface for workflow data. + * + * All entity mutations (runs, steps, hooks) MUST go through events.create(). + * The World implementation atomically creates the entity when processing the corresponding event. + * + * Entity methods are read-only: + * - runs: get, list + * - steps: get, list + * - hooks: get, getByToken, list + * + * State changes are done via events: + * - run_cancelled event for run cancellation + * - hook_disposed event for explicit hook disposal (optional) + * + * Note: Hooks are automatically disposed by the World implementation when a workflow + * reaches a terminal state (run_completed, run_failed, run_cancelled). This releases + * hook tokens for reuse by future workflows. The hook_disposed event is only needed + * for explicit disposal before workflow completion. + */ export interface Storage { runs: { - create(data: CreateWorkflowRunRequest): Promise; get(id: string, params?: GetWorkflowRunParams): Promise; - update(id: string, data: UpdateWorkflowRunRequest): Promise; list( params?: ListWorkflowRunsParams ): Promise>; - cancel(id: string, params?: CancelWorkflowRunParams): Promise; - pause(id: string, params?: PauseWorkflowRunParams): Promise; - resume(id: string, params?: ResumeWorkflowRunParams): Promise; }; steps: { - create(runId: string, data: CreateStepRequest): Promise; get( runId: string | undefined, stepId: string, params?: GetStepParams ): Promise; - update( - runId: string, - stepId: string, - data: UpdateStepRequest - ): Promise; list(params: ListWorkflowRunStepsParams): Promise>; }; events: { + /** + * Create a run_created event to start a new workflow run. + * The runId parameter must be null - the server generates and returns the runId. + * + * @param runId - Must be null for run_created events + * @param data - The run_created event data + * @param params - Optional parameters for event creation + * @returns Promise resolving to the created event and run entity + */ + create( + runId: null, + data: RunCreatedEventRequest, + params?: CreateEventParams + ): Promise; + + /** + * Create an event for an existing workflow run and atomically update the entity. + * Returns both the event and the affected entity (run/step/hook). + * + * @param runId - The workflow run ID (required for all events except run_created) + * @param data - The event to create + * @param params - Optional parameters for event creation + * @returns Promise resolving to the created event and affected entity + */ create( runId: string, data: CreateEventRequest, params?: CreateEventParams - ): Promise; + ): Promise; + list(params: ListEventsParams): Promise>; listByCorrelationId( params: ListEventsByCorrelationIdParams @@ -86,15 +110,9 @@ export interface Storage { }; hooks: { - create( - runId: string, - data: CreateHookRequest, - params?: GetHookParams - ): Promise; get(hookId: string, params?: GetHookParams): Promise; getByToken(token: string, params?: GetHookParams): Promise; list(params: ListHooksParams): Promise>; - dispose(hookId: string, params?: GetHookParams): Promise; }; } diff --git a/packages/world/src/runs.ts b/packages/world/src/runs.ts index d6aa4cd69..64451c6f1 100644 --- a/packages/world/src/runs.ts +++ b/packages/world/src/runs.ts @@ -13,7 +13,6 @@ export const WorkflowRunStatusSchema = z.enum([ 'running', 'completed', 'failed', - 'paused', 'cancelled', ]); @@ -41,7 +40,7 @@ export const WorkflowRunBaseSchema = z.object({ export const WorkflowRunSchema = z.discriminatedUnion('status', [ // Non-final states WorkflowRunBaseSchema.extend({ - status: z.enum(['pending', 'running', 'paused']), + status: z.enum(['pending', 'running']), output: z.undefined(), error: z.undefined(), completedAt: z.undefined(), @@ -102,11 +101,3 @@ export interface ListWorkflowRunsParams { export interface CancelWorkflowRunParams { resolveData?: ResolveData; } - -export interface PauseWorkflowRunParams { - resolveData?: ResolveData; -} - -export interface ResumeWorkflowRunParams { - resolveData?: ResolveData; -} diff --git a/packages/world/src/steps.ts b/packages/world/src/steps.ts index 8c973f6b9..db1518c02 100644 --- a/packages/world/src/steps.ts +++ b/packages/world/src/steps.ts @@ -24,8 +24,17 @@ export const StepSchema = z.object({ status: StepStatusSchema, input: z.array(z.any()), output: z.any().optional(), + /** + * The error from a step_retrying or step_failed event. + * This tracks the most recent error the step encountered, which may + * be from a retry attempt (step_retrying) or the final failure (step_failed). + */ error: StructuredErrorSchema.optional(), attempt: z.number(), + /** + * When the step first started executing. Set by the first step_started event + * and not updated on subsequent retries. + */ startedAt: z.coerce.date().optional(), completedAt: z.coerce.date().optional(), createdAt: z.coerce.date(),