From 92b09932207c496a380aa69efa5f85a8a101531e Mon Sep 17 00:00:00 2001 From: Frank Midigo Date: Wed, 22 Apr 2026 08:25:36 +0300 Subject: [PATCH 01/21] PoC for webhook responses --- lib/lightning/workflows/trigger.ex | 2 + lib/lightning_web/channels/run_channel.ex | 49 +++++++++++++++-------- 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/lib/lightning/workflows/trigger.ex b/lib/lightning/workflows/trigger.ex index c6aa449e350..60b7b79362e 100644 --- a/lib/lightning/workflows/trigger.ex +++ b/lib/lightning/workflows/trigger.ex @@ -55,6 +55,8 @@ defmodule Lightning.Workflows.Trigger do field :delete, :boolean, virtual: true field :has_auth_method, :boolean, virtual: true + field :webhook_response_success_code, :integer, virtual: true, default: 200 + field :webhook_response_error_code, :integer, virtual: true, default: 400 many_to_many :webhook_auth_methods, Lightning.Workflows.WebhookAuthMethod, join_through: "trigger_webhook_auth_methods", diff --git a/lib/lightning_web/channels/run_channel.ex b/lib/lightning_web/channels/run_channel.ex index 4ff9d65ceef..c7df66b0d04 100644 --- a/lib/lightning_web/channels/run_channel.ex +++ b/lib/lightning_web/channels/run_channel.ex @@ -210,6 +210,7 @@ defmodule LightningWeb.RunChannel do reply_with(socket, {:error, changeset}) {:ok, step} -> + maybe_broadcast_custom_webhook_response(socket.assigns.run, payload) reply_with(socket, {:ok, %{step_id: step.id}}) end end @@ -306,6 +307,32 @@ defmodule LightningWeb.RunChannel do # Ignore other messages def handle_info(_msg, socket), do: {:noreply, socket} + defp maybe_broadcast_custom_webhook_response(run, payload) do + case payload["webhook_response"] do + nil -> + :ok + + webhook_response -> + run_with_preloads = Repo.preload(run, work_order: [:trigger]) + trigger = run_with_preloads.work_order.trigger + + if trigger && trigger.type == :webhook && + trigger.webhook_reply == :custom do + topic = + "work_order:#{run_with_preloads.work_order.id}:webhook_response" + + status_code = webhook_response["code"] || 200 + body = webhook_response["body"] || %{} + + Phoenix.PubSub.broadcast( + Lightning.PubSub, + topic, + {:webhook_response, status_code, body} + ) + end + end + end + defp maybe_broadcast_webhook_response(run, payload) do work_order = run.work_order trigger = work_order.trigger @@ -313,10 +340,7 @@ defmodule LightningWeb.RunChannel do if trigger && trigger.type == :webhook && trigger.webhook_reply == :after_completion do topic = "work_order:#{work_order.id}:webhook_response" - - # TODO - Later allow workflow authors to customize the status code - # and body of the reply. - status_code = determine_status_code(run.state) + status_code = determine_status_code(run.state, trigger) body = %{ data: payload["final_state"], @@ -340,18 +364,11 @@ defmodule LightningWeb.RunChannel do end end - # TODO - decide how we should respond... do we use HTTP codes for run states? - defp determine_status_code(state) do - case state do - :success -> 201 - :failed -> 201 - :crashed -> 201 - :exception -> 201 - :killed -> 201 - :cancelled -> 201 - _other -> 201 - end - end + defp determine_status_code(:success, trigger), + do: trigger.webhook_response_success_code + + defp determine_status_code(_state, trigger), + do: trigger.webhook_response_error_code defp update_scrubber(nil, samples, basic_auth) do Scrubber.start_link(samples: samples, basic_auth: basic_auth) From 53b09e6755e98c50523afd5882e0f903aeae0917 Mon Sep 17 00:00:00 2001 From: Frank Midigo Date: Wed, 22 Apr 2026 08:59:59 +0300 Subject: [PATCH 02/21] Allow users to change status codes from UI --- .../adapters/YAMLStateToYDoc.ts | 8 + .../components/inspector/TriggerForm.tsx | 139 +++++++++++++++++- .../js/collaborative-editor/types/session.ts | 4 +- .../js/collaborative-editor/types/trigger.ts | 18 ++- assets/js/yaml/types.ts | 2 + .../collaboration/workflow_serializer.ex | 22 ++- lib/lightning/workflows/trigger.ex | 27 +++- lib/lightning_web/channels/run_channel.ex | 4 +- ...add_webhook_response_codes_to_triggers.exs | 10 ++ 9 files changed, 223 insertions(+), 11 deletions(-) create mode 100644 priv/repo/migrations/20260422052858_add_webhook_response_codes_to_triggers.exs diff --git a/assets/js/collaborative-editor/adapters/YAMLStateToYDoc.ts b/assets/js/collaborative-editor/adapters/YAMLStateToYDoc.ts index 1ea725db71d..f3f946f62f4 100644 --- a/assets/js/collaborative-editor/adapters/YAMLStateToYDoc.ts +++ b/assets/js/collaborative-editor/adapters/YAMLStateToYDoc.ts @@ -77,6 +77,14 @@ export class YAMLStateToYDoc { if (trigger.type === 'webhook') { triggerMap.set('webhook_reply', trigger.webhook_reply ?? null); + triggerMap.set( + 'webhook_response_success_code', + trigger.webhook_response_success_code ?? null + ); + triggerMap.set( + 'webhook_response_error_code', + trigger.webhook_response_error_code ?? null + ); } return triggerMap; diff --git a/assets/js/collaborative-editor/components/inspector/TriggerForm.tsx b/assets/js/collaborative-editor/components/inspector/TriggerForm.tsx index a1be264f046..01e336f8527 100644 --- a/assets/js/collaborative-editor/components/inspector/TriggerForm.tsx +++ b/assets/js/collaborative-editor/components/inspector/TriggerForm.tsx @@ -398,6 +398,7 @@ export function TriggerForm({ trigger }: TriggerFormProps) { e.target.value as | 'before_start' | 'after_completion' + | 'custom' ) } onBlur={field.handleBlur} @@ -418,14 +419,17 @@ export function TriggerForm({ trigger }: TriggerFormProps) { Async (default) +

{field.state.value === 'after_completion' - ? 'Responds with the final output state after the run completes. (Note that depending on your queue size and the duration of the workflow itself, this could take a long time.)' - : 'Responds immediately with the enqueued work order ID.'} + ? 'Responds with the final output state after the run completes. You can customise the HTTP status codes returned below.' + : field.state.value === 'custom' + ? 'The job code itself sends the HTTP response using the send() function.' + : 'Responds immediately with the enqueued work order ID.'}

{field.state.meta.errors.map(error => (

)} + + {/* Conditional response code fields for after_completion */} + + {replyField => { + if (replyField.state.value !== 'after_completion') { + return null; + } + + return ( +

+ {/* Success Status Code */} + + {field => ( +
+ + + field.handleChange( + e.target.value === '' + ? null + : parseInt(e.target.value, 10) + ) + } + onBlur={field.handleBlur} + disabled={isReadOnly} + className={cn( + 'block w-full px-3 py-2', + 'border rounded-md text-sm', + field.state.meta.errors.length > 0 + ? 'border-red-300 text-red-900 focus:border-red-500 focus:ring-red-500' + : 'border-slate-300 focus:border-indigo-500 focus:ring-indigo-500', + 'focus:outline-none focus:ring-1', + 'disabled:opacity-50 disabled:cursor-not-allowed' + )} + /> +

+ HTTP status code returned when the run + succeeds (default: 200) +

+ {field.state.meta.errors.map(error => ( +

+ {error} +

+ ))} +
+ )} +
+ + {/* Error Status Code */} + + {field => ( +
+ + + field.handleChange( + e.target.value === '' + ? null + : parseInt(e.target.value, 10) + ) + } + onBlur={field.handleBlur} + disabled={isReadOnly} + className={cn( + 'block w-full px-3 py-2', + 'border rounded-md text-sm', + field.state.meta.errors.length > 0 + ? 'border-red-300 text-red-900 focus:border-red-500 focus:ring-red-500' + : 'border-slate-300 focus:border-indigo-500 focus:ring-indigo-500', + 'focus:outline-none focus:ring-1', + 'disabled:opacity-50 disabled:cursor-not-allowed' + )} + /> +

+ HTTP status code returned when the run + fails (default: 400) +

+ {field.state.meta.errors.map(error => ( +

+ {error} +

+ ))} +
+ )} +
+
+ ); + }} + ); diff --git a/assets/js/collaborative-editor/types/session.ts b/assets/js/collaborative-editor/types/session.ts index 3c418b7bcfb..e56a339a11e 100644 --- a/assets/js/collaborative-editor/types/session.ts +++ b/assets/js/collaborative-editor/types/session.ts @@ -70,7 +70,9 @@ export namespace Session { enabled: boolean; cron_expression: string | null; has_auth_method: boolean; - webhook_reply: 'before_start' | 'after_completion' | null; + webhook_reply: 'before_start' | 'after_completion' | 'custom' | null; + webhook_response_success_code: number | null; + webhook_response_error_code: number | null; webhook_auth_methods: Array<{ id: string; name: string; diff --git a/assets/js/collaborative-editor/types/trigger.ts b/assets/js/collaborative-editor/types/trigger.ts index 6605f9f212f..1f6dfd209eb 100644 --- a/assets/js/collaborative-editor/types/trigger.ts +++ b/assets/js/collaborative-editor/types/trigger.ts @@ -21,9 +21,23 @@ const webhookTriggerSchema = baseTriggerSchema.extend({ cron_cursor_job_id: z.null().default(null), kafka_configuration: z.null().default(null), webhook_reply: z - .enum(['before_start', 'after_completion']) + .enum(['before_start', 'after_completion', 'custom']) .nullable() .default('before_start'), + webhook_response_success_code: z + .number() + .int() + .min(100) + .max(599) + .nullable() + .default(null), + webhook_response_error_code: z + .number() + .int() + .min(100) + .max(599) + .nullable() + .default(null), }); // Cron trigger schema with professional validation using cron-validator @@ -137,6 +151,8 @@ export const createDefaultTrigger = ( cron_cursor_job_id: null, kafka_configuration: null, webhook_reply: 'before_start' as const, + webhook_response_success_code: null, + webhook_response_error_code: null, }; case 'cron': diff --git a/assets/js/yaml/types.ts b/assets/js/yaml/types.ts index 321d29ffffa..7a5f42b307c 100644 --- a/assets/js/yaml/types.ts +++ b/assets/js/yaml/types.ts @@ -29,6 +29,8 @@ export type StateWebhookTrigger = { enabled: boolean; type: 'webhook'; webhook_reply: 'before_start' | 'after_completion' | 'custom' | null; + webhook_response_success_code?: number | null; + webhook_response_error_code?: number | null; }; export type StateKafkaTrigger = { diff --git a/lib/lightning/collaboration/workflow_serializer.ex b/lib/lightning/collaboration/workflow_serializer.ex index c65a2215e62..08e7a594a38 100644 --- a/lib/lightning/collaboration/workflow_serializer.ex +++ b/lib/lightning/collaboration/workflow_serializer.ex @@ -228,7 +228,10 @@ defmodule Lightning.Collaboration.WorkflowSerializer do "id" => trigger.id, "type" => trigger.type |> to_string(), "webhook_reply" => - trigger.webhook_reply && to_string(trigger.webhook_reply) + trigger.webhook_reply && to_string(trigger.webhook_reply), + "webhook_response_success_code" => + trigger.webhook_response_success_code, + "webhook_response_error_code" => trigger.webhook_response_error_code }) Yex.Array.push(triggers_array, trigger_map) @@ -274,9 +277,11 @@ defmodule Lightning.Collaboration.WorkflowSerializer do |> Enum.map(fn trigger -> trigger |> Map.take( - ~w(id type enabled cron_expression cron_cursor_job_id webhook_reply kafka_configuration) + ~w(id type enabled cron_expression cron_cursor_job_id webhook_reply + kafka_configuration webhook_response_success_code webhook_response_error_code) ) |> normalize_kafka_configuration() + |> normalize_response_codes() end) end @@ -299,6 +304,19 @@ defmodule Lightning.Collaboration.WorkflowSerializer do defp normalize_kafka_configuration(trigger), do: trigger + defp normalize_response_codes(trigger) do + trigger + |> normalize_integer_field("webhook_response_success_code") + |> normalize_integer_field("webhook_response_error_code") + end + + defp normalize_integer_field(map, key) do + case Map.fetch(map, key) do + {:ok, value} when is_float(value) -> Map.put(map, key, trunc(value)) + _ -> map + end + end + defp extract_positions(positions_map) do Yex.Map.to_json(positions_map) end diff --git a/lib/lightning/workflows/trigger.ex b/lib/lightning/workflows/trigger.ex index 60b7b79362e..1cd9328cd2b 100644 --- a/lib/lightning/workflows/trigger.ex +++ b/lib/lightning/workflows/trigger.ex @@ -55,8 +55,8 @@ defmodule Lightning.Workflows.Trigger do field :delete, :boolean, virtual: true field :has_auth_method, :boolean, virtual: true - field :webhook_response_success_code, :integer, virtual: true, default: 200 - field :webhook_response_error_code, :integer, virtual: true, default: 400 + field :webhook_response_success_code, :integer + field :webhook_response_error_code, :integer many_to_many :webhook_auth_methods, Lightning.Workflows.WebhookAuthMethod, join_through: "trigger_webhook_auth_methods", @@ -106,7 +106,9 @@ defmodule Lightning.Workflows.Trigger do :cron_expression, :cron_cursor_job_id, :has_auth_method, - :webhook_reply + :webhook_reply, + :webhook_response_success_code, + :webhook_response_error_code ]) end @@ -143,6 +145,7 @@ defmodule Lightning.Workflows.Trigger do |> put_change(:cron_cursor_job_id, nil) |> put_change(:kafka_configuration, nil) |> put_default(:webhook_reply, :before_start) + |> maybe_reset_response_codes() :cron -> changeset @@ -150,6 +153,8 @@ defmodule Lightning.Workflows.Trigger do |> validate_cron() |> put_change(:kafka_configuration, nil) |> put_change(:webhook_reply, nil) + |> put_change(:webhook_response_success_code, nil) + |> put_change(:webhook_response_error_code, nil) :kafka -> changeset @@ -157,12 +162,28 @@ defmodule Lightning.Workflows.Trigger do |> put_change(:cron_cursor_job_id, nil) |> validate_required([:kafka_configuration]) |> put_change(:webhook_reply, nil) + |> put_change(:webhook_response_success_code, nil) + |> put_change(:webhook_response_error_code, nil) nil -> changeset end end + defp maybe_reset_response_codes(changeset) do + case fetch_field!(changeset, :webhook_reply) do + :after_completion -> + changeset + |> put_default(:webhook_response_success_code, 200) + |> put_default(:webhook_response_error_code, 400) + + _ -> + changeset + |> put_change(:webhook_response_success_code, nil) + |> put_change(:webhook_response_error_code, nil) + end + end + defp put_default(changeset, field, value) do changeset |> get_field(field) diff --git a/lib/lightning_web/channels/run_channel.ex b/lib/lightning_web/channels/run_channel.ex index c7df66b0d04..fbc9b399092 100644 --- a/lib/lightning_web/channels/run_channel.ex +++ b/lib/lightning_web/channels/run_channel.ex @@ -365,10 +365,10 @@ defmodule LightningWeb.RunChannel do end defp determine_status_code(:success, trigger), - do: trigger.webhook_response_success_code + do: trigger.webhook_response_success_code || 200 defp determine_status_code(_state, trigger), - do: trigger.webhook_response_error_code + do: trigger.webhook_response_error_code || 400 defp update_scrubber(nil, samples, basic_auth) do Scrubber.start_link(samples: samples, basic_auth: basic_auth) diff --git a/priv/repo/migrations/20260422052858_add_webhook_response_codes_to_triggers.exs b/priv/repo/migrations/20260422052858_add_webhook_response_codes_to_triggers.exs new file mode 100644 index 00000000000..828169cbaa0 --- /dev/null +++ b/priv/repo/migrations/20260422052858_add_webhook_response_codes_to_triggers.exs @@ -0,0 +1,10 @@ +defmodule Lightning.Repo.Migrations.AddWebhookResponseCodesToTriggers do + use Ecto.Migration + + def change do + alter table(:triggers) do + add :webhook_response_success_code, :integer, null: true + add :webhook_response_error_code, :integer, null: true + end + end +end From 5dd0bfe6446ee9e58eddf548a93b13a72acd3354 Mon Sep 17 00:00:00 2001 From: Frank Midigo Date: Wed, 22 Apr 2026 09:08:20 +0300 Subject: [PATCH 03/21] add fields to snapshot --- lib/lightning/workflows/snapshot.ex | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/lightning/workflows/snapshot.ex b/lib/lightning/workflows/snapshot.ex index 07108390739..468b9b2196d 100644 --- a/lib/lightning/workflows/snapshot.ex +++ b/lib/lightning/workflows/snapshot.ex @@ -64,6 +64,9 @@ defmodule Lightning.Workflows.Snapshot do values: [:before_start, :after_completion, :custom], default: :before_start + field :webhook_response_success_code, :integer + field :webhook_response_error_code, :integer + many_to_many :webhook_auth_methods, WebhookAuthMethod, join_through: "trigger_webhook_auth_methods", on_replace: :delete From 54535d165e42a4f886b921e48ba0f20882f05a32 Mon Sep 17 00:00:00 2001 From: Frank Midigo Date: Fri, 24 Apr 2026 07:48:45 +0300 Subject: [PATCH 04/21] use jsonb config --- .../adapters/YAMLStateToYDoc.ts | 8 +- .../components/inspector/TriggerForm.tsx | 247 +++++++++--------- .../js/collaborative-editor/types/session.ts | 8 +- .../js/collaborative-editor/types/trigger.ts | 22 +- assets/js/yaml/types.ts | 8 +- .../collaboration/workflow_serializer.ex | 43 +-- lib/lightning/workflows/trigger.ex | 47 ++-- .../triggers/sync_webhook_response_config.ex | 25 ++ lib/lightning_web/channels/run_channel.ex | 88 +++---- ...dd_webhook_response_codes_to_triggers.exs} | 3 +- 10 files changed, 261 insertions(+), 238 deletions(-) create mode 100644 lib/lightning/workflows/triggers/sync_webhook_response_config.ex rename priv/repo/migrations/{20260422052858_add_webhook_response_codes_to_triggers.exs => 20260422052859_add_webhook_response_codes_to_triggers.exs} (56%) diff --git a/assets/js/collaborative-editor/adapters/YAMLStateToYDoc.ts b/assets/js/collaborative-editor/adapters/YAMLStateToYDoc.ts index f3f946f62f4..c62ab4fcf86 100644 --- a/assets/js/collaborative-editor/adapters/YAMLStateToYDoc.ts +++ b/assets/js/collaborative-editor/adapters/YAMLStateToYDoc.ts @@ -78,12 +78,8 @@ export class YAMLStateToYDoc { if (trigger.type === 'webhook') { triggerMap.set('webhook_reply', trigger.webhook_reply ?? null); triggerMap.set( - 'webhook_response_success_code', - trigger.webhook_response_success_code ?? null - ); - triggerMap.set( - 'webhook_response_error_code', - trigger.webhook_response_error_code ?? null + 'sync_webhook_response_config', + trigger.sync_webhook_response_config ?? null ); } diff --git a/assets/js/collaborative-editor/components/inspector/TriggerForm.tsx b/assets/js/collaborative-editor/components/inspector/TriggerForm.tsx index 01e336f8527..06dbb1ffc0c 100644 --- a/assets/js/collaborative-editor/components/inspector/TriggerForm.tsx +++ b/assets/js/collaborative-editor/components/inspector/TriggerForm.tsx @@ -398,7 +398,6 @@ export function TriggerForm({ trigger }: TriggerFormProps) { e.target.value as | 'before_start' | 'after_completion' - | 'custom' ) } onBlur={field.handleBlur} @@ -419,17 +418,14 @@ export function TriggerForm({ trigger }: TriggerFormProps) { Async (default) -

{field.state.value === 'after_completion' - ? 'Responds with the final output state after the run completes. You can customise the HTTP status codes returned below.' - : field.state.value === 'custom' - ? 'The job code itself sends the HTTP response using the send() function.' - : 'Responds immediately with the enqueued work order ID.'} + ? 'Holds the HTTP connection open and responds when the run completes. Jobs can send an early response using setWebhookResponse().' + : 'Responds immediately with the enqueued work order ID.'}

{field.state.meta.errors.map(error => (

- {/* Conditional response code fields for after_completion */} + {/* Conditional config fields for after_completion */} {replyField => { if (replyField.state.value !== 'after_completion') { @@ -451,125 +447,136 @@ export function TriggerForm({ trigger }: TriggerFormProps) { } return ( -

- {/* Success Status Code */} - - {field => ( -
- - - field.handleChange( - e.target.value === '' - ? null - : parseInt(e.target.value, 10) - ) - } - onBlur={field.handleBlur} - disabled={isReadOnly} - className={cn( - 'block w-full px-3 py-2', - 'border rounded-md text-sm', - field.state.meta.errors.length > 0 - ? 'border-red-300 text-red-900 focus:border-red-500 focus:ring-red-500' - : 'border-slate-300 focus:border-indigo-500 focus:ring-indigo-500', - 'focus:outline-none focus:ring-1', - 'disabled:opacity-50 disabled:cursor-not-allowed' - )} - /> -

- HTTP status code returned when the run - succeeds (default: 200) -

- {field.state.meta.errors.map(error => ( -

+ {field => { + const config = field.state.value as { + code: number | null; + body: Record | null; + } | null; + + return ( +

+ {/* Status Code */} +
+ + { + const raw = e.target.value; + const parsed = + raw === '' + ? null + : parseInt(raw, 10); + field.handleChange({ + ...(config ?? { body: null }), + code: parsed, + }); + }} + onBlur={field.handleBlur} + disabled={isReadOnly} + className={cn( + 'block w-full px-3 py-2', + 'border rounded-md text-sm', + field.state.meta.errors.length > 0 + ? 'border-red-300 text-red-900 ' + + 'focus:border-red-500 ' + + 'focus:ring-red-500' + : 'border-slate-300 ' + + 'focus:border-indigo-500 ' + + 'focus:ring-indigo-500', + 'focus:outline-none focus:ring-1', + 'disabled:opacity-50 ' + + 'disabled:cursor-not-allowed' + )} + /> +

+ HTTP status code returned to the caller + (default: 200 for success, 400 + otherwise).

- ))} -
- )} - +
- {/* Error Status Code */} - - {field => ( -
- - - field.handleChange( - e.target.value === '' - ? null - : parseInt(e.target.value, 10) - ) - } - onBlur={field.handleBlur} - disabled={isReadOnly} - className={cn( - 'block w-full px-3 py-2', - 'border rounded-md text-sm', - field.state.meta.errors.length > 0 - ? 'border-red-300 text-red-900 focus:border-red-500 focus:ring-red-500' - : 'border-slate-300 focus:border-indigo-500 focus:ring-indigo-500', - 'focus:outline-none focus:ring-1', - 'disabled:opacity-50 disabled:cursor-not-allowed' - )} - /> -

- HTTP status code returned when the run - fails (default: 400) -

- {field.state.meta.errors.map(error => ( -

+ +