From 86ee0980b78f0b50a1af606a4dbb5814114806e2 Mon Sep 17 00:00:00 2001 From: joseph-sentry Date: Mon, 15 Jun 2026 13:06:44 -0400 Subject: [PATCH 1/3] feat(pipedream): allow rollback final_pipeline override Add optional `rollback.final_pipeline` to anchor the rollback material on an earlier group instead of the last pipeline in the chain. Useful when the tail group is flaky and would otherwise starve the rollback target pool. Errors at compile time if the named group doesn't exist. Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 13 ++++++ libs/pipedream.libsonnet | 17 ++++++- test/pipedream.js | 25 +++++++++++ ...ollback-bad-final-pipeline.failing.jsonnet | 43 ++++++++++++++++++ .../rollback-final-pipeline-override.jsonnet | 44 +++++++++++++++++++ 5 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 test/testdata/fixtures/pipedream/rollback-bad-final-pipeline.failing.jsonnet create mode 100644 test/testdata/fixtures/pipedream/rollback-final-pipeline-override.jsonnet diff --git a/README.md b/README.md index 0c2d6fd..125cf6c 100644 --- a/README.md +++ b/README.md @@ -149,6 +149,19 @@ local pipedream_config = { stage: 'example_stage', # The elastic agent profile to run the rollback pipeline as elastic_profile_id: 'example_profile', + # (Optional) The stage on the source-of-truth pipeline that the rollback + # material watches. Defaults to that pipeline's final stage + # (`pipeline-complete`). Set to an earlier stage (e.g. 'deploy-primary') + # so a SHA becomes rollback-eligible without waiting for the trailing + # noop stage. + # final_stage: 'deploy-primary', + # (Optional) Which group pipeline is the source of truth for rollback + # revisions. Defaults to the last group in the chain (e.g. `st`). Set to a + # group name (e.g. 'us') to anchor rollback eligibility on an earlier group + # when the tail group is flaky and would otherwise starve the rollback + # target pool. Trade-off: an upstream-anchored target may not have been + # validated on the downstream groups the rollback then deploys it to. + # final_pipeline: 'us', }, # Set to true to auto-deploy changes (defaults to true) diff --git a/libs/pipedream.libsonnet b/libs/pipedream.libsonnet index 3c4434d..cdcf108 100644 --- a/libs/pipedream.libsonnet +++ b/libs/pipedream.libsonnet @@ -137,7 +137,22 @@ local pipedream_trigger_pipeline(pipedream_config) = local pipedream_rollback_pipeline(pipedream_config, service_pipelines, trigger_pipeline) = if std.objectHas(pipedream_config, 'rollback') then local name = pipedream_config.name; - local final_pipeline = service_pipelines[std.length(service_pipelines) - 1]; + // The rollback material's "source of truth" pipeline. By default this is + // the last pipeline in the chain (the final group, e.g. `st`), so a SHA is + // only rollback-eligible once it has deployed all the way through. Set + // `rollback.final_pipeline` to a group name (e.g. 'us') to anchor rollback + // eligibility on an earlier group instead — useful when the tail group is + // flaky and would otherwise starve the rollback target pool. Note the + // trade-off: anchoring upstream means a rollback target may not have been + // validated on the downstream groups it then gets deployed to. + local final_pipeline = + if std.objectHas(pipedream_config.rollback, 'final_pipeline') then + local target = pipeline_name(name, pipedream_config.rollback.final_pipeline); + local matches = std.filter(function(p) p.name == target, service_pipelines); + assert std.length(matches) > 0 : "Rollback final_pipeline '" + target + "' not found in service pipelines"; + matches[0] + else + service_pipelines[std.length(service_pipelines) - 1]; // Rollbacks work by calling two devinfra-deployment-infra scripts: // gocd-pause-and-cancel-pipelines diff --git a/test/pipedream.js b/test/pipedream.js index 109ac59..e6c0e78 100644 --- a/test/pipedream.js +++ b/test/pipedream.js @@ -150,6 +150,31 @@ test("rollback: invalid final stage errors", (t) => { ); }); +test("rollback: final_pipeline anchors material on an earlier group", async (t) => { + const got = await render_fixture( + "pipedream/rollback-final-pipeline-override.jsonnet", + false, + ); + + const r = got.pipelines["rollback-example"]; + t.truthy(r); + // Material watches the `us` group's final stage, not the default last group. + t.truthy(r.materials["deploy-example-us-deploy"]); + t.is(r.materials["deploy-example-us-deploy"].pipeline, "deploy-example-us"); + // Rollback still re-runs every region pipeline. + t.truthy(r.environment_variables.REGION_PIPELINE_FLAGS.includes("deploy-example-st")); +}); + +test("rollback: unknown final_pipeline errors", (t) => { + const err = t.throws(() => + get_fixture_content( + "pipedream/rollback-bad-final-pipeline.failing.jsonnet", + false, + ), + ); + t.true(err.message.includes("not found in service pipelines")); +}); + test("conflicting stage properties across regions errors", (t) => { const err = t.throws(() => get_fixture_content( diff --git a/test/testdata/fixtures/pipedream/rollback-bad-final-pipeline.failing.jsonnet b/test/testdata/fixtures/pipedream/rollback-bad-final-pipeline.failing.jsonnet new file mode 100644 index 0000000..9545972 --- /dev/null +++ b/test/testdata/fixtures/pipedream/rollback-bad-final-pipeline.failing.jsonnet @@ -0,0 +1,43 @@ +local pipedream = import '../../../../libs/pipedream.libsonnet'; + +// final_pipeline names a group that doesn't exist -> should error at compile +// time rather than producing a rollback pipeline with a dangling material. + +local pipedream_config = { + name: 'example', + auto_deploy: true, + rollback: { + material_name: 'example_repo', + stage: 'deploy', + elastic_profile_id: 'example', + final_pipeline: 'this-group-does-not-exist', + }, +}; + +local sample = { + pipeline(region):: { + materials: { + example_repo: { + git: 'git@github.com:getsentry/example.git', + branch: 'master', + destination: 'example', + }, + }, + stages: [ + { + deploy: { + jobs: { + deploy: { + elastic_profile_id: 'example', + tasks: [ + { script: './deploy.sh --region=' + region }, + ], + }, + }, + }, + }, + ], + }, +}; + +pipedream.render(pipedream_config, sample.pipeline) diff --git a/test/testdata/fixtures/pipedream/rollback-final-pipeline-override.jsonnet b/test/testdata/fixtures/pipedream/rollback-final-pipeline-override.jsonnet new file mode 100644 index 0000000..23e98b2 --- /dev/null +++ b/test/testdata/fixtures/pipedream/rollback-final-pipeline-override.jsonnet @@ -0,0 +1,44 @@ +local pipedream = import '../../../../libs/pipedream.libsonnet'; + +// Anchor the rollback material on the `us` group instead of the default last +// group (`st`). A SHA becomes rollback-eligible once `us` reaches its final +// stage, so a flaky tail group no longer starves the rollback target pool. + +local pipedream_config = { + name: 'example', + auto_deploy: true, + rollback: { + material_name: 'example_repo', + stage: 'deploy', + elastic_profile_id: 'example', + final_pipeline: 'us', + }, +}; + +local sample = { + pipeline(region):: { + materials: { + example_repo: { + git: 'git@github.com:getsentry/example.git', + branch: 'master', + destination: 'example', + }, + }, + stages: [ + { + deploy: { + jobs: { + deploy: { + elastic_profile_id: 'example', + tasks: [ + { script: './deploy.sh --region=' + region }, + ], + }, + }, + }, + }, + ], + }, +}; + +pipedream.render(pipedream_config, sample.pipeline) From 3775a550320d95f659257aea13c4867d47f78d1d Mon Sep 17 00:00:00 2001 From: joseph-sentry Date: Mon, 15 Jun 2026 13:11:31 -0400 Subject: [PATCH 2/3] test(pipedream): fix rollback material key assertion Material is keyed {pipeline}-{final_stage}; final_stage defaults to the appended pipeline-complete stage, not the deploy stage. Co-Authored-By: Claude Opus 4.8 (1M context) --- test/pipedream.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/pipedream.js b/test/pipedream.js index e6c0e78..18f3ce7 100644 --- a/test/pipedream.js +++ b/test/pipedream.js @@ -159,8 +159,9 @@ test("rollback: final_pipeline anchors material on an earlier group", async (t) const r = got.pipelines["rollback-example"]; t.truthy(r); // Material watches the `us` group's final stage, not the default last group. - t.truthy(r.materials["deploy-example-us-deploy"]); - t.is(r.materials["deploy-example-us-deploy"].pipeline, "deploy-example-us"); + t.truthy(r.materials["deploy-example-us-pipeline-complete"]); + t.is(r.materials["deploy-example-us-pipeline-complete"].pipeline, "deploy-example-us"); + t.is(r.materials["deploy-example-us-pipeline-complete"].stage, "pipeline-complete"); // Rollback still re-runs every region pipeline. t.truthy(r.environment_variables.REGION_PIPELINE_FLAGS.includes("deploy-example-st")); }); From 513e57522ccedb8ac8071d6321cbfb013c8c4170 Mon Sep 17 00:00:00 2001 From: joseph-sentry Date: Mon, 15 Jun 2026 13:17:09 -0400 Subject: [PATCH 3/3] style: prettier format test/pipedream.js Co-Authored-By: Claude Opus 4.8 (1M context) --- test/pipedream.js | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/test/pipedream.js b/test/pipedream.js index 18f3ce7..cc58d74 100644 --- a/test/pipedream.js +++ b/test/pipedream.js @@ -160,10 +160,18 @@ test("rollback: final_pipeline anchors material on an earlier group", async (t) t.truthy(r); // Material watches the `us` group's final stage, not the default last group. t.truthy(r.materials["deploy-example-us-pipeline-complete"]); - t.is(r.materials["deploy-example-us-pipeline-complete"].pipeline, "deploy-example-us"); - t.is(r.materials["deploy-example-us-pipeline-complete"].stage, "pipeline-complete"); + t.is( + r.materials["deploy-example-us-pipeline-complete"].pipeline, + "deploy-example-us", + ); + t.is( + r.materials["deploy-example-us-pipeline-complete"].stage, + "pipeline-complete", + ); // Rollback still re-runs every region pipeline. - t.truthy(r.environment_variables.REGION_PIPELINE_FLAGS.includes("deploy-example-st")); + t.truthy( + r.environment_variables.REGION_PIPELINE_FLAGS.includes("deploy-example-st"), + ); }); test("rollback: unknown final_pipeline errors", (t) => {