From a84c153db7bd125eb649535e20af19106947dca2 Mon Sep 17 00:00:00 2001 From: Chris Stephens Date: Fri, 12 Jun 2026 12:30:46 -0700 Subject: [PATCH 1/3] acc: Prefix unique test resource names with CI run id for attribution and sweeping Acceptance tests name cloud resources with a bare random string ($UNIQUE_NAME, e.g. 'osr5mzrrvzb73juixjoviti24y'). When such resources leak into shared test workspaces, nothing ties them back to a repo or CI run, which made a recent quota-exhaustion incident hard to diagnose and makes targeted cleanup impossible. When running against a cloud env (CLOUD_ENV set) with GITHUB_RUN_ID present, the harness now generates names of the form 'ci--', truncated to the same 26-character length as before so that length-constrained names built from $UNIQUE_NAME (e.g. 'app-$UNIQUE_NAME', exactly 30 chars, the app name limit) keep fitting. The character set stays lowercase alphanumerics plus '-', which tests already use adjacent to $UNIQUE_NAME in every resource type. Local runs and testserver runs are unchanged, and output masking is unaffected because the whole generated name (prefix included) is still replaced with [UNIQUE_NAME]. Also adds tools/sweep_test_resources.py, a small helper that lists (and with --delete removes) warehouses, pipelines and jobs whose names start with a given prefix, to be wired into CI job-end cleanup later. Co-authored-by: Isaac --- acceptance/acceptance_test.go | 28 +++++++++++ acceptance/unique_name_test.go | 24 ++++++++++ tools/sweep_test_resources.py | 85 ++++++++++++++++++++++++++++++++++ 3 files changed, 137 insertions(+) create mode 100644 acceptance/unique_name_test.go create mode 100755 tools/sweep_test_resources.py diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index a137d356f1..ffcd2b6d55 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -609,6 +609,29 @@ func getSkipReason(config *internal.TestConfig, configPath string) string { return "" } +// ciRunID matches a plausible numeric GitHub Actions run id. +var ciRunID = regexp.MustCompile(`^[0-9]{1,16}$`) + +// ciUniqueName embeds a CI run id into the random unique name: "ci--". +// The result has the same length as the input random name. This matters because some +// resource names built from $UNIQUE_NAME are at their length limit already (for example, +// "app-$UNIQUE_NAME" is exactly 30 characters, the maximum length of an app name). +// The character set also stays compatible with all uses: lowercase alphanumerics plus "-", +// which existing tests already use adjacent to $UNIQUE_NAME in every resource type. +// Returns the random name unchanged when runID is absent, malformed, or too long to leave +// enough random characters for collision avoidance between tests in the same run. +func ciUniqueName(runID, random string) string { + if !ciRunID.MatchString(runID) { + return random + } + prefix := "ci-" + runID + "-" + randLen := len(random) - len(prefix) + if randLen < 8 { + return random + } + return prefix + random[:randLen] +} + func runTest(t *testing.T, dir string, variant int, @@ -643,6 +666,11 @@ func runTest(t *testing.T, id := uuid.New() uniqueName := strings.ToLower(strings.Trim(base32.StdEncoding.EncodeToString(id[:]), "=")) + if isRunningOnCloud { + // In CI runs against real workspaces, embed the workflow run id into the name so + // that leaked resources can be attributed to a repo/CI run and swept by prefix. + uniqueName = ciUniqueName(os.Getenv("GITHUB_RUN_ID"), uniqueName) + } repls.Set(uniqueName, "[UNIQUE_NAME]") var tmpDir string diff --git a/acceptance/unique_name_test.go b/acceptance/unique_name_test.go new file mode 100644 index 0000000000..6af5105c21 --- /dev/null +++ b/acceptance/unique_name_test.go @@ -0,0 +1,24 @@ +package acceptance_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCIUniqueName(t *testing.T) { + // 26 lowercase base32 characters, like the generated unique name. + random := "osr5mzrrvzb73juixjoviti24y" + + // Run id embedded, same length as input, sweepable prefix. + assert.Equal(t, "ci-15799017600-osr5mzrrvzb", ciUniqueName("15799017600", random)) + assert.Equal(t, "ci-1-osr5mzrrvzb73juixjovi", ciUniqueName("1", random)) + + // No or invalid run id: unchanged. + assert.Equal(t, random, ciUniqueName("", random)) + assert.Equal(t, random, ciUniqueName("abc123", random)) + assert.Equal(t, random, ciUniqueName("123 456", random)) + + // Run id too long to leave enough randomness: unchanged. + assert.Equal(t, random, ciUniqueName("123456789012345", random)) +} diff --git a/tools/sweep_test_resources.py b/tools/sweep_test_resources.py new file mode 100755 index 0000000000..8279e2cf3f --- /dev/null +++ b/tools/sweep_test_resources.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Sweep leaked acceptance-test resources by name prefix. + +Lists (and with --delete, deletes) warehouses, pipelines and jobs whose names +start with the given prefix, e.g. the per-run prefix "ci--" that +the acceptance harness embeds into $UNIQUE_NAME on CI cloud runs. + +Authentication is taken from the environment (DATABRICKS_HOST, DATABRICKS_TOKEN +or any other auth supported by the databricks CLI). + +Usage: + tools/sweep_test_resources.py ci-15799017600- # dry run: list only + tools/sweep_test_resources.py ci-15799017600- --delete # delete matches +""" + +import argparse +import json +import subprocess +import sys + + +def run_json(*args): + out = subprocess.check_output(["databricks", *args, "--output", "json"], text=True) + return json.loads(out) if out.strip() else [] + + +def sweep(kind, items, name_of, id_of, delete_args, prefix, delete): + failures = 0 + for item in items: + name = name_of(item) or "" + if not name.startswith(prefix): + continue + res_id = str(id_of(item)) + print(f"{kind}\t{res_id}\t{name}") + if delete: + try: + subprocess.check_call(["databricks", *delete_args, res_id]) + except subprocess.CalledProcessError as e: + print(f"failed to delete {kind} {res_id}: {e}", file=sys.stderr) + failures += 1 + return failures + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("prefix", help="resource name prefix, e.g. ci--") + parser.add_argument("--delete", action="store_true", help="delete matches (default: list only)") + args = parser.parse_args() + + if not args.prefix: + parser.error("prefix must not be empty") + + failures = 0 + failures += sweep( + "warehouse", + run_json("warehouses", "list"), + lambda w: w.get("name"), + lambda w: w.get("id"), + ["warehouses", "delete"], + args.prefix, + args.delete, + ) + failures += sweep( + "pipeline", + run_json("pipelines", "list-pipelines"), + lambda p: p.get("name"), + lambda p: p.get("pipeline_id"), + ["pipelines", "delete"], + args.prefix, + args.delete, + ) + failures += sweep( + "job", + run_json("jobs", "list"), + lambda j: j.get("settings", {}).get("name"), + lambda j: j.get("job_id"), + ["jobs", "delete"], + args.prefix, + args.delete, + ) + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main()) From dd8cb44a17c75aa5254a21d1c823e18d65bcaddb Mon Sep 17 00:00:00 2001 From: Chris Stephens Date: Fri, 12 Jun 2026 13:07:34 -0700 Subject: [PATCH 2/3] acc: Tighten comments Co-authored-by: Isaac --- acceptance/acceptance_test.go | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index ffcd2b6d55..ebc7ed74bb 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -609,17 +609,13 @@ func getSkipReason(config *internal.TestConfig, configPath string) string { return "" } -// ciRunID matches a plausible numeric GitHub Actions run id. var ciRunID = regexp.MustCompile(`^[0-9]{1,16}$`) -// ciUniqueName embeds a CI run id into the random unique name: "ci--". -// The result has the same length as the input random name. This matters because some -// resource names built from $UNIQUE_NAME are at their length limit already (for example, -// "app-$UNIQUE_NAME" is exactly 30 characters, the maximum length of an app name). -// The character set also stays compatible with all uses: lowercase alphanumerics plus "-", -// which existing tests already use adjacent to $UNIQUE_NAME in every resource type. -// Returns the random name unchanged when runID is absent, malformed, or too long to leave -// enough random characters for collision avoidance between tests in the same run. +// ciUniqueName embeds a CI run id into the random unique name as "ci--", +// preserving the input length: names built from $UNIQUE_NAME can already be at a +// resource name limit ("app-$UNIQUE_NAME" is exactly the 30-char app name maximum). +// Returns random unchanged when runID is absent, malformed, or too long to leave +// enough random characters. func ciUniqueName(runID, random string) string { if !ciRunID.MatchString(runID) { return random @@ -667,8 +663,7 @@ func runTest(t *testing.T, id := uuid.New() uniqueName := strings.ToLower(strings.Trim(base32.StdEncoding.EncodeToString(id[:]), "=")) if isRunningOnCloud { - // In CI runs against real workspaces, embed the workflow run id into the name so - // that leaked resources can be attributed to a repo/CI run and swept by prefix. + // Embed the CI run id so leaked resources can be attributed to a run and swept by prefix. uniqueName = ciUniqueName(os.Getenv("GITHUB_RUN_ID"), uniqueName) } repls.Set(uniqueName, "[UNIQUE_NAME]") From 162ca50809414ede5db5a421c2f8134656a3fb3b Mon Sep 17 00:00:00 2001 From: Chris Stephens Date: Fri, 12 Jun 2026 13:20:44 -0700 Subject: [PATCH 3/3] acc: Apply CI run id prefix whenever GITHUB_RUN_ID is present Drop the CLOUD_ENV gate: ciUniqueName already no-ops without a valid run id, and prefixing testserver runs in CI exercises the masking path everywhere instead of only on cloud runs. Co-authored-by: Isaac --- acceptance/acceptance_test.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index ebc7ed74bb..96845ea8d2 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -662,10 +662,8 @@ func runTest(t *testing.T, id := uuid.New() uniqueName := strings.ToLower(strings.Trim(base32.StdEncoding.EncodeToString(id[:]), "=")) - if isRunningOnCloud { - // Embed the CI run id so leaked resources can be attributed to a run and swept by prefix. - uniqueName = ciUniqueName(os.Getenv("GITHUB_RUN_ID"), uniqueName) - } + // Embed the CI run id, when present, so leaked resources can be attributed to a run and swept by prefix. + uniqueName = ciUniqueName(os.Getenv("GITHUB_RUN_ID"), uniqueName) repls.Set(uniqueName, "[UNIQUE_NAME]") var tmpDir string