From c0cad409b1495aa487779d1e0159812ed48fedce Mon Sep 17 00:00:00 2001 From: Alfonso Noriega Date: Mon, 27 Apr 2026 16:59:47 +0200 Subject: [PATCH 1/5] Add bin/update-observe.js to automate post-release Observe updates --- bin/observe-cli-resources.json | 90 ++++++++++++++++ bin/update-observe.js | 185 +++++++++++++++++++++++++++++++++ package.json | 1 + 3 files changed, 276 insertions(+) create mode 100644 bin/observe-cli-resources.json create mode 100755 bin/update-observe.js diff --git a/bin/observe-cli-resources.json b/bin/observe-cli-resources.json new file mode 100644 index 00000000000..86d88af2f2e --- /dev/null +++ b/bin/observe-cli-resources.json @@ -0,0 +1,90 @@ +{ + "$schema": "Templates for Observe resources that pin the latest CLI version. Use ${version} as placeholder. Consumed by bin/update-observe.js.", + "endpoint": "https://shopify-monitoring.shopifycloud.com/query", + "service": "cli", + "vaultTeamId": "2238", + "slos": [ + { + "id": "6cb7fa5f-6f65-417b-aca1-3986c5976068", + "name": "[CLI - Dev Platform] Correctness - App Deploy (${version})", + "kind": "raw", + "expression": "1 - (sum(rate(cli_commands_total{job=~\"@shopify/(app|store):.*\", exit=\"unexpected_error\", cli_version=\"${version}\"} [15m])) / sum(rate(cli_commands_total{job=~\"@shopify/(app|store):.*\", cli_version=\"${version}\"} [15m]))) * on() (sum(rate(cli_commands_total{job=~\"@shopify/(app|store):.*\", cli_version=\"${version}\"} [15m])) > 0.1)" + }, + { + "id": "d0131f4a-b9de-4704-9462-9c6ae3de080a", + "name": "[CLI - Dev Platform] Correctness (Version ${version})", + "kind": "raw", + "expression": "1 - (sum(rate(cli_commands_total{job=~\"@shopify/(app|store):.*\", exit=\"unexpected_error\", cli_version=\"${version}\"} [15m])) / sum(rate(cli_commands_total{job=~\"@shopify/(app|store):.*\", cli_version=\"${version}\"} [15m]))) * on() (sum(rate(cli_commands_total{job=~\"@shopify/(app|store):.*\", cli_version=\"${version}\"}[15m])) > 0.1)" + }, + { + "id": "c1a167e1-9f23-4788-bb4e-7240e556d9b5", + "name": "[CLI - Dev Platform] p50 Latency (Version ${version})", + "kind": "histogram", + "metricName": "cli_commands_duration_ms", + "metricType": "NATIVE_HISTOGRAM", + "minThreshold": "0", + "maxThreshold": "10000", + "filters": [ + {"key": "job", "op": "!=", "value": "@shopify/app::app dev"}, + {"key": "job", "op": "=~", "value": "@shopify/(app|store):.*"}, + {"key": "job", "op": "!=", "value": "@shopify/cli::app dev"}, + {"key": "cli_version", "op": "=", "value": "${version}"} + ] + }, + { + "id": "1415409f-2750-4bdc-9333-7696f6cf1204", + "name": "[CLI - Dev Platform] p75 Latency (Version ${version})", + "kind": "histogram", + "metricName": "cli_commands_duration_ms", + "metricType": "NATIVE_HISTOGRAM", + "minThreshold": "0", + "maxThreshold": "10000", + "filters": [ + {"key": "job", "op": "!=", "value": "@shopify/app::app dev"}, + {"key": "job", "op": "=~", "value": "@shopify/(app|store):.*"}, + {"key": "job", "op": "!=", "value": "@shopify/cli::app dev"}, + {"key": "cli_version", "op": "=", "value": "${version}"} + ] + } + ], + "alertRules": [ + { + "id": "2e527671-b84d-48af-9a52-2100cf295627", + "name": "[CLI - Dev Platform] Spike in Errors (v${version})", + "customErrorFilters": { + "filters": null, + "filter_groups": [ + { + "filters": [ + {"column": "resource.service.name", "op": "=", "value": "cli"}, + {"column": "handled", "op": "=", "value": "false"}, + {"column": "app.version", "op": "=", "value": "${version}"} + ], + "filter_groups": null, + "conjunction": "AND" + } + ], + "conjunction": "AND" + } + } + ], + "errorProjects": [ + { + "id": "8fdca84d-03fa-4eb2-9131-bd019e87c3b2", + "errorFilters": { + "filters": null, + "filter_groups": [ + { + "filters": [ + {"column": "handled", "op": "!=", "value": "true"}, + {"column": "app.version", "op": "=", "value": "${version}"} + ], + "filter_groups": null, + "conjunction": "AND" + } + ], + "conjunction": "AND" + } + } + ] +} diff --git a/bin/update-observe.js b/bin/update-observe.js new file mode 100755 index 00000000000..7f4ff12421d --- /dev/null +++ b/bin/update-observe.js @@ -0,0 +1,185 @@ +#!/usr/bin/env node +/** + * Updates Observe resources (SLOs, alert rules, error projects) that pin the + * latest CLI version. Replaces the manual post-release steps in the release + * runbook (https://github.com/Shopify/develop-app-inner-loop/issues/2694) that + * require clicking through observe.shopify.io to update version filters by hand. + * + * Usage: + * pnpm update-observe # uses version from packages/cli-kit/package.json + * pnpm update-observe -- --version=3.94.2 # explicit version + * pnpm update-observe -- --dry-run # print payloads without sending + * + * Auth: requires a Shopify Monitoring API token in $SHOPIFY_MONITORING_TOKEN. + * Get one at https://observe.shopify.io/profile (under "API Tokens") or via + * `dev observe-token` if that command is available. + * + * Templates live in bin/observe-cli-resources.json. To add or remove a managed + * resource, edit that file — no script changes required. + */ +import {readFileSync} from 'node:fs' +import {dirname, join} from 'node:path' +import {fileURLToPath} from 'node:url' +import {parseArgs} from 'node:util' + +const __dirname = dirname(fileURLToPath(import.meta.url)) +const repoRoot = join(__dirname, '..') + +const {values: args} = parseArgs({ + options: { + version: {type: 'string'}, + 'dry-run': {type: 'boolean', default: false}, + }, + strict: true, +}) + +const dryRun = args['dry-run'] +const version = args.version ?? defaultVersion() +if (!/^\d+\.\d+\.\d+$/.test(version)) { + fail(`Version must be semver X.Y.Z (got: ${version})`) +} + +const config = JSON.parse(readFileSync(join(__dirname, 'observe-cli-resources.json'), 'utf-8')) +const TOKEN = process.env.SHOPIFY_MONITORING_TOKEN +if (!dryRun && !TOKEN) { + fail('SHOPIFY_MONITORING_TOKEN is not set. See header of bin/update-observe.js for how to obtain one.') +} + +const interpolate = (value) => { + if (typeof value === 'string') return value.replaceAll('${version}', version) + if (Array.isArray(value)) return value.map(interpolate) + if (value && typeof value === 'object') { + return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, interpolate(v)])) + } + return value +} + +const SLO_MUTATION = ` + mutation Upsert($input: UpsertSLODefinitionInput!) { + upsertSLODefinition(input: $input) { + sloDefinition { id name } + error + } + } +` + +const RULE_MUTATION = ` + mutation Upsert($input: UpsertRuleConfigsInput!) { + upsertRuleConfigs(input: $input) { + validationErrors { field message } + } + } +` + +const ERROR_PROJECT_MUTATION = ` + mutation Upsert($input: ErrorProjectInput!) { + upsertErrorProject(input: $input) { + validationErrors { field message } + } + } +` + +const main = async () => { + console.log(`Updating Observe resources for ${config.service} → cli_version=${version}${dryRun ? ' (dry run)' : ''}`) + + const tasks = [ + ...config.slos.map((slo) => () => updateSlo(slo)), + ...config.alertRules.map((rule) => () => updateAlertRule(rule)), + ...config.errorProjects.map((project) => () => updateErrorProject(project)), + ] + + const results = await Promise.all(tasks.map((run) => run())) + const failed = results.filter((ok) => !ok).length + if (failed > 0) { + fail(`${failed} of ${results.length} updates failed.`) + } + console.log(`✓ ${results.length} resources updated.`) +} + +const updateSlo = async (slo) => { + const base = {id: slo.id, name: interpolate(slo.name)} + const sloSpec = + slo.kind === 'raw' + ? {...base, rawSLIInput: {expression: interpolate(slo.expression), weightExpression: 'vector(1)'}} + : { + ...base, + histogramSLIInput: { + metric: { + metricName: slo.metricName, + metricType: slo.metricType, + filters: interpolate(slo.filters), + }, + minThreshold: slo.minThreshold, + maxThreshold: slo.maxThreshold, + }, + } + return runMutation('SLO', slo.id, SLO_MUTATION, {input: {slo: sloSpec}}, (data) => data.upsertSLODefinition?.error) +} + +const updateAlertRule = async (rule) => { + const ruleConfigInput = { + id: rule.id, + name: interpolate(rule.name), + vaultTeamId: config.vaultTeamId, + customErrorFilters: JSON.stringify(interpolate(rule.customErrorFilters)), + } + return runMutation( + 'AlertRule', + rule.id, + RULE_MUTATION, + {input: {ruleConfigInputs: [ruleConfigInput]}}, + (data) => data.upsertRuleConfigs?.validationErrors?.length ? JSON.stringify(data.upsertRuleConfigs.validationErrors) : null, + ) +} + +const updateErrorProject = async (project) => { + const errorProjectInput = { + id: project.id, + name: config.service, + errorFilters: JSON.stringify(interpolate(project.errorFilters)), + } + return runMutation( + 'ErrorProject', + project.id, + ERROR_PROJECT_MUTATION, + {input: errorProjectInput}, + (data) => data.upsertErrorProject?.validationErrors?.length ? JSON.stringify(data.upsertErrorProject.validationErrors) : null, + ) +} + +const runMutation = async (kind, id, query, variables, getError) => { + if (dryRun) { + console.log(`-- ${kind} ${id} --`) + console.log(JSON.stringify(variables, null, 2)) + return true + } + try { + const res = await fetch(config.endpoint, { + method: 'POST', + headers: {'Content-Type': 'application/json', Authorization: `Bearer ${TOKEN}`}, + body: JSON.stringify({query, variables}), + }) + const json = await res.json() + const err = json.errors ? JSON.stringify(json.errors) : getError(json.data ?? {}) + if (err) { + console.error(`✖ ${kind} ${id}: ${err}`) + return false + } + console.log(`✓ ${kind} ${id}`) + return true + } catch (error) { + console.error(`✖ ${kind} ${id}: ${error.message}`) + return false + } +} + +function defaultVersion() { + return JSON.parse(readFileSync(join(repoRoot, 'packages/cli-kit/package.json'), 'utf-8')).version +} + +function fail(message) { + console.error(`Error: ${message}`) + process.exit(1) +} + +await main() diff --git a/package.json b/package.json index 74677c4fbec..0ffe1bf61d1 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "refresh-readme": "nx run-many --target=refresh-readme --all --skip-nx-cache", "release": "./bin/release", "post-release": "./bin/post-release", + "update-observe": "node bin/update-observe.js", "shopify:run": "node packages/cli/bin/dev.js", "shopify": "nx build cli && node packages/cli/bin/dev.js", "test:e2e": "nx run-many --target=build --projects=cli,create-app --skip-nx-cache && pnpm --filter e2e exec playwright test", From b1181652e55cec9e4ac3167d816220fc927e58f9 Mon Sep 17 00:00:00 2001 From: Alfonso Noriega Date: Mon, 27 Apr 2026 17:19:57 +0200 Subject: [PATCH 2/5] Make --version mandatory; add --resource flag for single-resource updates --- bin/observe-cli-resources.json | 6 ++++ bin/update-observe.js | 52 ++++++++++++++++++++++------------ 2 files changed, 40 insertions(+), 18 deletions(-) diff --git a/bin/observe-cli-resources.json b/bin/observe-cli-resources.json index 86d88af2f2e..02d077cb7f4 100644 --- a/bin/observe-cli-resources.json +++ b/bin/observe-cli-resources.json @@ -5,18 +5,21 @@ "vaultTeamId": "2238", "slos": [ { + "key": "slo-correctness-app-deploy", "id": "6cb7fa5f-6f65-417b-aca1-3986c5976068", "name": "[CLI - Dev Platform] Correctness - App Deploy (${version})", "kind": "raw", "expression": "1 - (sum(rate(cli_commands_total{job=~\"@shopify/(app|store):.*\", exit=\"unexpected_error\", cli_version=\"${version}\"} [15m])) / sum(rate(cli_commands_total{job=~\"@shopify/(app|store):.*\", cli_version=\"${version}\"} [15m]))) * on() (sum(rate(cli_commands_total{job=~\"@shopify/(app|store):.*\", cli_version=\"${version}\"} [15m])) > 0.1)" }, { + "key": "slo-correctness", "id": "d0131f4a-b9de-4704-9462-9c6ae3de080a", "name": "[CLI - Dev Platform] Correctness (Version ${version})", "kind": "raw", "expression": "1 - (sum(rate(cli_commands_total{job=~\"@shopify/(app|store):.*\", exit=\"unexpected_error\", cli_version=\"${version}\"} [15m])) / sum(rate(cli_commands_total{job=~\"@shopify/(app|store):.*\", cli_version=\"${version}\"} [15m]))) * on() (sum(rate(cli_commands_total{job=~\"@shopify/(app|store):.*\", cli_version=\"${version}\"}[15m])) > 0.1)" }, { + "key": "slo-p50-latency", "id": "c1a167e1-9f23-4788-bb4e-7240e556d9b5", "name": "[CLI - Dev Platform] p50 Latency (Version ${version})", "kind": "histogram", @@ -32,6 +35,7 @@ ] }, { + "key": "slo-p75-latency", "id": "1415409f-2750-4bdc-9333-7696f6cf1204", "name": "[CLI - Dev Platform] p75 Latency (Version ${version})", "kind": "histogram", @@ -49,6 +53,7 @@ ], "alertRules": [ { + "key": "alert-spike-errors", "id": "2e527671-b84d-48af-9a52-2100cf295627", "name": "[CLI - Dev Platform] Spike in Errors (v${version})", "customErrorFilters": { @@ -70,6 +75,7 @@ ], "errorProjects": [ { + "key": "error-project-cli", "id": "8fdca84d-03fa-4eb2-9131-bd019e87c3b2", "errorFilters": { "filters": null, diff --git a/bin/update-observe.js b/bin/update-observe.js index 7f4ff12421d..4611934072f 100755 --- a/bin/update-observe.js +++ b/bin/update-observe.js @@ -6,9 +6,15 @@ * require clicking through observe.shopify.io to update version filters by hand. * * Usage: - * pnpm update-observe # uses version from packages/cli-kit/package.json - * pnpm update-observe -- --version=3.94.2 # explicit version - * pnpm update-observe -- --dry-run # print payloads without sending + * pnpm update-observe -- --version=3.94.2 # update all resources + * pnpm update-observe -- --version=3.94.2 --resource=slo-p50-latency # update one resource + * pnpm update-observe -- --version=3.94.2 --dry-run # print payloads without sending + * + * --version is required and must be semver X.Y.Z. + * --resource selects a single resource by its `key` from + * bin/observe-cli-resources.json (e.g. slo-correctness-app-deploy, + * slo-correctness, slo-p50-latency, slo-p75-latency, alert-spike-errors, + * error-project-cli). Omit to update all of them. * * Auth: requires a Shopify Monitoring API token in $SHOPIFY_MONITORING_TOKEN. * Get one at https://observe.shopify.io/profile (under "API Tokens") or via @@ -23,23 +29,41 @@ import {fileURLToPath} from 'node:url' import {parseArgs} from 'node:util' const __dirname = dirname(fileURLToPath(import.meta.url)) -const repoRoot = join(__dirname, '..') const {values: args} = parseArgs({ options: { version: {type: 'string'}, + resource: {type: 'string'}, 'dry-run': {type: 'boolean', default: false}, }, strict: true, }) const dryRun = args['dry-run'] -const version = args.version ?? defaultVersion() +const version = args.version +if (!version) { + fail('--version is required (e.g. --version=3.94.2)') +} if (!/^\d+\.\d+\.\d+$/.test(version)) { fail(`Version must be semver X.Y.Z (got: ${version})`) } const config = JSON.parse(readFileSync(join(__dirname, 'observe-cli-resources.json'), 'utf-8')) + +const allResources = [ + ...config.slos.map((slo) => ({type: 'slo', value: slo})), + ...config.alertRules.map((rule) => ({type: 'alert', value: rule})), + ...config.errorProjects.map((project) => ({type: 'errorProject', value: project})), +] + +let selectedResources = allResources +if (args.resource) { + selectedResources = allResources.filter((r) => r.value.key === args.resource) + if (selectedResources.length === 0) { + const keys = allResources.map((r) => r.value.key).join(', ') + fail(`No resource with key "${args.resource}". Valid keys: ${keys}`) + } +} const TOKEN = process.env.SHOPIFY_MONITORING_TOKEN if (!dryRun && !TOKEN) { fail('SHOPIFY_MONITORING_TOKEN is not set. See header of bin/update-observe.js for how to obtain one.') @@ -80,20 +104,16 @@ const ERROR_PROJECT_MUTATION = ` ` const main = async () => { - console.log(`Updating Observe resources for ${config.service} → cli_version=${version}${dryRun ? ' (dry run)' : ''}`) - - const tasks = [ - ...config.slos.map((slo) => () => updateSlo(slo)), - ...config.alertRules.map((rule) => () => updateAlertRule(rule)), - ...config.errorProjects.map((project) => () => updateErrorProject(project)), - ] + const scope = args.resource ? `resource=${args.resource}` : `${selectedResources.length} resources` + console.log(`Updating Observe ${scope} for ${config.service} → cli_version=${version}${dryRun ? ' (dry run)' : ''}`) - const results = await Promise.all(tasks.map((run) => run())) + const handlers = {slo: updateSlo, alert: updateAlertRule, errorProject: updateErrorProject} + const results = await Promise.all(selectedResources.map(({type, value}) => handlers[type](value))) const failed = results.filter((ok) => !ok).length if (failed > 0) { fail(`${failed} of ${results.length} updates failed.`) } - console.log(`✓ ${results.length} resources updated.`) + console.log(`✓ ${results.length} resource${results.length === 1 ? '' : 's'} updated.`) } const updateSlo = async (slo) => { @@ -173,10 +193,6 @@ const runMutation = async (kind, id, query, variables, getError) => { } } -function defaultVersion() { - return JSON.parse(readFileSync(join(repoRoot, 'packages/cli-kit/package.json'), 'utf-8')).version -} - function fail(message) { console.error(`Error: ${message}`) process.exit(1) From 1ff2872465291977e53e0c015d11280f5a51f17a Mon Sep 17 00:00:00 2001 From: Alfonso Noriega Date: Mon, 27 Apr 2026 17:33:29 +0200 Subject: [PATCH 3/5] Switch to fetch-then-patch to preserve non-version fields on upsert --- bin/observe-cli-resources.json | 88 ++----- bin/update-observe.js | 412 ++++++++++++++++++++++++--------- 2 files changed, 321 insertions(+), 179 deletions(-) diff --git a/bin/observe-cli-resources.json b/bin/observe-cli-resources.json index 02d077cb7f4..29d96655a43 100644 --- a/bin/observe-cli-resources.json +++ b/bin/observe-cli-resources.json @@ -1,96 +1,46 @@ { - "$schema": "Templates for Observe resources that pin the latest CLI version. Use ${version} as placeholder. Consumed by bin/update-observe.js.", + "$schema": "Resources to keep in sync with the latest CLI version. Consumed by bin/update-observe.js, which fetches the live config for each resource and only patches the version-relevant fields. Use ${version} as placeholder.", "endpoint": "https://shopify-monitoring.shopifycloud.com/query", "service": "cli", - "vaultTeamId": "2238", - "slos": [ + "resources": [ { "key": "slo-correctness-app-deploy", + "kind": "slo", "id": "6cb7fa5f-6f65-417b-aca1-3986c5976068", - "name": "[CLI - Dev Platform] Correctness - App Deploy (${version})", - "kind": "raw", - "expression": "1 - (sum(rate(cli_commands_total{job=~\"@shopify/(app|store):.*\", exit=\"unexpected_error\", cli_version=\"${version}\"} [15m])) / sum(rate(cli_commands_total{job=~\"@shopify/(app|store):.*\", cli_version=\"${version}\"} [15m]))) * on() (sum(rate(cli_commands_total{job=~\"@shopify/(app|store):.*\", cli_version=\"${version}\"} [15m])) > 0.1)" + "nameTemplate": "[CLI - Dev Platform] Correctness - App Deploy (${version})", + "rawExpressionTemplate": "1 - (sum(rate(cli_commands_total{job=~\"@shopify/(app|store):.*\", exit=\"unexpected_error\", cli_version=\"${version}\"} [15m])) / sum(rate(cli_commands_total{job=~\"@shopify/(app|store):.*\", cli_version=\"${version}\"} [15m]))) * on() (sum(rate(cli_commands_total{job=~\"@shopify/(app|store):.*\", cli_version=\"${version}\"} [15m])) > 0.1)" }, { "key": "slo-correctness", + "kind": "slo", "id": "d0131f4a-b9de-4704-9462-9c6ae3de080a", - "name": "[CLI - Dev Platform] Correctness (Version ${version})", - "kind": "raw", - "expression": "1 - (sum(rate(cli_commands_total{job=~\"@shopify/(app|store):.*\", exit=\"unexpected_error\", cli_version=\"${version}\"} [15m])) / sum(rate(cli_commands_total{job=~\"@shopify/(app|store):.*\", cli_version=\"${version}\"} [15m]))) * on() (sum(rate(cli_commands_total{job=~\"@shopify/(app|store):.*\", cli_version=\"${version}\"}[15m])) > 0.1)" + "nameTemplate": "[CLI - Dev Platform] Correctness (Version ${version})", + "rawExpressionTemplate": "1 - (sum(rate(cli_commands_total{job=~\"@shopify/(app|store):.*\", exit=\"unexpected_error\", cli_version=\"${version}\"} [15m])) / sum(rate(cli_commands_total{job=~\"@shopify/(app|store):.*\", cli_version=\"${version}\"} [15m]))) * on() (sum(rate(cli_commands_total{job=~\"@shopify/(app|store):.*\", cli_version=\"${version}\"}[15m])) > 0.1)" }, { "key": "slo-p50-latency", + "kind": "slo", "id": "c1a167e1-9f23-4788-bb4e-7240e556d9b5", - "name": "[CLI - Dev Platform] p50 Latency (Version ${version})", - "kind": "histogram", - "metricName": "cli_commands_duration_ms", - "metricType": "NATIVE_HISTOGRAM", - "minThreshold": "0", - "maxThreshold": "10000", - "filters": [ - {"key": "job", "op": "!=", "value": "@shopify/app::app dev"}, - {"key": "job", "op": "=~", "value": "@shopify/(app|store):.*"}, - {"key": "job", "op": "!=", "value": "@shopify/cli::app dev"}, - {"key": "cli_version", "op": "=", "value": "${version}"} - ] + "nameTemplate": "[CLI - Dev Platform] p50 Latency (Version ${version})" }, { "key": "slo-p75-latency", + "kind": "slo", "id": "1415409f-2750-4bdc-9333-7696f6cf1204", - "name": "[CLI - Dev Platform] p75 Latency (Version ${version})", - "kind": "histogram", - "metricName": "cli_commands_duration_ms", - "metricType": "NATIVE_HISTOGRAM", - "minThreshold": "0", - "maxThreshold": "10000", - "filters": [ - {"key": "job", "op": "!=", "value": "@shopify/app::app dev"}, - {"key": "job", "op": "=~", "value": "@shopify/(app|store):.*"}, - {"key": "job", "op": "!=", "value": "@shopify/cli::app dev"}, - {"key": "cli_version", "op": "=", "value": "${version}"} - ] - } - ], - "alertRules": [ + "nameTemplate": "[CLI - Dev Platform] p75 Latency (Version ${version})" + }, { "key": "alert-spike-errors", + "kind": "alert", "id": "2e527671-b84d-48af-9a52-2100cf295627", - "name": "[CLI - Dev Platform] Spike in Errors (v${version})", - "customErrorFilters": { - "filters": null, - "filter_groups": [ - { - "filters": [ - {"column": "resource.service.name", "op": "=", "value": "cli"}, - {"column": "handled", "op": "=", "value": "false"}, - {"column": "app.version", "op": "=", "value": "${version}"} - ], - "filter_groups": null, - "conjunction": "AND" - } - ], - "conjunction": "AND" - } - } - ], - "errorProjects": [ + "nameTemplate": "[CLI - Dev Platform] Spike in Errors (v${version})", + "versionFilterColumn": "app.version" + }, { "key": "error-project-cli", + "kind": "errorProject", "id": "8fdca84d-03fa-4eb2-9131-bd019e87c3b2", - "errorFilters": { - "filters": null, - "filter_groups": [ - { - "filters": [ - {"column": "handled", "op": "!=", "value": "true"}, - {"column": "app.version", "op": "=", "value": "${version}"} - ], - "filter_groups": null, - "conjunction": "AND" - } - ], - "conjunction": "AND" - } + "versionFilterColumn": "app.version" } ] } diff --git a/bin/update-observe.js b/bin/update-observe.js index 4611934072f..2e4b2a5c728 100755 --- a/bin/update-observe.js +++ b/bin/update-observe.js @@ -5,20 +5,24 @@ * runbook (https://github.com/Shopify/develop-app-inner-loop/issues/2694) that * require clicking through observe.shopify.io to update version filters by hand. * + * For each resource the script: + * 1. Fetches the live config from the Monitoring API. + * 2. Patches only the version-touching fields (name, raw SLI expression, + * cli_version filter, app.version filter). + * 3. Re-upserts the full resource so non-version fields (objective, alertType, + * notificationPolicyId, vaultTeamId, etc.) are preserved verbatim. + * * Usage: - * pnpm update-observe -- --version=3.94.2 # update all resources - * pnpm update-observe -- --version=3.94.2 --resource=slo-p50-latency # update one resource - * pnpm update-observe -- --version=3.94.2 --dry-run # print payloads without sending + * pnpm update-observe -- --version=3.94.2 # update all resources + * pnpm update-observe -- --version=3.94.2 --resource=slo-p50-latency # update one resource + * pnpm update-observe -- --version=3.94.2 --dry-run # print payloads without sending * * --version is required and must be semver X.Y.Z. * --resource selects a single resource by its `key` from - * bin/observe-cli-resources.json (e.g. slo-correctness-app-deploy, - * slo-correctness, slo-p50-latency, slo-p75-latency, alert-spike-errors, - * error-project-cli). Omit to update all of them. + * bin/observe-cli-resources.json. Omit to update all of them. * * Auth: requires a Shopify Monitoring API token in $SHOPIFY_MONITORING_TOKEN. - * Get one at https://observe.shopify.io/profile (under "API Tokens") or via - * `dev observe-token` if that command is available. + * Get one at https://observe.shopify.io/profile (under "API Tokens"). * * Templates live in bin/observe-cli-resources.json. To add or remove a managed * resource, edit that file — no script changes required. @@ -41,158 +45,346 @@ const {values: args} = parseArgs({ const dryRun = args['dry-run'] const version = args.version -if (!version) { - fail('--version is required (e.g. --version=3.94.2)') -} -if (!/^\d+\.\d+\.\d+$/.test(version)) { - fail(`Version must be semver X.Y.Z (got: ${version})`) -} +if (!version) fail('--version is required (e.g. --version=3.94.2)') +if (!/^\d+\.\d+\.\d+$/.test(version)) fail(`Version must be semver X.Y.Z (got: ${version})`) const config = JSON.parse(readFileSync(join(__dirname, 'observe-cli-resources.json'), 'utf-8')) - -const allResources = [ - ...config.slos.map((slo) => ({type: 'slo', value: slo})), - ...config.alertRules.map((rule) => ({type: 'alert', value: rule})), - ...config.errorProjects.map((project) => ({type: 'errorProject', value: project})), -] - -let selectedResources = allResources -if (args.resource) { - selectedResources = allResources.filter((r) => r.value.key === args.resource) - if (selectedResources.length === 0) { - const keys = allResources.map((r) => r.value.key).join(', ') - fail(`No resource with key "${args.resource}". Valid keys: ${keys}`) - } -} const TOKEN = process.env.SHOPIFY_MONITORING_TOKEN if (!dryRun && !TOKEN) { fail('SHOPIFY_MONITORING_TOKEN is not set. See header of bin/update-observe.js for how to obtain one.') } -const interpolate = (value) => { - if (typeof value === 'string') return value.replaceAll('${version}', version) - if (Array.isArray(value)) return value.map(interpolate) - if (value && typeof value === 'object') { - return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, interpolate(v)])) +let selected = config.resources +if (args.resource) { + selected = config.resources.filter((r) => r.key === args.resource) + if (selected.length === 0) { + fail(`No resource with key "${args.resource}". Valid keys: ${config.resources.map((r) => r.key).join(', ')}`) } - return value } +const interpolate = (template) => template.replaceAll('${version}', version) + +// -- Resource handlers ------------------------------------------------------ + +const SLO_QUERY = ` + query($id: ID!) { + sloDefinition(id: $id) { + id name objective summary message description alertType + useRecordingRules v2Enabled v2Summary v2Message escalatorEnabled + threshold { critical warning for aggregation lookback } + urls { href displayText } + labels { name value } + service { name } + vaultTeam { id } + notificationPolicy { id } + sli { + __typename + ... on RawSLI { expression weightExpression } + ... on HistogramSLI { + metric { metricName metricType filters { key value op } } + minThreshold maxThreshold + } + } + } + } +` + const SLO_MUTATION = ` mutation Upsert($input: UpsertSLODefinitionInput!) { - upsertSLODefinition(input: $input) { - sloDefinition { id name } - error + upsertSLODefinition(input: $input) { sloDefinition { id name } error } + } +` + +const RULE_QUERY = ` + query($id: ID!) { + ruleConfig(id: $id) { + id name expression for sustainType interval summary message + dataSourceType disabled noData offsetOverrideSeconds + v2Enabled v2Summary v2Message escalatorEnabled + errorAlertFlavor customErrorFilters includedVersionCount + threshold { critical warning operator type } + urls { href displayText } + labels { name value } + vaultTeam { id } + notificationPolicy { id } + servicesDbService { name } + errorAlertLabelFilters { label operator value } } } ` const RULE_MUTATION = ` mutation Upsert($input: UpsertRuleConfigsInput!) { - upsertRuleConfigs(input: $input) { - validationErrors { field message } + upsertRuleConfigs(input: $input) { validationErrors { fieldErrors { path message } } } + } +` + +const ERROR_PROJECT_QUERY = ` + query($id: String!) { + errorProject(id: $id) { + id name zone serviceName platformType priorityCountType + errorFilters includedVersionCount useLlm enableSegfaultIssues + vaultTeam { id } + severityThresholds { severity { id } value } } } ` const ERROR_PROJECT_MUTATION = ` mutation Upsert($input: ErrorProjectInput!) { - upsertErrorProject(input: $input) { - validationErrors { field message } - } + upsertErrorProject(input: $input) { validationErrors { fieldErrors { path message } } } } ` -const main = async () => { - const scope = args.resource ? `resource=${args.resource}` : `${selectedResources.length} resources` - console.log(`Updating Observe ${scope} for ${config.service} → cli_version=${version}${dryRun ? ' (dry run)' : ''}`) +const stripUndefined = (obj) => + Object.fromEntries(Object.entries(obj).filter(([, v]) => v !== undefined && v !== null)) - const handlers = {slo: updateSlo, alert: updateAlertRule, errorProject: updateErrorProject} - const results = await Promise.all(selectedResources.map(({type, value}) => handlers[type](value))) - const failed = results.filter((ok) => !ok).length - if (failed > 0) { - fail(`${failed} of ${results.length} updates failed.`) +const buildSloInput = (live, resource) => { + if (!live.sli || (live.sli.__typename !== 'RawSLI' && live.sli.__typename !== 'HistogramSLI')) { + throw new Error(`Unsupported SLI type: ${live.sli?.__typename}`) } - console.log(`✓ ${results.length} resource${results.length === 1 ? '' : 's'} updated.`) + + const input = stripUndefined({ + id: live.id, + name: interpolate(resource.nameTemplate), + objective: live.objective, + vaultTeamId: live.vaultTeam?.id, + serviceId: live.service?.name, + summary: live.summary || undefined, + message: live.message || undefined, + description: live.description || undefined, + alertType: live.alertType, + useRecordingRules: live.useRecordingRules, + v2Enabled: live.v2Enabled, + v2Summary: live.v2Summary || undefined, + v2Message: live.v2Message || undefined, + escalatorEnabled: live.escalatorEnabled, + notificationPolicyId: live.notificationPolicy?.id, + threshold: live.threshold + ? stripUndefined({ + critical: live.threshold.critical, + warning: live.threshold.warning, + for: live.threshold.for, + aggregation: live.threshold.aggregation, + lookback: live.threshold.lookback, + }) + : undefined, + urls: live.urls?.map((u) => ({href: u.href, displayText: u.displayText})), + labels: live.labels?.map((l) => ({name: l.name, value: l.value})), + }) + + if (live.sli.__typename === 'RawSLI') { + if (!resource.rawExpressionTemplate) { + throw new Error(`Resource ${resource.key} is a RawSLI but has no rawExpressionTemplate`) + } + input.rawSLIInput = { + expression: interpolate(resource.rawExpressionTemplate), + weightExpression: live.sli.weightExpression || undefined, + } + } else { + const filters = live.sli.metric.filters.map((f) => ({ + key: f.key, + op: f.op, + value: f.key === 'cli_version' && f.op === '=' ? version : f.value, + })) + input.histogramSLIInput = { + metric: { + metricName: live.sli.metric.metricName, + metricType: live.sli.metric.metricType, + filters, + }, + minThreshold: live.sli.minThreshold, + maxThreshold: live.sli.maxThreshold, + } + } + + return input } -const updateSlo = async (slo) => { - const base = {id: slo.id, name: interpolate(slo.name)} - const sloSpec = - slo.kind === 'raw' - ? {...base, rawSLIInput: {expression: interpolate(slo.expression), weightExpression: 'vector(1)'}} - : { - ...base, - histogramSLIInput: { - metric: { - metricName: slo.metricName, - metricType: slo.metricType, - filters: interpolate(slo.filters), - }, - minThreshold: slo.minThreshold, - maxThreshold: slo.maxThreshold, - }, - } - return runMutation('SLO', slo.id, SLO_MUTATION, {input: {slo: sloSpec}}, (data) => data.upsertSLODefinition?.error) +const patchVersionInFilterJson = (jsonString, column) => { + const parsed = JSON.parse(jsonString) + const patchGroup = (group) => { + if (group.filters) { + for (const f of group.filters) { + if (f.column === column) f.value = version + } + } + if (group.filter_groups) group.filter_groups.forEach(patchGroup) + } + patchGroup(parsed) + return JSON.stringify(parsed) } -const updateAlertRule = async (rule) => { - const ruleConfigInput = { - id: rule.id, - name: interpolate(rule.name), - vaultTeamId: config.vaultTeamId, - customErrorFilters: JSON.stringify(interpolate(rule.customErrorFilters)), +const buildRuleInput = (live, resource) => { + if (!live.customErrorFilters) { + throw new Error(`Alert rule ${resource.key} has no customErrorFilters to patch`) } - return runMutation( - 'AlertRule', - rule.id, - RULE_MUTATION, - {input: {ruleConfigInputs: [ruleConfigInput]}}, - (data) => data.upsertRuleConfigs?.validationErrors?.length ? JSON.stringify(data.upsertRuleConfigs.validationErrors) : null, - ) + return stripUndefined({ + id: live.id, + name: interpolate(resource.nameTemplate), + vaultTeamId: live.vaultTeam?.id, + expression: live.expression, + dataSourceType: live.dataSourceType, + for: live.for, + sustainType: live.sustainType || undefined, + interval: live.interval, + summary: live.summary || undefined, + message: live.message || undefined, + disabled: live.disabled, + noData: live.noData, + offsetOverrideSeconds: live.offsetOverrideSeconds, + v2Enabled: live.v2Enabled, + v2Summary: live.v2Summary || undefined, + v2Message: live.v2Message || undefined, + escalatorEnabled: live.escalatorEnabled, + notificationPolicyId: live.notificationPolicy?.id, + servicesDbServiceName: live.servicesDbService?.name, + errorAlertFlavor: live.errorAlertFlavor, + customErrorFilters: patchVersionInFilterJson(live.customErrorFilters, resource.versionFilterColumn), + includedVersionCount: live.includedVersionCount, + threshold: live.threshold + ? stripUndefined({ + critical: live.threshold.critical, + warning: live.threshold.warning, + operator: live.threshold.operator, + }) + : undefined, + urls: live.urls?.map((u) => ({href: u.href, displayText: u.displayText})), + labels: live.labels?.map((l) => ({name: l.name, value: l.value})), + errorAlertLabelFilters: live.errorAlertLabelFilters?.map((f) => ({ + label: f.label, + operator: f.operator, + value: f.value, + })), + }) } -const updateErrorProject = async (project) => { - const errorProjectInput = { - id: project.id, - name: config.service, - errorFilters: JSON.stringify(interpolate(project.errorFilters)), +const buildErrorProjectInput = (live, resource) => { + if (!live.errorFilters) { + throw new Error(`Error project ${resource.key} has no errorFilters to patch`) } - return runMutation( - 'ErrorProject', - project.id, - ERROR_PROJECT_MUTATION, - {input: errorProjectInput}, - (data) => data.upsertErrorProject?.validationErrors?.length ? JSON.stringify(data.upsertErrorProject.validationErrors) : null, - ) + return stripUndefined({ + id: live.id, + name: live.name, + zone: live.zone || undefined, + serviceName: live.serviceName || undefined, + vaultTeamID: live.vaultTeam?.id, + platformType: live.platformType || undefined, + priorityCountType: live.priorityCountType, + errorFilters: patchVersionInFilterJson(live.errorFilters, resource.versionFilterColumn), + severityThresholds: live.severityThresholds?.map((t) => ({severityId: t.severity.id, value: t.value})), + includedVersionCount: live.includedVersionCount, + useLlm: live.useLlm, + enableSegfaultIssues: live.enableSegfaultIssues, + }) } -const runMutation = async (kind, id, query, variables, getError) => { - if (dryRun) { - console.log(`-- ${kind} ${id} --`) - console.log(JSON.stringify(variables, null, 2)) - return true +const HANDLERS = { + slo: { + query: SLO_QUERY, + pickLive: (data) => data.sloDefinition, + buildInput: buildSloInput, + mutation: SLO_MUTATION, + wrapInput: (input) => ({slo: input}), + extractError: (data) => data.upsertSLODefinition?.error, + label: 'SLO', + }, + alert: { + query: RULE_QUERY, + pickLive: (data) => data.ruleConfig, + buildInput: buildRuleInput, + mutation: RULE_MUTATION, + wrapInput: (input) => ({ruleConfigInputs: [input]}), + extractError: (data) => { + const errs = data.upsertRuleConfigs?.validationErrors ?? [] + return errs.length ? JSON.stringify(errs) : null + }, + label: 'AlertRule', + }, + errorProject: { + query: ERROR_PROJECT_QUERY, + pickLive: (data) => data.errorProject, + buildInput: buildErrorProjectInput, + mutation: ERROR_PROJECT_MUTATION, + wrapInput: (input) => input, + extractError: (data) => { + const errs = data.upsertErrorProject?.validationErrors ?? [] + return errs.length ? JSON.stringify(errs) : null + }, + label: 'ErrorProject', + }, +} + +// -- HTTP ------------------------------------------------------------------- + +const graphql = async (query, variables) => { + const res = await fetch(config.endpoint, { + method: 'POST', + headers: {'Content-Type': 'application/json', Authorization: `Bearer ${TOKEN}`}, + body: JSON.stringify({query, variables}), + }) + const json = await res.json() + if (json.errors) throw new Error(JSON.stringify(json.errors)) + return json.data +} + +// -- Main loop -------------------------------------------------------------- + +const updateOne = async (resource) => { + const handler = HANDLERS[resource.kind] + if (!handler) { + console.error(`✖ ${resource.key}: unknown kind "${resource.kind}"`) + return false } try { - const res = await fetch(config.endpoint, { - method: 'POST', - headers: {'Content-Type': 'application/json', Authorization: `Bearer ${TOKEN}`}, - body: JSON.stringify({query, variables}), - }) - const json = await res.json() - const err = json.errors ? JSON.stringify(json.errors) : getError(json.data ?? {}) + let live + if (dryRun) { + // In dry-run we still query so the printed payload reflects real merging. + // If TOKEN is missing, fall back to a minimal stub. + live = TOKEN ? handler.pickLive(await graphql(handler.query, {id: resource.id})) : null + if (!live) { + console.log(`-- ${handler.label} ${resource.key} (dry-run, no live fetch) --`) + console.log('Note: set SHOPIFY_MONITORING_TOKEN to see the merged payload.') + return true + } + } else { + live = handler.pickLive(await graphql(handler.query, {id: resource.id})) + if (!live) throw new Error(`Resource ${resource.id} not found via API`) + } + + const input = handler.buildInput(live, resource) + const variables = {input: handler.wrapInput(input)} + + if (dryRun) { + console.log(`-- ${handler.label} ${resource.key} --`) + console.log(JSON.stringify(variables, null, 2)) + return true + } + + const data = await graphql(handler.mutation, variables) + const err = handler.extractError(data) if (err) { - console.error(`✖ ${kind} ${id}: ${err}`) + console.error(`✖ ${handler.label} ${resource.key}: ${err}`) return false } - console.log(`✓ ${kind} ${id}`) + console.log(`✓ ${handler.label} ${resource.key}`) return true } catch (error) { - console.error(`✖ ${kind} ${id}: ${error.message}`) + console.error(`✖ ${resource.key}: ${error.message}`) return false } } +const main = async () => { + const scope = args.resource ? `resource=${args.resource}` : `${selected.length} resources` + console.log(`Updating Observe ${scope} for ${config.service} → cli_version=${version}${dryRun ? ' (dry run)' : ''}`) + + const results = await Promise.all(selected.map(updateOne)) + const failed = results.filter((ok) => !ok).length + if (failed > 0) fail(`${failed} of ${results.length} updates failed.`) + console.log(`✓ ${results.length} resource${results.length === 1 ? '' : 's'} updated.`) +} + function fail(message) { console.error(`Error: ${message}`) process.exit(1) From 390de73448180783349dcac4a72744f573eb8b3d Mon Sep 17 00:00:00 2001 From: Alfonso Noriega Date: Mon, 27 Apr 2026 17:45:36 +0200 Subject: [PATCH 4/5] Auth via the observe CLI cookie cache instead of a Bearer token --- bin/update-observe.js | 76 +++++++++++++++++++++++++++++++------------ 1 file changed, 55 insertions(+), 21 deletions(-) diff --git a/bin/update-observe.js b/bin/update-observe.js index 2e4b2a5c728..ae94d427e10 100755 --- a/bin/update-observe.js +++ b/bin/update-observe.js @@ -21,13 +21,20 @@ * --resource selects a single resource by its `key` from * bin/observe-cli-resources.json. Omit to update all of them. * - * Auth: requires a Shopify Monitoring API token in $SHOPIFY_MONITORING_TOKEN. - * Get one at https://observe.shopify.io/profile (under "API Tokens"). + * Auth: reuses the cookie cache from the Shopify `observe` Rust CLI + * (https://github.com/Shopify/world/tree/main/areas/tools/observe-cli). One-time + * setup: + * + * observe auth # opens browser, signs in via Okta, caches cookies to disk + * + * After that this script just reads the cached cookie file. If the cookie has + * expired, re-run `observe auth`. * * Templates live in bin/observe-cli-resources.json. To add or remove a managed * resource, edit that file — no script changes required. */ -import {readFileSync} from 'node:fs' +import {existsSync, readFileSync} from 'node:fs' +import {homedir, platform} from 'node:os' import {dirname, join} from 'node:path' import {fileURLToPath} from 'node:url' import {parseArgs} from 'node:util' @@ -49,10 +56,7 @@ if (!version) fail('--version is required (e.g. --version=3.94.2)') if (!/^\d+\.\d+\.\d+$/.test(version)) fail(`Version must be semver X.Y.Z (got: ${version})`) const config = JSON.parse(readFileSync(join(__dirname, 'observe-cli-resources.json'), 'utf-8')) -const TOKEN = process.env.SHOPIFY_MONITORING_TOKEN -if (!dryRun && !TOKEN) { - fail('SHOPIFY_MONITORING_TOKEN is not set. See header of bin/update-observe.js for how to obtain one.') -} +const COOKIE = dryRun ? loadCookieOrNull() : loadCookie() let selected = config.resources if (args.resource) { @@ -320,14 +324,49 @@ const HANDLERS = { const graphql = async (query, variables) => { const res = await fetch(config.endpoint, { method: 'POST', - headers: {'Content-Type': 'application/json', Authorization: `Bearer ${TOKEN}`}, + headers: {'Content-Type': 'application/json', Cookie: COOKIE}, body: JSON.stringify({query, variables}), }) - const json = await res.json() + const text = await res.text() + if (text.trimStart().startsWith('<')) { + throw new Error('Auth failed (received HTML, likely an SSO redirect). Run `observe auth` to refresh cookies.') + } + const json = JSON.parse(text) if (json.errors) throw new Error(JSON.stringify(json.errors)) return json.data } +// -- Cookie cache (shared with the Rust `observe` CLI) --------------------- + +function observeConfigDir() { + if (platform() === 'darwin') return join(homedir(), 'Library', 'Application Support', 'observe') + if (platform() === 'win32') return join(process.env.APPDATA ?? join(homedir(), 'AppData', 'Roaming'), 'observe') + return join(process.env.XDG_CONFIG_HOME ?? join(homedir(), '.config'), 'observe') +} + +function loadCookieOrNull() { + const dir = observeConfigDir() + for (const name of ['graphql_cookies.txt', 'cookies.txt']) { + const path = join(dir, name) + if (existsSync(path)) { + const value = readFileSync(path, 'utf-8').trim() + if (value) return value + } + } + return null +} + +function loadCookie() { + const cookie = loadCookieOrNull() + if (!cookie) { + fail( + `No GraphQL cookies found in ${observeConfigDir()}.\n` + + 'Run `observe auth` first to sign in via Okta and cache cookies (see https://github.com/Shopify/world/tree/main/areas/tools/observe-cli).', + ) + } + return cookie +} + // -- Main loop -------------------------------------------------------------- const updateOne = async (resource) => { @@ -337,20 +376,15 @@ const updateOne = async (resource) => { return false } try { + // In dry-run we still query (when authed) so the printed payload reflects real merging. let live - if (dryRun) { - // In dry-run we still query so the printed payload reflects real merging. - // If TOKEN is missing, fall back to a minimal stub. - live = TOKEN ? handler.pickLive(await graphql(handler.query, {id: resource.id})) : null - if (!live) { - console.log(`-- ${handler.label} ${resource.key} (dry-run, no live fetch) --`) - console.log('Note: set SHOPIFY_MONITORING_TOKEN to see the merged payload.') - return true - } - } else { - live = handler.pickLive(await graphql(handler.query, {id: resource.id})) - if (!live) throw new Error(`Resource ${resource.id} not found via API`) + if (dryRun && !COOKIE) { + console.log(`-- ${handler.label} ${resource.key} (dry-run, no observe cookie) --`) + console.log('Note: run `observe auth` to see the merged payload.') + return true } + live = handler.pickLive(await graphql(handler.query, {id: resource.id})) + if (!live) throw new Error(`Resource ${resource.id} not found via API`) const input = handler.buildInput(live, resource) const variables = {input: handler.wrapInput(input)} From c41ca49a65d4ab8516a8ebe06a8d2ca127b8d126 Mon Sep 17 00:00:00 2001 From: Alfonso Noriega Date: Mon, 27 Apr 2026 17:57:12 +0200 Subject: [PATCH 5/5] Address review feedback: live-first patching, wildcard preservation, cookie cache fix --- bin/observe-cli-resources.json | 17 +-- bin/update-observe.js | 253 +++++++++++++++++++++++++-------- 2 files changed, 202 insertions(+), 68 deletions(-) diff --git a/bin/observe-cli-resources.json b/bin/observe-cli-resources.json index 29d96655a43..3d7bcbd1ead 100644 --- a/bin/observe-cli-resources.json +++ b/bin/observe-cli-resources.json @@ -1,39 +1,32 @@ { - "$schema": "Resources to keep in sync with the latest CLI version. Consumed by bin/update-observe.js, which fetches the live config for each resource and only patches the version-relevant fields. Use ${version} as placeholder.", + "_comment": "Resources to keep in sync with the latest CLI version. Consumed by bin/update-observe.js, which fetches the live config for each resource and only patches the version-relevant fields (in-place: any X.Y.Z literal found in the name, raw SLI expression, or version filter is rewritten). For alerts and error projects, versionFilterColumn names the filter holding the CLI version.", "endpoint": "https://shopify-monitoring.shopifycloud.com/query", "service": "cli", "resources": [ { "key": "slo-correctness-app-deploy", "kind": "slo", - "id": "6cb7fa5f-6f65-417b-aca1-3986c5976068", - "nameTemplate": "[CLI - Dev Platform] Correctness - App Deploy (${version})", - "rawExpressionTemplate": "1 - (sum(rate(cli_commands_total{job=~\"@shopify/(app|store):.*\", exit=\"unexpected_error\", cli_version=\"${version}\"} [15m])) / sum(rate(cli_commands_total{job=~\"@shopify/(app|store):.*\", cli_version=\"${version}\"} [15m]))) * on() (sum(rate(cli_commands_total{job=~\"@shopify/(app|store):.*\", cli_version=\"${version}\"} [15m])) > 0.1)" + "id": "6cb7fa5f-6f65-417b-aca1-3986c5976068" }, { "key": "slo-correctness", "kind": "slo", - "id": "d0131f4a-b9de-4704-9462-9c6ae3de080a", - "nameTemplate": "[CLI - Dev Platform] Correctness (Version ${version})", - "rawExpressionTemplate": "1 - (sum(rate(cli_commands_total{job=~\"@shopify/(app|store):.*\", exit=\"unexpected_error\", cli_version=\"${version}\"} [15m])) / sum(rate(cli_commands_total{job=~\"@shopify/(app|store):.*\", cli_version=\"${version}\"} [15m]))) * on() (sum(rate(cli_commands_total{job=~\"@shopify/(app|store):.*\", cli_version=\"${version}\"}[15m])) > 0.1)" + "id": "d0131f4a-b9de-4704-9462-9c6ae3de080a" }, { "key": "slo-p50-latency", "kind": "slo", - "id": "c1a167e1-9f23-4788-bb4e-7240e556d9b5", - "nameTemplate": "[CLI - Dev Platform] p50 Latency (Version ${version})" + "id": "c1a167e1-9f23-4788-bb4e-7240e556d9b5" }, { "key": "slo-p75-latency", "kind": "slo", - "id": "1415409f-2750-4bdc-9333-7696f6cf1204", - "nameTemplate": "[CLI - Dev Platform] p75 Latency (Version ${version})" + "id": "1415409f-2750-4bdc-9333-7696f6cf1204" }, { "key": "alert-spike-errors", "kind": "alert", "id": "2e527671-b84d-48af-9a52-2100cf295627", - "nameTemplate": "[CLI - Dev Platform] Spike in Errors (v${version})", "versionFilterColumn": "app.version" }, { diff --git a/bin/update-observe.js b/bin/update-observe.js index ae94d427e10..3452c76f373 100755 --- a/bin/update-observe.js +++ b/bin/update-observe.js @@ -7,8 +7,11 @@ * * For each resource the script: * 1. Fetches the live config from the Monitoring API. - * 2. Patches only the version-touching fields (name, raw SLI expression, - * cli_version filter, app.version filter). + * 2. Patches only the version-touching fields by rewriting any X.Y.Z literal + * it finds in the live name, raw SLI expression, and version filter + * values. The JSON config holds no templates — the live values from + * Observe are the source of truth, so manual edits to surrounding text + * are preserved. * 3. Re-upserts the full resource so non-version fields (objective, alertType, * notificationPolicyId, vaultTeamId, etc.) are preserved verbatim. * @@ -17,23 +20,29 @@ * pnpm update-observe -- --version=3.94.2 --resource=slo-p50-latency # update one resource * pnpm update-observe -- --version=3.94.2 --dry-run # print payloads without sending * - * --version is required and must be semver X.Y.Z. + * --version is required and must be concrete semver X.Y.Z. Wildcards + * (`*`) are intentionally not allowed in the CLI argument — if a live + * resource in Observe pins a partial version like `3.*.*` or `4.90.*`, the + * script preserves those wildcard components and only rewrites the numeric + * ones. So given `--version=4.95.3`, a live `3.*.*` becomes `4.*.*` and a + * live `4.90.*` becomes `4.95.*`. * --resource selects a single resource by its `key` from * bin/observe-cli-resources.json. Omit to update all of them. * - * Auth: reuses the cookie cache from the Shopify `observe` Rust CLI - * (https://github.com/Shopify/world/tree/main/areas/tools/observe-cli). One-time - * setup: + * Auth: the first run opens https://shopify-monitoring.shopifycloud.com/gql in + * your browser, you sign in via Okta, copy the MINERVA_TOKEN cookie value from + * DevTools, and paste it here. The script caches it at + * ~/.cache/shopify-cli/observe-cookie (0600). Subsequent runs reuse the cache. + * If the cookie has expired the script auto-detects the SSO redirect and + * re-prompts. * - * observe auth # opens browser, signs in via Okta, caches cookies to disk - * - * After that this script just reads the cached cookie file. If the cookie has - * expired, re-run `observe auth`. + * For non-interactive use (CI, scripts), set $MINERVA_TOKEN to skip the prompt. * * Templates live in bin/observe-cli-resources.json. To add or remove a managed * resource, edit that file — no script changes required. */ -import {existsSync, readFileSync} from 'node:fs' +import {spawn} from 'node:child_process' +import {chmodSync, existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync} from 'node:fs' import {homedir, platform} from 'node:os' import {dirname, join} from 'node:path' import {fileURLToPath} from 'node:url' @@ -53,10 +62,30 @@ const {values: args} = parseArgs({ const dryRun = args['dry-run'] const version = args.version if (!version) fail('--version is required (e.g. --version=3.94.2)') -if (!/^\d+\.\d+\.\d+$/.test(version)) fail(`Version must be semver X.Y.Z (got: ${version})`) +if (!/^\d+\.\d+\.\d+$/.test(version)) fail(`Version must be concrete semver X.Y.Z (got: ${version})`) + +// Matches a 3-component version literal: digits or `*` separated by dots. +// Used to find/rewrite versions inside live names, expressions, and filter values. +const VERSION_LITERAL_RE = /(? { + const eParts = existing.split('.') + if (eParts.length !== 3) return version // unexpected shape; fall back to the literal target + return eParts.map((e, i) => (e === '*' ? '*' : targetParts[i])).join('.') +} + +// Rewrite every version literal found inside an arbitrary string. +const patchVersionString = (str) => + typeof str === 'string' ? str.replace(VERSION_LITERAL_RE, (match) => applyVersion(match)) : str const config = JSON.parse(readFileSync(join(__dirname, 'observe-cli-resources.json'), 'utf-8')) -const COOKIE = dryRun ? loadCookieOrNull() : loadCookie() +let COOKIE = null // populated lazily; see authenticate() let selected = config.resources if (args.resource) { @@ -66,8 +95,6 @@ if (args.resource) { } } -const interpolate = (template) => template.replaceAll('${version}', version) - // -- Resource handlers ------------------------------------------------------ const SLO_QUERY = ` @@ -150,7 +177,7 @@ const buildSloInput = (live, resource) => { const input = stripUndefined({ id: live.id, - name: interpolate(resource.nameTemplate), + name: patchVersionString(live.name), objective: live.objective, vaultTeamId: live.vaultTeam?.id, serviceId: live.service?.name, @@ -178,18 +205,18 @@ const buildSloInput = (live, resource) => { }) if (live.sli.__typename === 'RawSLI') { - if (!resource.rawExpressionTemplate) { - throw new Error(`Resource ${resource.key} is a RawSLI but has no rawExpressionTemplate`) + if (!live.sli.expression) { + throw new Error(`Resource ${resource.key} is a RawSLI but has no live expression to patch`) } input.rawSLIInput = { - expression: interpolate(resource.rawExpressionTemplate), - weightExpression: live.sli.weightExpression || undefined, + expression: patchVersionString(live.sli.expression), + weightExpression: patchVersionString(live.sli.weightExpression) || undefined, } } else { const filters = live.sli.metric.filters.map((f) => ({ key: f.key, op: f.op, - value: f.key === 'cli_version' && f.op === '=' ? version : f.value, + value: f.key === 'cli_version' && f.op === '=' ? applyVersion(f.value) : f.value, })) input.histogramSLIInput = { metric: { @@ -210,7 +237,7 @@ const patchVersionInFilterJson = (jsonString, column) => { const patchGroup = (group) => { if (group.filters) { for (const f of group.filters) { - if (f.column === column) f.value = version + if (f.column === column && typeof f.value === 'string') f.value = applyVersion(f.value) } } if (group.filter_groups) group.filter_groups.forEach(patchGroup) @@ -225,9 +252,9 @@ const buildRuleInput = (live, resource) => { } return stripUndefined({ id: live.id, - name: interpolate(resource.nameTemplate), + name: patchVersionString(live.name), vaultTeamId: live.vaultTeam?.id, - expression: live.expression, + expression: patchVersionString(live.expression), dataSourceType: live.dataSourceType, for: live.for, sustainType: live.sustainType || undefined, @@ -269,7 +296,7 @@ const buildErrorProjectInput = (live, resource) => { } return stripUndefined({ id: live.id, - name: live.name, + name: patchVersionString(live.name), zone: live.zone || undefined, serviceName: live.serviceName || undefined, vaultTeamID: live.vaultTeam?.id, @@ -321,6 +348,8 @@ const HANDLERS = { // -- HTTP ------------------------------------------------------------------- +class AuthError extends Error {} + const graphql = async (query, variables) => { const res = await fetch(config.endpoint, { method: 'POST', @@ -329,42 +358,155 @@ const graphql = async (query, variables) => { }) const text = await res.text() if (text.trimStart().startsWith('<')) { - throw new Error('Auth failed (received HTML, likely an SSO redirect). Run `observe auth` to refresh cookies.') + throw new AuthError('SSO redirect — cookie missing or expired') } const json = JSON.parse(text) if (json.errors) throw new Error(JSON.stringify(json.errors)) return json.data } -// -- Cookie cache (shared with the Rust `observe` CLI) --------------------- +// -- Cookie management ------------------------------------------------------ + +const MONITORING_GQL_URL = 'https://shopify-monitoring.shopifycloud.com/gql' + +function cookieCachePath() { + const cacheRoot = + platform() === 'darwin' + ? join(homedir(), 'Library', 'Caches') + : platform() === 'win32' + ? process.env.LOCALAPPDATA ?? join(homedir(), 'AppData', 'Local') + : process.env.XDG_CACHE_HOME ?? join(homedir(), '.cache') + return join(cacheRoot, 'shopify-cli', 'observe-cookie') +} + +function readCachedCookie() { + const path = cookieCachePath() + if (!existsSync(path)) return null + const value = readFileSync(path, 'utf-8').trim() + return value || null +} + +function writeCachedCookie(cookie) { + const path = cookieCachePath() + mkdirSync(dirname(path), {recursive: true}) + writeFileSync(path, cookie) + try { + chmodSync(path, 0o600) + } catch { + // chmod is best-effort; on Windows it's a no-op + } +} + +function clearCachedCookie() { + const path = cookieCachePath() + if (existsSync(path)) unlinkSync(path) +} + +function normalizeCookie(input) { + const trimmed = input.trim().replace(/^Cookie:\s*/i, '') + return trimmed.includes('=') ? trimmed : `MINERVA_TOKEN=${trimmed}` +} -function observeConfigDir() { - if (platform() === 'darwin') return join(homedir(), 'Library', 'Application Support', 'observe') - if (platform() === 'win32') return join(process.env.APPDATA ?? join(homedir(), 'AppData', 'Roaming'), 'observe') - return join(process.env.XDG_CONFIG_HOME ?? join(homedir(), '.config'), 'observe') +function openInBrowser(url) { + const [cmd, ...args] = + platform() === 'darwin' ? ['open', url] + : platform() === 'win32' ? ['cmd', '/c', 'start', '', url] + : ['xdg-open', url] + try { + spawn(cmd, args, {stdio: 'ignore', detached: true}).unref() + } catch { + // best-effort; user can open manually + } } -function loadCookieOrNull() { - const dir = observeConfigDir() - for (const name of ['graphql_cookies.txt', 'cookies.txt']) { - const path = join(dir, name) - if (existsSync(path)) { - const value = readFileSync(path, 'utf-8').trim() - if (value) return value +function promptHidden(message) { + return new Promise((resolve, reject) => { + process.stdout.write(message) + const stdin = process.stdin + if (!stdin.isTTY) { + reject(new Error('Cannot prompt for cookie: stdin is not a TTY. Set $MINERVA_TOKEN instead.')) + return } + stdin.setRawMode(true) + stdin.resume() + stdin.setEncoding('utf8') + let buf = '' + const onData = (ch) => { + switch (ch) { + case '\n': case '\r': case '\u0004': + stdin.setRawMode(false) + stdin.pause() + stdin.removeListener('data', onData) + process.stdout.write('\n') + resolve(buf) + break + case '\u0003': + process.stdout.write('\n') + process.exit(130) + break + case '\u007f': case '\b': + if (buf.length > 0) buf = buf.slice(0, -1) + break + default: + buf += ch + } + } + stdin.on('data', onData) + }) +} + +async function promptForCookie() { + if (!process.stdin.isTTY) { + fail('No valid Observe session found and stdin is not a TTY. Set $MINERVA_TOKEN to skip the interactive prompt.') } - return null + console.log('\nNo valid Observe session found. Opening your browser to sign in...') + console.log(` ${MONITORING_GQL_URL}`) + console.log('\nAfter the page loads (Okta SSO if needed):') + console.log(' 1. Open DevTools → Application → Cookies → shopify-monitoring.shopifycloud.com') + console.log(' 2. Find the MINERVA_TOKEN cookie and copy its value') + console.log(' 3. Paste it below (input is hidden)\n') + openInBrowser(MONITORING_GQL_URL) + const value = await promptHidden('MINERVA_TOKEN: ') + if (!value.trim()) fail('Empty cookie value.') + return normalizeCookie(value) } -function loadCookie() { - const cookie = loadCookieOrNull() - if (!cookie) { - fail( - `No GraphQL cookies found in ${observeConfigDir()}.\n` + - 'Run `observe auth` first to sign in via Okta and cache cookies (see https://github.com/Shopify/world/tree/main/areas/tools/observe-cli).', - ) +async function probeAuth() { + await graphql('query { __typename }', {}) +} + +async function authenticate() { + if (process.env.MINERVA_TOKEN) { + COOKIE = normalizeCookie(process.env.MINERVA_TOKEN) + } else { + COOKIE = readCachedCookie() } - return cookie + if (COOKIE) { + try { + await probeAuth() + return + } catch (error) { + if (!(error instanceof AuthError)) throw error + if (process.env.MINERVA_TOKEN) { + fail('$MINERVA_TOKEN is set but rejected by the API (cookie likely expired). Refresh it from your browser.') + } + console.log('Cached cookie is no longer valid — re-authenticating.') + clearCachedCookie() + COOKIE = null + } + } + COOKIE = await promptForCookie() + try { + await probeAuth() // throws AuthError if the user pasted a bad value + } catch (error) { + if (error instanceof AuthError) { + fail('The cookie value you pasted was rejected by the API. Double-check the MINERVA_TOKEN value and try again.') + } + throw error + } + // Only persist after we've confirmed the cookie actually works. + writeCachedCookie(COOKIE) + console.log(`Saved to ${cookieCachePath()} (mode 0600). It will be reused on future runs until it expires.\n`) } // -- Main loop -------------------------------------------------------------- @@ -376,14 +518,7 @@ const updateOne = async (resource) => { return false } try { - // In dry-run we still query (when authed) so the printed payload reflects real merging. - let live - if (dryRun && !COOKIE) { - console.log(`-- ${handler.label} ${resource.key} (dry-run, no observe cookie) --`) - console.log('Note: run `observe auth` to see the merged payload.') - return true - } - live = handler.pickLive(await graphql(handler.query, {id: resource.id})) + const live = handler.pickLive(await graphql(handler.query, {id: resource.id})) if (!live) throw new Error(`Resource ${resource.id} not found via API`) const input = handler.buildInput(live, resource) @@ -410,6 +545,8 @@ const updateOne = async (resource) => { } const main = async () => { + await authenticate() + const scope = args.resource ? `resource=${args.resource}` : `${selected.length} resources` console.log(`Updating Observe ${scope} for ${config.service} → cli_version=${version}${dryRun ? ' (dry run)' : ''}`) @@ -424,4 +561,8 @@ function fail(message) { process.exit(1) } -await main() +try { + await main() +} catch (error) { + fail(error.message) +}