From 1a474380dd852982e5d697708dab9de8bbfdcb7d Mon Sep 17 00:00:00 2001 From: Oleksii Makieiev Date: Wed, 3 Jun 2026 11:16:14 +0300 Subject: [PATCH 1/3] Add runtime scenario fixture profiles --- backend/lined/.beads/interactions.jsonl | 1 + backend/lined/.beads/issues.jsonl | 16 +- backend/lined/docs/experiment-tasks.md | 2 +- backend/lined/docs/load-test-baseline.md | 20 + .../lined/docs/runtime-scenario-summaries.md | 43 +- .../runtime-scenarios/command-runner.mjs | 22 + .../fixture-profiles-v1.json | 75 ++ .../runtime-scenarios/fixture-profiles.mjs | 104 +++ .../runtime-scenarios/k6-adapter.mjs | 77 ++ .../runtime-scenarios/kubernetes-adapter.mjs | 270 +++++++ .../runtime-scenarios/runtime-summary.mjs | 150 ++++ .../runtime-scenarios/scenario-runner-cli.mjs | 27 + .../runtime-scenarios/scenario-runner.mjs | 336 ++++++++ .../scenario-runner.test.mjs | 759 ++++++++++++++++++ 14 files changed, 1882 insertions(+), 20 deletions(-) create mode 100644 backend/lined/load-tests/runtime-scenarios/command-runner.mjs create mode 100644 backend/lined/load-tests/runtime-scenarios/fixture-profiles-v1.json create mode 100644 backend/lined/load-tests/runtime-scenarios/fixture-profiles.mjs create mode 100644 backend/lined/load-tests/runtime-scenarios/k6-adapter.mjs create mode 100644 backend/lined/load-tests/runtime-scenarios/kubernetes-adapter.mjs create mode 100644 backend/lined/load-tests/runtime-scenarios/runtime-summary.mjs create mode 100644 backend/lined/load-tests/runtime-scenarios/scenario-runner-cli.mjs create mode 100644 backend/lined/load-tests/runtime-scenarios/scenario-runner.mjs create mode 100644 backend/lined/load-tests/runtime-scenarios/scenario-runner.test.mjs diff --git a/backend/lined/.beads/interactions.jsonl b/backend/lined/.beads/interactions.jsonl index 0e967a5..7ca776c 100644 --- a/backend/lined/.beads/interactions.jsonl +++ b/backend/lined/.beads/interactions.jsonl @@ -4,3 +4,4 @@ {"id":"int-4377e589","kind":"field_change","created_at":"2026-05-29T08:40:48.238866Z","actor":"Oleksii Makieiev","issue_id":"lined-vbf","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Implemented runtime scenario summary docs and CLI with critic review, verification, and Notion write-back."}} {"id":"int-5e657a3c","kind":"field_change","created_at":"2026-05-30T16:21:49.590801Z","actor":"Oleksii Makieiev","issue_id":"lined-z7y","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Implemented account provisioning policy seam and shared role resolver; focused tests and ./gradlew check pass."}} {"id":"int-7ae524e7","kind":"field_change","created_at":"2026-06-01T14:19:13.525997Z","actor":"Oleksii Makieiev","issue_id":"lined-0zb","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Completed calendar time-window refactor with validated window seam, conflict analyzer, header-bound requester checks, critic review, and ./gradlew check."}} +{"id":"int-be0a8ce1","kind":"field_change","created_at":"2026-06-03T08:15:00.608728Z","actor":"Oleksii Makieiev","issue_id":"lined-6cx","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Implemented versioned scenario fixture profiles, runner integration, docs, tests, and verified runtime scenario render checks."}} diff --git a/backend/lined/.beads/issues.jsonl b/backend/lined/.beads/issues.jsonl index 3bc20ba..9147be9 100644 --- a/backend/lined/.beads/issues.jsonl +++ b/backend/lined/.beads/issues.jsonl @@ -1,15 +1 @@ -{"_type":"issue","id":"lined-51s","title":"Add scenario runner seam","description":"Architecture review bug bug/scenario-runner-seam: runtime evidence generation is split across procedural docs, kubectl ordering, k6 execution, telemetry collection, and hand-built summaries. Add a narrow runner seam that accepts scenario, workload, and output root, coordinates Kubernetes/k6/state adapters, and writes sanitized collector-ready runtime-summary artifacts.","acceptance_criteria":"A CLI under load-tests/runtime-scenarios accepts allowlisted scenario/workload inputs, coordinates kubectl and k6 through testable adapters, writes runtime-summary.json only for successful runs, writes sanitized manifests, includes unit tests, and documents the workflow.","status":"in_progress","priority":2,"issue_type":"bug","assignee":"Oleksii Makieiev","owner":"alexmakeev2703@gmail.com","created_at":"2026-06-01T15:08:34Z","created_by":"Oleksii Makieiev","updated_at":"2026-06-01T15:08:49Z","started_at":"2026-06-01T15:08:49Z","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"lined-0zb","title":"Deepen calendar time-window handling","description":"Architecture review bug: event flows pass raw OffsetDateTime pairs and repeat start/end validation across create, update, list, conflict, and user-conflict paths. Add one validated calendar time-window path and keep overlap/conflict rules local to the event scheduling module without changing public API shape.","status":"closed","priority":2,"issue_type":"bug","assignee":"Oleksii Makieiev","owner":"alexmakeev2703@gmail.com","created_at":"2026-06-01T14:06:34Z","created_by":"Oleksii Makieiev","updated_at":"2026-06-01T14:19:13Z","started_at":"2026-06-01T14:06:46Z","closed_at":"2026-06-01T14:19:13Z","close_reason":"Completed calendar time-window refactor with validated window seam, conflict analyzer, header-bound requester checks, critic review, and ./gradlew check.","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"lined-z7y","title":"Fix account provisioning policy seam","description":"Architecture review bug from docs/experiment-tasks.md: AccountApplicationService exposes provisioning booleans and hard-coded default role/free-plan policy, while role resolution knowledge is duplicated between user and role modules. Implement one clear registration provisioning path, centralize default role/plan policy, and share role resolution.","status":"closed","priority":2,"issue_type":"bug","assignee":"Oleksii Makieiev","owner":"alexmakeev2703@gmail.com","created_at":"2026-05-30T16:16:47Z","created_by":"Oleksii Makieiev","updated_at":"2026-05-30T16:21:50Z","started_at":"2026-05-30T16:16:55Z","closed_at":"2026-05-30T16:21:50Z","close_reason":"Implemented account provisioning policy seam and shared role resolver; focused tests and ./gradlew check pass.","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"lined-9pq","title":"Define SLO constraint thresholds","description":"Define initial local experiment SLO and constraint thresholds for classifying runtime-summary evidence for experiment/slo-constraint-thresholds. Keep backend behavior and scoring unchanged; add docs/config only.","status":"closed","priority":2,"issue_type":"task","assignee":"Oleksii Makieiev","owner":"alexmakeev2703@gmail.com","created_at":"2026-05-30T05:18:03Z","created_by":"Oleksii Makieiev","updated_at":"2026-05-30T05:24:19Z","started_at":"2026-05-30T05:18:18Z","closed_at":"2026-05-30T05:24:19Z","close_reason":"Completed SLO threshold docs and versioned config","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"lined-vbf","title":"Add runtime scenario summary workflow","description":"Implement experiment/runtime-scenario-summaries: add a repeatable local workflow for running kind deployment scenarios under k6 and producing sanitized runtime-summary.json artifacts for collector ingestion.","status":"closed","priority":2,"issue_type":"task","assignee":"Oleksii Makieiev","owner":"alexmakeev2703@gmail.com","created_at":"2026-05-29T08:25:25Z","created_by":"Oleksii Makieiev","updated_at":"2026-05-29T08:40:48Z","started_at":"2026-05-29T08:25:33Z","closed_at":"2026-05-29T08:40:48Z","close_reason":"Implemented runtime scenario summary docs and CLI with critic review, verification, and Notion write-back.","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"lined-8jq","title":"Add Prometheus telemetry pipeline","description":"Implement experiment/prometheus-telemetry-pipeline from docs/experiment-tasks.md by adding local kind Prometheus manifests and documentation for collecting Actuator runtime metrics.","status":"closed","priority":2,"issue_type":"task","assignee":"Oleksii Makieiev","owner":"alexmakeev2703@gmail.com","created_at":"2026-05-28T20:30:07Z","created_by":"Oleksii Makieiev","updated_at":"2026-05-28T20:36:12Z","started_at":"2026-05-28T20:30:16Z","closed_at":"2026-05-28T20:36:12Z","close_reason":"Implemented Prometheus telemetry pipeline manifests and documentation","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"lined-hgb","title":"Add runtime fitness extension","description":"Implement experiment/fitness-runtime-extension by documenting runtime-aware fitness inputs and adding optional summarized runtime metrics ingestion without changing the existing structural fitnessScore.","acceptance_criteria":"Runtime fitness design is documented and linked; collector optionally reads summarized runtime metrics without failing when absent; existing fitnessScore semantics and analyzer compatibility are preserved; Notion research pages are updated and verified.","status":"closed","priority":2,"issue_type":"task","assignee":"Oleksii Makieiev","owner":"alexmakeev2703@gmail.com","created_at":"2026-05-28T06:21:08Z","created_by":"Oleksii Makieiev","updated_at":"2026-05-28T06:31:02Z","started_at":"2026-05-28T06:21:22Z","closed_at":"2026-05-28T06:31:02Z","close_reason":"Implemented runtime fitness design docs, optional collector runtime summary ingestion, critic review loop, verification, and Notion write-back.","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"lined-byj","title":"Add load-test baseline","description":"Implement experiment/load-test-baseline by adding a repeatable k6 workload for users, lobbies, tasks, and events against the local kind backend baseline.","acceptance_criteria":"k6 baseline script creates synthetic data and exercises valid workflows; documentation explains kind port-forward, k6/Docker commands, synthetic data, and Prometheus metrics verification; no Spring business behavior changes.","status":"closed","priority":2,"issue_type":"task","assignee":"Oleksii Makieiev","owner":"alexmakeev2703@gmail.com","created_at":"2026-05-25T19:40:17Z","created_by":"Oleksii Makieiev","updated_at":"2026-05-26T04:33:19Z","started_at":"2026-05-25T19:40:27Z","closed_at":"2026-05-26T04:33:19Z","close_reason":"Completed: added bounded k6 load-test baseline script, documented smoke/baseline workflows, Docker fallback, synthetic data behavior, and runtime metrics verification; Gradle test/check and static script checks passed.","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"lined-cy8","title":"Runtime metrics baseline","description":"Implement experiment/runtime-metrics-baseline from docs/experiment-tasks.md: expose Prometheus-compatible backend metrics and document key runtime signals so /actuator/prometheus can be collected for latency, error, and resource analysis.","status":"closed","priority":2,"issue_type":"task","assignee":"Oleksii Makieiev","owner":"alexmakeev2703@gmail.com","created_at":"2026-05-25T19:11:10Z","created_by":"Oleksii Makieiev","updated_at":"2026-05-25T19:22:23Z","started_at":"2026-05-25T19:11:18Z","closed_at":"2026-05-25T19:22:23Z","close_reason":"Implemented runtime metrics baseline with Prometheus scrape metadata, documentation, and local verification.","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"lined-stn","title":"Add backend Kubernetes health probes","description":"Implement experiment/backend-health-probes by configuring the kind backend Deployment with Spring Boot Actuator readiness and liveness probes, documenting verification steps, and preserving backend business behavior.","acceptance_criteria":"Backend Deployment has readinessProbe and livenessProbe using Actuator endpoints; readiness and liveness health groups are explicit; kind baseline docs explain probe verification; Gradle quality gates and kind rollout verification pass.","status":"closed","priority":2,"issue_type":"task","assignee":"Oleksii Makieiev","owner":"alexmakeev2703@gmail.com","created_at":"2026-05-25T18:08:23Z","created_by":"Oleksii Makieiev","updated_at":"2026-05-25T18:14:45Z","started_at":"2026-05-25T18:08:35Z","closed_at":"2026-05-25T18:14:45Z","close_reason":"Completed: configured Actuator readiness and liveness health groups, added Kubernetes probes to the kind backend Deployment, documented verification steps, and verified Gradle test/check/JaCoCo plus kind rollout and Actuator smoke tests.","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"lined-wq2","title":"Deploy backend baseline to kind","description":"Implement experiment/kind-postgres-backend-baseline by adding local kind Kubernetes manifests for PostgreSQL and the Spring Boot backend, plus documentation for building, loading, applying, port-forwarding, and verifying the Actuator health endpoint.","acceptance_criteria":"PostgreSQL and backend Kubernetes manifests exist under k8s/kind; docs explain the reproducible kind workflow; backend health can be verified at /actuator/health after port-forwarding; Java business behavior is unchanged.","status":"closed","priority":2,"issue_type":"task","assignee":"Oleksii Makieiev","owner":"alexmakeev2703@gmail.com","created_at":"2026-05-23T17:57:50Z","created_by":"Oleksii Makieiev","updated_at":"2026-05-23T18:03:39Z","started_at":"2026-05-23T17:57:59Z","closed_at":"2026-05-23T18:03:39Z","close_reason":"Completed: added k8s/kind manifests, documented the kind baseline workflow, and verified Docker build, kind load, rollout status, and /actuator/health smoke test.","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"lined-62z","title":"Add backend Docker image","description":"Add reproducible Docker image support and documented build/run flow for the Spring Boot backend as the experiment/backend-containerization task.","acceptance_criteria":"Dockerfile builds the backend image reproducibly; container run flow is documented; docs index routes to the containerization guide; backend behavior is unchanged.","status":"open","priority":2,"issue_type":"task","owner":"alexmakeev2703@gmail.com","created_at":"2026-05-23T17:35:06Z","created_by":"Oleksii Makieiev","updated_at":"2026-05-23T17:35:06Z","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"lined-15y","title":"Remove redundant calendar requesterId query parameter","description":"PR #43 review follow-up: conflict endpoints now bind requester identity to X-User-Id, making the requesterId query parameter redundant. Deprecate and remove requesterId from /api/calendar/conflicts and /api/calendar/user-conflict in a compatibility-focused cycle.","status":"open","priority":3,"issue_type":"task","owner":"alexmakeev2703@gmail.com","created_at":"2026-06-01T14:40:15Z","created_by":"Oleksii Makieiev","updated_at":"2026-06-01T14:40:15Z","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"lined-oss","title":"Document Notion knowledge-base workflow","description":"Add backend documentation for using Notion as the durable knowledge base, including write-back checklist, verification after write, fallback policy, and entry template.","status":"closed","priority":3,"issue_type":"task","owner":"alexmakeev2703@gmail.com","created_at":"2026-05-26T20:53:14Z","created_by":"Oleksii Makieiev","updated_at":"2026-05-26T20:53:24Z","closed_at":"2026-05-26T20:53:24Z","close_reason":"Completed: added Notion knowledge-base workflow doc, linked it from backend docs index and AGENTS routing, and verified the documentation diff.","dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"lined-azh","title":"Verify backend Docker image with running daemon","description":"Docker build and container smoke checks for experiment/backend-containerization could not run in this session because Docker daemon socket /Users/oleksii_makieiev/.docker/run/docker.sock was missing. Start Docker Desktop or another daemon, run docker build -t lined-backend:local ., then run the image against PostgreSQL and verify /actuator/health and /swagger-ui.html.","acceptance_criteria":"docker build succeeds; container starts against PostgreSQL; actuator health and Swagger UI are reachable from localhost.","status":"open","priority":3,"issue_type":"task","owner":"alexmakeev2703@gmail.com","created_at":"2026-05-23T17:39:18Z","created_by":"Oleksii Makieiev","updated_at":"2026-05-23T17:39:18Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"lined-62z","title":"Add backend Docker image","description":"Add reproducible Docker image support and documented build/run flow for the Spring Boot backend as the experiment/backend-containerization task.","acceptance_criteria":"Dockerfile builds the backend image reproducibly; container run flow is documented; docs index routes to the containerization guide; backend behavior is unchanged.","status":"closed","priority":2,"issue_type":"task","assignee":"Oleksii Makieiev","owner":"alexmakeev2703@gmail.com","created_at":"2026-05-23T17:35:06Z","created_by":"Oleksii Makieiev","updated_at":"2026-05-23T17:41:10Z","started_at":"2026-05-23T17:35:17Z","closed_at":"2026-05-23T17:41:10Z","close_reason":"Implemented Dockerfile, .dockerignore, and containerization docs; Gradle check and JaCoCo passed. Docker daemon-backed build/run verification is tracked separately in lined-azh because the local Docker socket was unavailable.","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/backend/lined/docs/experiment-tasks.md b/backend/lined/docs/experiment-tasks.md index eac1f9b..22918f2 100644 --- a/backend/lined/docs/experiment-tasks.md +++ b/backend/lined/docs/experiment-tasks.md @@ -15,7 +15,7 @@ scientific experiment work. | `experiment/prometheus-telemetry-pipeline` | Task | Runtime infrastructure | Yes | Prometheus telemetry pipeline | Add a local Prometheus collection path for the kind backend, including scrape configuration and documentation for collecting Actuator runtime metrics. | Runtime metrics from the kind backend can be collected persistently enough for scenario comparison. | | `experiment/runtime-scenario-summaries` | Task | Runtime evidence | Partial / branch-only | Runtime scenario summaries | Add a repeatable workflow for running each deployment scenario under the selected k6 workload and producing sanitized `runtime-summary.json` artifacts. | Each scenario has comparable runtime summaries ready for collector ingestion. | | `experiment/runtime-provenance-manifest` | Task | Runtime evidence | Partial / branch-only | Runtime provenance manifest | Add provenance metadata to runtime fitness outputs, including commit SHA, image tag, workload, telemetry window, configuration hash, constraint version, and fitness vector. | Runtime fitness results are traceable and reproducible for audit and paper evidence. | -| `experiment/scenario-fixture-discipline` | Task | Runtime evidence | No | Scenario fixture discipline | Define explicit workload/context profiles and repeatable input setup for Lined experiment scenario runs. | Deployment/runtime comparisons use stable fixtures instead of manual setup. | +| `experiment/scenario-fixture-discipline` | Task | Runtime evidence | Yes | Scenario fixture discipline | Define explicit workload/context profiles and repeatable input setup for Lined experiment scenario runs. | Deployment/runtime comparisons use stable fixtures instead of manual setup. | | `experiment/slo-constraint-thresholds` | Task | Runtime evidence | Yes | SLO and constraint thresholds | Define initial latency, error-rate, availability, restart, readiness, and resource-efficiency thresholds for classifying valid experiment variants. | Runtime evidence can be evaluated against explicit constraints instead of ad hoc interpretation. | | `experiment/fitness-runtime-extension` | Task | Runtime scoring | Yes | Runtime fitness extension | Extend experiment documentation and/or collector design to include telemetry metrics. | Fixed CI fitness can be compared with runtime-aware adaptive fitness. | | `experiment/runtime-aware-scoring` | Task | Runtime scoring | No | Runtime-aware scoring | Add a versioned runtime fitness score that uses summarized runtime metrics while preserving the existing structural `fitnessScore`. | Runtime-aware scalar fitness can be computed without changing historical CI fitness semantics. | diff --git a/backend/lined/docs/load-test-baseline.md b/backend/lined/docs/load-test-baseline.md index 462b535..f076193 100644 --- a/backend/lined/docs/load-test-baseline.md +++ b/backend/lined/docs/load-test-baseline.md @@ -123,6 +123,26 @@ k6 run \ load-tests/k6/load-test-baseline.js ``` +## Fixture Profiles + +For deployment/runtime comparison runs, prefer the scenario-runner fixture +profiles in `load-tests/runtime-scenarios/fixture-profiles-v1.json` instead of +passing ad hoc k6 environment values. The profiles pin the workload and setup +inputs used by the k6 script, including user count, seeded task and event +counts, VU or stress settings, duration, and think time. + +Run a stable comparison fixture through the scenario runner: + +```bash +node load-tests/runtime-scenarios/scenario-runner-cli.mjs \ + --scenario fixed-medium \ + --fixture-profile comparison-baseline \ + --base-url http://localhost:8080 +``` + +Use direct `k6 run` commands for one-off local checks. Use fixture profiles for +experiment evidence that will be compared across deployment scenarios. + ## Run with Docker If k6 is not installed locally, use the official Grafana k6 image from the diff --git a/backend/lined/docs/runtime-scenario-summaries.md b/backend/lined/docs/runtime-scenario-summaries.md index 4f89f37..5be0152 100644 --- a/backend/lined/docs/runtime-scenario-summaries.md +++ b/backend/lined/docs/runtime-scenario-summaries.md @@ -59,6 +59,23 @@ smoke, baseline, read-heavy, write-heavy, mixed, stress, negative-smoke Use `smoke` for command validation and one of the longer non-negative profiles for scenario comparison. +Fixture profiles are versioned workload/context presets from +`load-tests/runtime-scenarios/fixture-profiles-v1.json`: + +| Fixture profile | Workload | Purpose | +|-----------------|----------|---------| +| `local-smoke` | `smoke` | Minimal local command validation. | +| `comparison-baseline` | `baseline` | Stable default runtime comparison fixture. | +| `comparison-read-heavy` | `read-heavy` | Read-oriented comparison over bounded setup data. | +| `comparison-write-heavy` | `write-heavy` | Write-oriented comparison with per-iteration cleanup. | +| `comparison-mixed` | `mixed` | Mixed reads, updates, and bounded writes. | +| `comparison-stress` | `stress` | Ramping-VU local stress comparison. | + +Profiles make the workload setup explicit by pinning allowed k6 inputs such as +`USER_COUNT`, `SEED_TASK_COUNT`, `SEED_EVENT_COUNT`, `VUS`, `DURATION`, +`STRESS_MAX_VUS`, `STRESS_STAGE_DURATION`, and `THINK_TIME_SECONDS`. +They do not change backend behavior or deployment manifests. + ## Run One Scenario Run the fixed-medium scenario with the smoke workload: @@ -70,15 +87,31 @@ node load-tests/runtime-scenarios/scenario-runner-cli.mjs \ --base-url http://localhost:8080 ``` -Run the same scenario with the default baseline workload: +Run the same scenario with the stable baseline fixture: ```bash node load-tests/runtime-scenarios/scenario-runner-cli.mjs \ --scenario fixed-medium \ - --workload baseline \ + --fixture-profile comparison-baseline \ --base-url http://localhost:8080 ``` +The fixture profile supplies default workload and k6 environment inputs. +Explicit CLI options still win: + +```bash +node load-tests/runtime-scenarios/scenario-runner-cli.mjs \ + --scenario fixed-medium \ + --fixture-profile comparison-baseline \ + --workload read-heavy \ + --k6-env VUS=2 \ + --base-url http://localhost:8080 +``` + +Use overrides only when the run intentionally differs from the named fixture; +the manifest records both the selected profile and the effective workload +environment. + The runner applies the selected scenario, waits for the backend rollout, runs k6 with summary export, collects summarized Kubernetes state, and writes: @@ -119,6 +152,7 @@ side effect. The runner keeps inputs narrow: - scenario and workload names are hardcoded allowlists; +- fixture profile names and profile k6 environment keys are allowlisted; - extra k6 environment variables are allowlisted; - `BASE_URL` must point to `localhost`, `127.0.0.1`, or `[::1]` unless `--allow-remote-base-url` is provided; @@ -179,8 +213,9 @@ the metric is omitted and listed in `missing`. `runtime-summary-manifest.json` is not collector input. It records sanitized provenance such as scenario path, workload variables, git commit, CLI version, -start/end timestamps, whether the scenario was applied, HPA cleanup status, -raw pre/post restart snapshots, the restart delta, and k6 exit code. +fixture profile metadata, start/end timestamps, whether the scenario was +applied, HPA cleanup status, raw pre/post restart snapshots, the restart delta, +and k6 exit code. ## Validate Locally diff --git a/backend/lined/load-tests/runtime-scenarios/command-runner.mjs b/backend/lined/load-tests/runtime-scenarios/command-runner.mjs new file mode 100644 index 0000000..5f8c6d6 --- /dev/null +++ b/backend/lined/load-tests/runtime-scenarios/command-runner.mjs @@ -0,0 +1,22 @@ +import { spawnSync } from 'node:child_process'; + +export const runCommand = ( + command, + args, + { allowFailure = false, capture = false, cwd } = {} +) => { + const result = spawnSync(command, args, { + cwd, + encoding: capture ? 'utf-8' : undefined, + stdio: capture ? ['ignore', 'pipe', 'pipe'] : 'inherit', + }); + + if (result.error && !allowFailure) { + throw result.error; + } + if (!allowFailure && result.status !== 0) { + throw new Error(`${command} ${args.join(' ')} failed with exit code ${result.status}`); + } + + return result; +}; diff --git a/backend/lined/load-tests/runtime-scenarios/fixture-profiles-v1.json b/backend/lined/load-tests/runtime-scenarios/fixture-profiles-v1.json new file mode 100644 index 0000000..928f726 --- /dev/null +++ b/backend/lined/load-tests/runtime-scenarios/fixture-profiles-v1.json @@ -0,0 +1,75 @@ +{ + "schema_version": 1, + "profiles": { + "local-smoke": { + "description": "Minimal fixture for local command validation.", + "workload": "smoke", + "k6_env": { + "USER_COUNT": "2", + "SEED_TASK_COUNT": "2", + "SEED_EVENT_COUNT": "2", + "THINK_TIME_SECONDS": "0" + } + }, + "comparison-baseline": { + "description": "Standard fixture for stable baseline scenario comparison.", + "workload": "baseline", + "k6_env": { + "USER_COUNT": "4", + "SEED_TASK_COUNT": "12", + "SEED_EVENT_COUNT": "8", + "VUS": "5", + "DURATION": "2m", + "THINK_TIME_SECONDS": "1" + } + }, + "comparison-read-heavy": { + "description": "Read-oriented fixture over bounded users, lobby, tasks, and events.", + "workload": "read-heavy", + "k6_env": { + "USER_COUNT": "4", + "SEED_TASK_COUNT": "12", + "SEED_EVENT_COUNT": "8", + "VUS": "5", + "DURATION": "2m", + "THINK_TIME_SECONDS": "1" + } + }, + "comparison-write-heavy": { + "description": "Write-oriented fixture with bounded setup and per-iteration cleanup.", + "workload": "write-heavy", + "k6_env": { + "USER_COUNT": "4", + "SEED_TASK_COUNT": "12", + "SEED_EVENT_COUNT": "8", + "VUS": "5", + "DURATION": "2m", + "THINK_TIME_SECONDS": "1" + } + }, + "comparison-mixed": { + "description": "Mixed read, update, and bounded write fixture for scenario comparison.", + "workload": "mixed", + "k6_env": { + "USER_COUNT": "4", + "SEED_TASK_COUNT": "12", + "SEED_EVENT_COUNT": "8", + "VUS": "5", + "DURATION": "2m", + "THINK_TIME_SECONDS": "1" + } + }, + "comparison-stress": { + "description": "Ramping-VU fixture for local stress comparison.", + "workload": "stress", + "k6_env": { + "USER_COUNT": "4", + "SEED_TASK_COUNT": "12", + "SEED_EVENT_COUNT": "8", + "STRESS_MAX_VUS": "20", + "STRESS_STAGE_DURATION": "30s", + "THINK_TIME_SECONDS": "1" + } + } + } +} diff --git a/backend/lined/load-tests/runtime-scenarios/fixture-profiles.mjs b/backend/lined/load-tests/runtime-scenarios/fixture-profiles.mjs new file mode 100644 index 0000000..013048c --- /dev/null +++ b/backend/lined/load-tests/runtime-scenarios/fixture-profiles.mjs @@ -0,0 +1,104 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +export const FIXTURE_PROFILES_PATH = 'load-tests/runtime-scenarios/fixture-profiles-v1.json'; +export const K6_ENV_KEYS = new Set([ + 'RUN_ID', + 'USER_COUNT', + 'SEED_TASK_COUNT', + 'SEED_EVENT_COUNT', + 'VUS', + 'DURATION', + 'STRESS_MAX_VUS', + 'STRESS_STAGE_DURATION', + 'THINK_TIME_SECONDS', +]); + +const PROFILE_KEYS = new Set(['description', 'workload', 'k6_env']); + +export const fixtureProfileNames = ({ cwd = process.cwd(), file = FIXTURE_PROFILES_PATH } = {}) => { + const artifact = readFixtureArtifact(cwd, file); + return Object.keys(artifact.profiles).sort(); +}; + +export const loadFixtureProfile = ( + name, + { cwd = process.cwd(), file = FIXTURE_PROFILES_PATH } = {} +) => { + const artifact = readFixtureArtifact(cwd, file); + const profile = artifact.profiles[name]; + if (!profile) { + throw new Error(`--fixture-profile must be one of: ${Object.keys(artifact.profiles).sort().join(', ')}`); + } + validateProfile(name, profile); + return { + description: profile.description, + k6Env: { ...profile.k6_env }, + name, + schemaVersion: artifact.schema_version, + workload: profile.workload, + }; +}; + +export const applyFixtureProfileDefaults = ( + options, + { + cwd = process.cwd(), + explicitK6Env = {}, + fixtureFile = FIXTURE_PROFILES_PATH, + workloadExplicit = false, + } = {} +) => { + if (!options.fixtureProfile) { + return options; + } + + const profile = loadFixtureProfile(options.fixtureProfile, { cwd, file: fixtureFile }); + const merged = { + ...options, + fixtureProfileData: profile, + k6Env: { + ...profile.k6Env, + ...explicitK6Env, + }, + }; + if (!workloadExplicit) { + merged.workload = profile.workload; + } + return merged; +}; + +const readFixtureArtifact = (cwd, file) => { + const artifactPath = path.resolve(cwd, file); + const parsed = JSON.parse(fs.readFileSync(artifactPath, 'utf-8')); + if (!isRecord(parsed) || parsed.schema_version !== 1 || !isRecord(parsed.profiles)) { + throw new Error('fixture profile artifact must contain schema_version 1 and profiles'); + } + return parsed; +}; + +const validateProfile = (name, profile) => { + if (!isRecord(profile)) { + throw new Error(`fixture profile ${name} must be an object`); + } + const unknownKeys = Object.keys(profile).filter((key) => !PROFILE_KEYS.has(key)); + if (unknownKeys.length > 0) { + throw new Error(`fixture profile ${name} has unsupported keys: ${unknownKeys.join(', ')}`); + } + if (typeof profile.workload !== 'string' || profile.workload.length === 0) { + throw new Error(`fixture profile ${name} must define workload`); + } + if (!isRecord(profile.k6_env)) { + throw new Error(`fixture profile ${name} must define k6_env`); + } + for (const [key, value] of Object.entries(profile.k6_env)) { + if (!K6_ENV_KEYS.has(key)) { + throw new Error(`fixture profile ${name} has unsupported k6 env ${key}`); + } + if (typeof value !== 'string') { + throw new Error(`fixture profile ${name} k6 env ${key} must be a string`); + } + } +}; + +const isRecord = (value) => typeof value === 'object' && value !== null && !Array.isArray(value); diff --git a/backend/lined/load-tests/runtime-scenarios/k6-adapter.mjs b/backend/lined/load-tests/runtime-scenarios/k6-adapter.mjs new file mode 100644 index 0000000..94da01c --- /dev/null +++ b/backend/lined/load-tests/runtime-scenarios/k6-adapter.mjs @@ -0,0 +1,77 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import { runCommand } from './command-runner.mjs'; +import { parseK6Summary } from './runtime-summary.mjs'; + +export const SUMMARY_TREND_STATS = 'p(95),p(99),avg,min,max'; + +export const assertK6Available = ( + k6Bin, + { commandRunner = runCommand, cwd = process.cwd() } = {} +) => { + const result = commandRunner(k6Bin, ['version'], { + allowFailure: true, + capture: true, + cwd, + }); + + if (result.error?.code === 'ENOENT') { + throw new Error( + `k6 executable not found: ${k6Bin}. ` + + 'Install k6 and make it available in PATH, or pass --k6-bin /absolute/path/to/k6. ' + + 'On macOS with Homebrew: brew install k6.' + ); + } + if (result.error) { + throw result.error; + } + if (result.status !== 0) { + throw new Error(`k6 preflight failed with exit code ${result.status}`); + } +}; + +export const runK6 = ( + options, + { commandRunner = runCommand, cwd = process.cwd() } = {} +) => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'lined-k6-summary-')); + const summaryPath = path.join(tempDir, 'summary.json'); + const args = [ + 'run', + '--summary-export', + summaryPath, + '--summary-trend-stats', + SUMMARY_TREND_STATS, + '-e', + `WORKLOAD=${options.workload}`, + '-e', + `BASE_URL=${options.baseUrl}`, + ]; + + for (const [key, value] of Object.entries(options.k6Env)) { + args.push('-e', `${key}=${value}`); + } + if (options.allowRemoteBaseUrl) { + args.push('-e', 'ALLOW_REMOTE_BASE_URL=true'); + } + args.push(options.script); + + const result = commandRunner(options.k6Bin, args, { + allowFailure: true, + cwd, + }); + + try { + return { + args, + exitCode: result.status, + summary: fs.existsSync(summaryPath) + ? parseK6Summary(fs.readFileSync(summaryPath, 'utf-8')) + : undefined, + }; + } finally { + fs.rmSync(tempDir, { force: true, recursive: true }); + } +}; diff --git a/backend/lined/load-tests/runtime-scenarios/kubernetes-adapter.mjs b/backend/lined/load-tests/runtime-scenarios/kubernetes-adapter.mjs new file mode 100644 index 0000000..41e26fb --- /dev/null +++ b/backend/lined/load-tests/runtime-scenarios/kubernetes-adapter.mjs @@ -0,0 +1,270 @@ +import { runCommand } from './command-runner.mjs'; + +export const NAMESPACE = 'lined'; +export const BACKEND_DEPLOYMENT = 'lined-backend'; +export const BACKEND_LABEL = 'app.kubernetes.io/name=lined-backend'; + +export const cleanupHpaIfNeeded = ( + options, + scenario, + { commandRunner = runCommand, cwd = process.cwd() } = {} +) => { + if (!scenario.fixedReplicas || options.skipHpaCleanup) { + return false; + } + + commandRunner('kubectl', [ + '-n', + NAMESPACE, + 'delete', + 'hpa', + BACKEND_DEPLOYMENT, + '--ignore-not-found', + ], { cwd }); + + return true; +}; + +export const applyScenarioIfNeeded = ( + options, + scenario, + { commandRunner = runCommand, cwd = process.cwd() } = {} +) => { + if (!options.apply) { + return false; + } + + commandRunner('kubectl', ['apply', '-k', scenario.path], { cwd }); + return true; +}; + +export const waitForRollout = ( + { commandRunner = runCommand, cwd = process.cwd() } = {} +) => { + commandRunner('kubectl', [ + '-n', + NAMESPACE, + 'rollout', + 'status', + `deployment/${BACKEND_DEPLOYMENT}`, + ], { cwd }); +}; + +export const collectKubernetesState = ( + scenarioName, + { commandRunner = runCommand, cwd = process.cwd() } = {} +) => { + const deployment = readKubectlJson(commandRunner, [ + '-n', + NAMESPACE, + 'get', + 'deployment', + BACKEND_DEPLOYMENT, + '-o', + 'json', + ], cwd); + const pods = readKubectlJson(commandRunner, [ + '-n', + NAMESPACE, + 'get', + 'pods', + '-l', + BACKEND_LABEL, + '-o', + 'json', + ], cwd); + const hpa = readOptionalKubectlJson(commandRunner, [ + '-n', + NAMESPACE, + 'get', + 'hpa', + BACKEND_DEPLOYMENT, + '-o', + 'json', + ], cwd); + const top = readOptionalKubectlText(commandRunner, [ + '-n', + NAMESPACE, + 'top', + 'pods', + '-l', + BACKEND_LABEL, + '--no-headers', + ], cwd); + + return { + ...summarizeKubernetesState({ + deployment, + hpa, + pods, + topOutput: top, + }), + scenario: scenarioName, + }; +}; + +export const summarizeKubernetesState = ({ deployment, hpa, pods, topOutput }) => { + const backendContainer = findBackendContainer(deployment); + const cpuRequest = parseCpuQuantity(backendContainer?.resources?.requests?.cpu); + const memoryLimit = parseMemoryQuantity(backendContainer?.resources?.limits?.memory); + const backendPods = Array.isArray(pods?.items) ? pods.items : []; + const podCount = Math.max(backendPods.length, 1); + const usage = parseTopPods(topOutput); + + const cpuUsage = sumPodUsage(usage, 'cpuMillicores'); + const memoryUsage = sumPodUsage(usage, 'memoryBytes'); + + return { + cpuUtilization: ratioOrUndefined( + cpuUsage, + cpuRequest === undefined ? undefined : cpuRequest * podCount + ), + hpa: summarizeHpa(hpa), + memoryUtilization: ratioOrUndefined( + memoryUsage, + memoryLimit === undefined ? undefined : memoryLimit * podCount + ), + metricsServerAvailable: usage.length > 0, + replicas: deployment?.status?.replicas, + restartCount: sumRestartCount(backendPods), + }; +}; + +export const parseCpuQuantity = (value) => { + if (value === undefined) { + return undefined; + } + const raw = String(value).trim(); + if (/^\d+(\.\d+)?m$/.test(raw)) { + return Number.parseFloat(raw.slice(0, -1)); + } + if (/^\d+(\.\d+)?$/.test(raw)) { + return Number.parseFloat(raw) * 1000; + } + if (/^\d+(\.\d+)?u$/.test(raw)) { + return Number.parseFloat(raw.slice(0, -1)) / 1000; + } + if (/^\d+(\.\d+)?n$/.test(raw)) { + return Number.parseFloat(raw.slice(0, -1)) / 1000000; + } + throw new Error(`Unsupported CPU quantity: ${value}`); +}; + +export const parseMemoryQuantity = (value) => { + if (value === undefined) { + return undefined; + } + const raw = String(value).trim(); + const match = /^(\d+(?:\.\d+)?)([A-Za-z]+)?$/.exec(raw); + if (!match) { + throw new Error(`Unsupported memory quantity: ${value}`); + } + + const amount = Number.parseFloat(match[1]); + const suffix = match[2] ?? ''; + const multipliers = { + '': 1, + K: 1000, + Ki: 1024, + M: 1000 ** 2, + Mi: 1024 ** 2, + G: 1000 ** 3, + Gi: 1024 ** 3, + T: 1000 ** 4, + Ti: 1024 ** 4, + }; + const multiplier = multipliers[suffix]; + if (multiplier === undefined) { + throw new Error(`Unsupported memory quantity: ${value}`); + } + + return amount * multiplier; +}; + +export const parseTopPods = (content = '') => content + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean) + .map((line) => { + const [name, cpu, memory] = line.split(/\s+/); + return { + cpuMillicores: parseCpuQuantity(cpu), + memoryBytes: parseMemoryQuantity(memory), + name, + }; + }); + +const readKubectlJson = (commandRunner, args, cwd) => JSON.parse(commandRunner( + 'kubectl', + args, + { + capture: true, + cwd, + } +).stdout); + +const readOptionalKubectlJson = (commandRunner, args, cwd) => { + const result = commandRunner('kubectl', args, { + allowFailure: true, + capture: true, + cwd, + }); + const output = String(result.stdout ?? ''); + if (result.error || result.status !== 0 || output.trim() === '') { + return undefined; + } + return JSON.parse(output); +}; + +const readOptionalKubectlText = (commandRunner, args, cwd) => { + const result = commandRunner('kubectl', args, { + allowFailure: true, + capture: true, + cwd, + }); + return result.error || result.status !== 0 ? '' : String(result.stdout ?? ''); +}; + +const findBackendContainer = (deployment) => { + const containers = deployment?.spec?.template?.spec?.containers; + if (!Array.isArray(containers)) { + return undefined; + } + return containers.find((container) => container.name === 'backend') ?? containers[0]; +}; + +const summarizeHpa = (hpa) => { + if (!hpa) { + return undefined; + } + const currentReplicas = hpa.status?.currentReplicas; + const desiredReplicas = hpa.status?.desiredReplicas; + if (typeof currentReplicas !== 'number' || typeof desiredReplicas !== 'number') { + return undefined; + } + return { + currentReplicas, + desiredReplicas, + }; +}; + +const sumRestartCount = (pods) => pods + .flatMap((pod) => Array.isArray(pod.status?.containerStatuses) + ? pod.status.containerStatuses + : []) + .filter((status) => status.name === 'backend' || pods.length === 1) + .reduce((total, status) => total + (status.restartCount ?? 0), 0); + +const sumPodUsage = (usage, field) => { + if (usage.length === 0) { + return undefined; + } + return usage.reduce((total, pod) => total + pod[field], 0); +}; + +const ratioOrUndefined = (numerator, denominator) => { + if (numerator === undefined || denominator === undefined || denominator <= 0) { + return undefined; + } + return Number((numerator / denominator).toFixed(6)); +}; diff --git a/backend/lined/load-tests/runtime-scenarios/runtime-summary.mjs b/backend/lined/load-tests/runtime-scenarios/runtime-summary.mjs new file mode 100644 index 0000000..b9a86fa --- /dev/null +++ b/backend/lined/load-tests/runtime-scenarios/runtime-summary.mjs @@ -0,0 +1,150 @@ +export const CLI_VERSION = 1; +export const SOURCE = 'local-kind'; + +export const parseK6Summary = (content) => { + const parsed = JSON.parse(content); + if (!isRecord(parsed) || !isRecord(parsed.metrics)) { + throw new Error('k6 summary export must contain metrics'); + } + return parsed; +}; + +export const buildRuntimeSummary = ({ k6Summary, kubernetes, scenario, workload }) => { + const missing = new Set(['availability']); + const summary = { + latency_p95_ms: requiredMetric(k6Summary, 'http_req_duration', 'p(95)'), + latency_p99_ms: requiredMetric(k6Summary, 'http_req_duration', 'p(99)'), + error_rate: requiredMetric(k6Summary, 'http_req_failed', 'rate', ['value']), + throughput_rps: requiredMetric(k6Summary, 'http_reqs', 'rate'), + restart_count: kubernetes.restartCount, + }; + + if (kubernetes.cpuUtilization === undefined) { + missing.add('cpu_utilization'); + } else { + summary.cpu_utilization = kubernetes.cpuUtilization; + } + + if (kubernetes.memoryUtilization === undefined) { + missing.add('memory_utilization'); + } else { + summary.memory_utilization = kubernetes.memoryUtilization; + } + + if (kubernetes.hpa) { + summary.hpa_current_replicas = kubernetes.hpa.currentReplicas; + summary.hpa_desired_replicas = kubernetes.hpa.desiredReplicas; + } else if (scenario === 'hpa-cpu') { + missing.add('hpa_current_replicas'); + missing.add('hpa_desired_replicas'); + } + + return { + schema_version: 1, + scenario, + workload, + source: SOURCE, + summary, + missing: Array.from(missing).sort(), + }; +}; + +export const buildWindowKubernetesState = (before, after) => ({ + ...after, + restartCount: restartDelta(before?.restartCount, after?.restartCount), + restartCountAfter: after?.restartCount, + restartCountBefore: before?.restartCount, +}); + +export const buildManifest = ({ + appliedScenario, + finishedAt, + git, + hpaCleanup, + k6, + kubernetes, + options, + scenario, + startedAt, + summaryExported, + summaryWritten, +}) => ({ + schema_version: 1, + artifact: 'runtime-scenario-summary', + cli_version: CLI_VERSION, + source: SOURCE, + scenario: options.scenario, + scenario_path: scenario.path, + workload: options.workload, + fixture_profile: options.fixtureProfileData ? { + name: options.fixtureProfileData.name, + schema_version: options.fixtureProfileData.schemaVersion, + workload: options.fixtureProfileData.workload, + k6_env: options.fixtureProfileData.k6Env, + } : undefined, + workload_env: { + BASE_URL: options.baseUrl, + WORKLOAD: options.workload, + ...options.k6Env, + }, + started_at: startedAt, + finished_at: finishedAt, + git, + k6: { + exit_code: k6.exitCode, + executable: options.k6Bin, + script: options.script, + summary_exported: summaryExported, + summary_trend_stats: k6.summaryTrendStats, + }, + collector_summary_written: summaryWritten, + kubernetes: { + applied_scenario: appliedScenario, + deployment: 'lined-backend', + hpa_cleanup: hpaCleanup, + metrics_server_available: kubernetes.metricsServerAvailable, + namespace: 'lined', + replicas: kubernetes.replicas, + restart_count_after: kubernetes.restartCountAfter, + restart_count_before: kubernetes.restartCountBefore, + restart_count_delta: kubernetes.restartCount, + }, +}); + +const requiredMetric = (k6Summary, metric, valueName, fallbackValueNames = []) => { + const metricValues = k6Summary.metrics?.[metric]; + const value = readK6MetricValue(metricValues, [valueName, ...fallbackValueNames]); + if (typeof value !== 'number' || !Number.isFinite(value)) { + throw new Error(`k6 summary missing numeric ${metric}.${valueName}`); + } + return value; +}; + +const readK6MetricValue = (metricValues, valueNames) => { + if (!metricValues) { + return undefined; + } + for (const valueName of valueNames) { + const nested = metricValues.values?.[valueName]; + if (nested !== undefined) { + return nested; + } + const flat = metricValues[valueName]; + if (flat !== undefined) { + return flat; + } + } + return undefined; +}; + +const restartDelta = (before, after) => { + if (typeof after !== 'number' || !Number.isFinite(after)) { + return undefined; + } + if (typeof before !== 'number' || !Number.isFinite(before)) { + return after; + } + return Math.max(0, after - before); +}; + +const isRecord = (value) => typeof value === 'object' && value !== null && !Array.isArray(value); diff --git a/backend/lined/load-tests/runtime-scenarios/scenario-runner-cli.mjs b/backend/lined/load-tests/runtime-scenarios/scenario-runner-cli.mjs new file mode 100644 index 0000000..ec49d18 --- /dev/null +++ b/backend/lined/load-tests/runtime-scenarios/scenario-runner-cli.mjs @@ -0,0 +1,27 @@ +#!/usr/bin/env node +import { + ScenarioRunError, + parseArgs, + printHelp, + runScenario, +} from './scenario-runner.mjs'; + +try { + const options = parseArgs(process.argv.slice(2)); + if (options.help) { + console.log(printHelp()); + process.exit(0); + } + + const result = runScenario(options); + console.log(`Wrote collector summary: ${result.summaryPath}`); + console.log(`Wrote summary manifest: ${result.manifestPath}`); +} catch (error) { + if (error instanceof ScenarioRunError && error.result?.manifestPath) { + console.error(error.message); + console.error(`Wrote summary manifest: ${error.result.manifestPath}`); + process.exit(1); + } + console.error(error instanceof Error ? error.message : error); + process.exit(1); +} diff --git a/backend/lined/load-tests/runtime-scenarios/scenario-runner.mjs b/backend/lined/load-tests/runtime-scenarios/scenario-runner.mjs new file mode 100644 index 0000000..076f5e8 --- /dev/null +++ b/backend/lined/load-tests/runtime-scenarios/scenario-runner.mjs @@ -0,0 +1,336 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +import { + applyScenarioIfNeeded, + cleanupHpaIfNeeded, + collectKubernetesState, + waitForRollout, +} from './kubernetes-adapter.mjs'; +import { assertK6Available, runK6, SUMMARY_TREND_STATS } from './k6-adapter.mjs'; +import { + buildManifest, + buildRuntimeSummary, + buildWindowKubernetesState, +} from './runtime-summary.mjs'; +import { runCommand } from './command-runner.mjs'; +import { + FIXTURE_PROFILES_PATH, + K6_ENV_KEYS, + applyFixtureProfileDefaults, + fixtureProfileNames, +} from './fixture-profiles.mjs'; + +export const SCENARIOS = Object.freeze({ + 'fixed-small': { + fixedReplicas: true, + path: 'k8s/kind/scenarios/fixed-small', + }, + 'fixed-medium': { + fixedReplicas: true, + path: 'k8s/kind/scenarios/fixed-medium', + }, + 'replicas-2': { + fixedReplicas: true, + path: 'k8s/kind/scenarios/replicas-2', + }, + 'hpa-cpu': { + fixedReplicas: false, + path: 'k8s/kind/scenarios/hpa-cpu', + }, +}); + +export const WORKLOADS = Object.freeze([ + 'smoke', + 'baseline', + 'read-heavy', + 'write-heavy', + 'mixed', + 'stress', + 'negative-smoke', +]); + +const DEFAULT_BASE_URL = 'http://localhost:8080'; +const DEFAULT_K6_BIN = 'k6'; +const DEFAULT_OUTPUT_ROOT = 'load-tests/runtime-scenarios/output'; +const DEFAULT_SCRIPT = 'load-tests/k6/load-test-baseline.js'; +const DEFAULT_WORKLOAD = 'baseline'; + +export class ScenarioRunError extends Error { + constructor(message, result) { + super(message); + this.name = 'ScenarioRunError'; + this.result = result; + } +} + +export const defaultOptions = () => ({ + allowRemoteBaseUrl: false, + apply: true, + baseUrl: DEFAULT_BASE_URL, + fixtureProfile: undefined, + fixtureProfileFile: FIXTURE_PROFILES_PATH, + k6Bin: DEFAULT_K6_BIN, + k6Env: {}, + outputRoot: DEFAULT_OUTPUT_ROOT, + script: DEFAULT_SCRIPT, + skipHpaCleanup: false, + workload: DEFAULT_WORKLOAD, +}); + +export const parseArgs = (argv) => { + const options = defaultOptions(); + const explicitK6Env = {}; + let workloadExplicit = false; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === '--help' || arg === '-h') { + return { ...options, help: true }; + } + if (arg === '--scenario') { + options.scenario = readOptionValue(argv, ++index, arg); + } else if (arg === '--workload') { + options.workload = readOptionValue(argv, ++index, arg); + workloadExplicit = true; + } else if (arg === '--base-url') { + options.baseUrl = readOptionValue(argv, ++index, arg); + } else if (arg === '--fixture-profile') { + options.fixtureProfile = readOptionValue(argv, ++index, arg); + } else if (arg === '--fixture-profile-file') { + options.fixtureProfileFile = readOptionValue(argv, ++index, arg); + } else if (arg === '--output-root') { + options.outputRoot = readOptionValue(argv, ++index, arg); + } else if (arg === '--script') { + options.script = readOptionValue(argv, ++index, arg); + } else if (arg === '--k6-bin') { + options.k6Bin = readOptionValue(argv, ++index, arg); + } else if (arg === '--k6-env') { + addK6Env(explicitK6Env, readOptionValue(argv, ++index, arg)); + } else if (arg === '--skip-apply') { + options.apply = false; + } else if (arg === '--skip-hpa-cleanup') { + options.skipHpaCleanup = true; + } else if (arg === '--allow-remote-base-url') { + options.allowRemoteBaseUrl = true; + } else { + throw new Error(`Unknown option: ${arg}`); + } + } + + options.k6Env = { ...explicitK6Env }; + const resolvedOptions = applyFixtureProfileDefaults(options, { + explicitK6Env, + fixtureFile: options.fixtureProfileFile, + workloadExplicit, + }); + validateOptions(resolvedOptions); + return resolvedOptions; +}; + +export const printHelp = () => `Usage: + node load-tests/runtime-scenarios/scenario-runner-cli.mjs --scenario [options] + +Options: + --scenario ${Object.keys(SCENARIOS).join(', ')} + --workload ${WORKLOADS.join(', ')} (default: ${DEFAULT_WORKLOAD}) + --fixture-profile ${fixtureProfileNames().join(', ')} + --fixture-profile-file Fixture profile artifact (default: ${FIXTURE_PROFILES_PATH}) + --base-url Backend URL (default: ${DEFAULT_BASE_URL}) + --output-root Output root (default: ${DEFAULT_OUTPUT_ROOT}) + --script k6 script path (default: ${DEFAULT_SCRIPT}) + --k6-bin k6 executable (default: ${DEFAULT_K6_BIN}) + --k6-env KEY=value Extra k6 env; repeatable for ${Array.from(K6_ENV_KEYS).join(', ')} + --skip-apply Do not apply the selected kustomize scenario + --skip-hpa-cleanup Do not delete HPA before fixed-replica scenarios + --allow-remote-base-url Allow non-local BASE_URL and pass ALLOW_REMOTE_BASE_URL=true to k6 +`; + +export const runScenario = ( + options, + { + clock = () => new Date().toISOString(), + commandRunner = runCommand, + cwd = process.cwd(), + kubernetesAdapter = defaultKubernetesAdapter(commandRunner, cwd), + k6Adapter = defaultK6Adapter(commandRunner, cwd), + gitReader = defaultGitReader(commandRunner, cwd), + } = {} +) => { + const runOptions = resolveFixtureOptions(options); + validateOptions(runOptions); + const scenario = SCENARIOS[runOptions.scenario]; + k6Adapter.assertAvailable(runOptions.k6Bin); + + const startedAt = clock(); + const outputDir = outputDirectory(runOptions, startedAt, cwd); + const hpaCleanup = kubernetesAdapter.cleanupHpaIfNeeded(runOptions, scenario); + const appliedScenario = kubernetesAdapter.applyScenarioIfNeeded(runOptions, scenario); + kubernetesAdapter.waitForRollout(); + + const beforeWorkload = kubernetesAdapter.collectState(runOptions.scenario); + const k6Result = k6Adapter.run(runOptions); + const afterWorkload = kubernetesAdapter.collectState(runOptions.scenario); + const kubernetes = buildWindowKubernetesState(beforeWorkload, afterWorkload); + const finishedAt = clock(); + const manifest = buildManifest({ + appliedScenario, + finishedAt, + git: gitReader(), + hpaCleanup, + k6: { + exitCode: k6Result.exitCode, + summaryTrendStats: SUMMARY_TREND_STATS, + }, + kubernetes, + options: runOptions, + scenario, + startedAt, + summaryExported: k6Result.summary !== undefined, + summaryWritten: k6Result.exitCode === 0 && k6Result.summary !== undefined, + }); + + const manifestPath = path.join(outputDir, 'runtime-summary-manifest.json'); + fs.mkdirSync(outputDir, { recursive: true }); + writeJson(manifestPath, manifest); + + if (k6Result.exitCode !== 0) { + throw new ScenarioRunError( + `k6 failed with exit code ${k6Result.exitCode}; ` + + `wrote manifest ${manifestPath} but did not write collector summary`, + { + manifest, + manifestPath, + } + ); + } + + if (k6Result.summary === undefined) { + throw new Error('k6 completed without a summary export; collector summary was not written'); + } + + const summary = buildRuntimeSummary({ + k6Summary: k6Result.summary, + kubernetes, + scenario: runOptions.scenario, + workload: runOptions.workload, + }); + const summaryPath = path.join(outputDir, 'runtime-summary.json'); + writeJson(summaryPath, summary); + + return { + manifest, + manifestPath, + summary, + summaryPath, + }; +}; + +const resolveFixtureOptions = (options) => { + if (!options.fixtureProfile || options.fixtureProfileData) { + return options; + } + return applyFixtureProfileDefaults( + { + ...options, + fixtureProfileFile: options.fixtureProfileFile ?? FIXTURE_PROFILES_PATH, + k6Env: options.k6Env ?? {}, + }, + { + explicitK6Env: options.k6Env ?? {}, + fixtureFile: options.fixtureProfileFile ?? FIXTURE_PROFILES_PATH, + workloadExplicit: options.workload !== undefined && options.workload !== DEFAULT_WORKLOAD, + } + ); +}; + +export const ensureLocalBaseUrl = (baseUrl, allowRemoteBaseUrl) => { + if (allowRemoteBaseUrl) { + return; + } + const local = /^https?:\/\/(localhost|127\.0\.0\.1|\[::1\])(?::|\/|$)/.test(baseUrl); + if (!local) { + throw new Error( + 'BASE_URL must point to localhost, 127.0.0.1, or [::1]. ' + + 'Use --allow-remote-base-url only for an intentional controlled target.' + ); + } +}; + +const defaultKubernetesAdapter = (commandRunner, cwd) => ({ + applyScenarioIfNeeded: (options, scenario) => applyScenarioIfNeeded( + options, + scenario, + { commandRunner, cwd } + ), + cleanupHpaIfNeeded: (options, scenario) => cleanupHpaIfNeeded( + options, + scenario, + { commandRunner, cwd } + ), + collectState: (scenarioName) => collectKubernetesState( + scenarioName, + { commandRunner, cwd } + ), + waitForRollout: () => waitForRollout({ commandRunner, cwd }), +}); + +const defaultK6Adapter = (commandRunner, cwd) => ({ + assertAvailable: (k6Bin) => assertK6Available(k6Bin, { commandRunner, cwd }), + run: (options) => runK6(options, { commandRunner, cwd }), +}); + +const defaultGitReader = (commandRunner, cwd) => () => ({ + branch: readOptionalGit(commandRunner, ['branch', '--show-current'], cwd), + commit: readOptionalGit(commandRunner, ['rev-parse', 'HEAD'], cwd), +}); + +const readOptionValue = (argv, index, option) => { + const value = argv[index]; + if (value === undefined || value.startsWith('--')) { + throw new Error(`${option} requires a value`); + } + return value; +}; + +const addK6Env = (k6Env, assignment) => { + const separator = assignment.indexOf('='); + if (separator < 1) { + throw new Error('--k6-env requires KEY=value'); + } + const key = assignment.slice(0, separator); + const value = assignment.slice(separator + 1); + if (!K6_ENV_KEYS.has(key)) { + throw new Error(`Unsupported k6 env ${key}; allowed: ${Array.from(K6_ENV_KEYS).join(', ')}`); + } + k6Env[key] = value; +}; + +const validateOptions = (options) => { + if (!options.scenario || SCENARIOS[options.scenario] === undefined) { + throw new Error(`--scenario must be one of: ${Object.keys(SCENARIOS).join(', ')}`); + } + if (!WORKLOADS.includes(options.workload)) { + throw new Error(`--workload must be one of: ${WORKLOADS.join(', ')}`); + } + ensureLocalBaseUrl(options.baseUrl, options.allowRemoteBaseUrl); +}; + +const outputDirectory = (options, startedAt, cwd) => { + const safeTimestamp = startedAt.replaceAll(/[:.]/g, '-'); + const outputName = `${options.scenario}-${options.workload}-${safeTimestamp}`; + return path.resolve(cwd, options.outputRoot, outputName); +}; + +const readOptionalGit = (commandRunner, args, cwd) => { + const result = commandRunner('git', args, { + allowFailure: true, + capture: true, + cwd, + }); + return result.status === 0 ? result.stdout.trim() : undefined; +}; + +const writeJson = (file, value) => { + fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`, 'utf-8'); +}; diff --git a/backend/lined/load-tests/runtime-scenarios/scenario-runner.test.mjs b/backend/lined/load-tests/runtime-scenarios/scenario-runner.test.mjs new file mode 100644 index 0000000..d168c77 --- /dev/null +++ b/backend/lined/load-tests/runtime-scenarios/scenario-runner.test.mjs @@ -0,0 +1,759 @@ +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { describe, it } from 'node:test'; + +import { assertK6Available, runK6 } from './k6-adapter.mjs'; +import { + cleanupHpaIfNeeded, + parseCpuQuantity, + parseMemoryQuantity, + parseTopPods, + summarizeKubernetesState, +} from './kubernetes-adapter.mjs'; +import { + ScenarioRunError, + ensureLocalBaseUrl, + parseArgs, + runScenario, +} from './scenario-runner.mjs'; +import { buildManifest, buildRuntimeSummary } from './runtime-summary.mjs'; +import { loadFixtureProfile } from './fixture-profiles.mjs'; + +const nestedK6Summary = { + metrics: { + http_req_duration: { + values: { + 'p(95)': 250.5, + 'p(99)': 550.25, + }, + }, + http_req_failed: { + values: { + rate: 0.002, + }, + }, + http_reqs: { + values: { + rate: 42.1, + }, + }, + }, +}; + +const flatK6Summary = { + metrics: { + http_req_duration: { + 'p(95)': 150.25, + 'p(99)': 275.5, + }, + http_req_failed: { + fails: 0, + passes: 100, + value: 0, + }, + http_reqs: { + count: 100, + rate: 25.5, + }, + }, +}; + +const deployment = { + spec: { + template: { + spec: { + containers: [{ + name: 'backend', + resources: { + limits: { + memory: '1Gi', + }, + requests: { + cpu: '500m', + }, + }, + }], + }, + }, + }, + status: { + replicas: 2, + }, +}; + +const pods = { + items: [{ + status: { + containerStatuses: [{ + name: 'backend', + restartCount: 1, + }], + }, + }, { + status: { + containerStatuses: [{ + name: 'backend', + restartCount: 2, + }], + }, + }], +}; + +describe('parseArgs', () => { + it('accepts valid scenario, workload, and allowlisted k6 env options', () => { + const options = parseArgs([ + '--scenario', + 'fixed-medium', + '--workload', + 'smoke', + '--k6-env', + 'VUS=2', + ]); + + assert.equal(options.scenario, 'fixed-medium'); + assert.equal(options.workload, 'smoke'); + assert.equal(options.k6Env.VUS, '2'); + }); + + it('applies a fixture profile as workload and k6 env defaults', () => { + const options = parseArgs([ + '--scenario', + 'fixed-medium', + '--fixture-profile', + 'comparison-baseline', + ]); + + assert.equal(options.fixtureProfileData.name, 'comparison-baseline'); + assert.equal(options.workload, 'baseline'); + assert.equal(options.k6Env.USER_COUNT, '4'); + assert.equal(options.k6Env.SEED_TASK_COUNT, '12'); + assert.equal(options.k6Env.VUS, '5'); + }); + + it('lets explicit workload and k6 env override fixture defaults', () => { + const options = parseArgs([ + '--scenario', + 'fixed-medium', + '--fixture-profile', + 'comparison-baseline', + '--workload', + 'read-heavy', + '--k6-env', + 'VUS=2', + '--k6-env', + 'THINK_TIME_SECONDS=0', + ]); + + assert.equal(options.workload, 'read-heavy'); + assert.equal(options.k6Env.USER_COUNT, '4'); + assert.equal(options.k6Env.VUS, '2'); + assert.equal(options.k6Env.THINK_TIME_SECONDS, '0'); + }); + + it('rejects unknown scenarios and workloads', () => { + assert.throws( + () => parseArgs(['--scenario', 'unknown']), + /--scenario must be one of/ + ); + assert.throws( + () => parseArgs(['--scenario', 'fixed-medium', '--workload', 'unknown']), + /--workload must be one of/ + ); + }); + + it('rejects unknown fixture profiles', () => { + assert.throws( + () => parseArgs(['--scenario', 'fixed-medium', '--fixture-profile', 'unknown']), + /--fixture-profile must be one of/ + ); + }); + + it('rejects unsupported k6 env keys so secrets are not forwarded', () => { + assert.throws( + () => parseArgs(['--scenario', 'fixed-medium', '--k6-env', 'TOKEN=secret']), + /Unsupported k6 env TOKEN/ + ); + }); + + it('rejects unsupported fixture k6 env keys', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'lined-fixtures-')); + const fixtureFile = path.join(directory, 'fixtures.json'); + fs.writeFileSync(fixtureFile, JSON.stringify({ + schema_version: 1, + profiles: { + unsafe: { + workload: 'baseline', + k6_env: { + TOKEN: 'secret', + }, + }, + }, + }), 'utf-8'); + + try { + assert.throws( + () => loadFixtureProfile('unsafe', { file: fixtureFile }), + /unsupported k6 env TOKEN/ + ); + } finally { + fs.rmSync(directory, { force: true, recursive: true }); + } + }); +}); + +describe('ensureLocalBaseUrl', () => { + it('accepts local targets by default', () => { + assert.doesNotThrow(() => ensureLocalBaseUrl('http://localhost:8080', false)); + assert.doesNotThrow(() => ensureLocalBaseUrl('http://127.0.0.1:8080', false)); + assert.doesNotThrow(() => ensureLocalBaseUrl('http://[::1]:8080', false)); + }); + + it('rejects remote targets unless explicitly allowed', () => { + assert.throws( + () => ensureLocalBaseUrl('http://example.com', false), + /BASE_URL must point to localhost/ + ); + assert.doesNotThrow(() => ensureLocalBaseUrl('http://example.com', true)); + }); +}); + +describe('runK6', () => { + it('reports a clear install hint when k6 is missing', () => { + assert.throws( + () => assertK6Available('missing-k6', { + commandRunner: () => ({ + error: Object.assign(new Error('spawn missing-k6 ENOENT'), { + code: 'ENOENT', + }), + status: null, + }), + }), + /Install k6/ + ); + }); + + it('builds argv arrays instead of shell command strings', () => { + const calls = []; + const commandRunner = (command, args) => { + calls.push({ args, command }); + return { status: 0 }; + }; + + runK6( + { + allowRemoteBaseUrl: true, + baseUrl: 'http://localhost:8080', + k6Bin: 'k6', + k6Env: { + VUS: '2', + }, + script: 'load-tests/k6/load-test-baseline.js', + workload: 'smoke', + }, + { commandRunner } + ); + + assert.equal(calls[0].command, 'k6'); + assert.ok(Array.isArray(calls[0].args)); + assert.ok(calls[0].args.includes('--summary-export')); + assert.ok(calls[0].args.includes('VUS=2')); + assert.ok(calls[0].args.includes('ALLOW_REMOTE_BASE_URL=true')); + }); +}); + +describe('Kubernetes state adapter helpers', () => { + it('deletes stale HPA for fixed scenarios unless skipped', () => { + const calls = []; + const commandRunner = (command, args) => { + calls.push({ args, command }); + return { status: 0 }; + }; + + const cleaned = cleanupHpaIfNeeded( + { skipHpaCleanup: false }, + { fixedReplicas: true }, + { commandRunner } + ); + + assert.equal(cleaned, true); + assert.deepEqual(calls[0], { + command: 'kubectl', + args: [ + '-n', + 'lined', + 'delete', + 'hpa', + 'lined-backend', + '--ignore-not-found', + ], + }); + }); + + it('skips HPA cleanup when requested', () => { + const calls = []; + const cleaned = cleanupHpaIfNeeded( + { skipHpaCleanup: true }, + { fixedReplicas: true }, + { + commandRunner: (command, args) => { + calls.push({ args, command }); + return { status: 0 }; + }, + } + ); + + assert.equal(cleaned, false); + assert.deepEqual(calls, []); + }); + + it('parses Kubernetes resource quantities and pod top output', () => { + assert.equal(parseCpuQuantity('500m'), 500); + assert.equal(parseCpuQuantity('1'), 1000); + assert.equal(parseCpuQuantity('250u'), 0.25); + assert.equal(parseMemoryQuantity('1Gi'), 1024 ** 3); + assert.equal(parseMemoryQuantity('512Mi'), 512 * 1024 ** 2); + assert.deepEqual(parseTopPods('lined-backend-a 250m 512Mi\n'), [{ + cpuMillicores: 250, + memoryBytes: 512 * 1024 ** 2, + name: 'lined-backend-a', + }]); + }); + + it('summarizes Kubernetes utilization and restarts', () => { + const result = summarizeKubernetesState({ + deployment, + pods, + topOutput: 'lined-backend-a 250m 512Mi\nlined-backend-b 250m 512Mi\n', + }); + + assert.equal(result.cpuUtilization, 0.5); + assert.equal(result.memoryUtilization, 0.5); + assert.equal(result.restartCount, 3); + assert.equal(result.metricsServerAvailable, true); + }); + + it('omits utilization when metrics-server data is missing', () => { + const result = summarizeKubernetesState({ + deployment, + pods, + topOutput: '', + }); + + assert.equal(result.cpuUtilization, undefined); + assert.equal(result.memoryUtilization, undefined); + assert.equal(result.metricsServerAvailable, false); + }); +}); + +describe('buildRuntimeSummary', () => { + it('builds a collector-compatible summary and records missing optional metrics', () => { + const summary = buildRuntimeSummary({ + k6Summary: nestedK6Summary, + kubernetes: { + metricsServerAvailable: false, + restartCount: 0, + }, + scenario: 'fixed-medium', + workload: 'smoke', + }); + + assert.deepEqual(summary, { + schema_version: 1, + scenario: 'fixed-medium', + workload: 'smoke', + source: 'local-kind', + summary: { + latency_p95_ms: 250.5, + latency_p99_ms: 550.25, + error_rate: 0.002, + throughput_rps: 42.1, + restart_count: 0, + }, + missing: [ + 'availability', + 'cpu_utilization', + 'memory_utilization', + ], + }); + }); + + it('reads flat k6 v2 summary exports', () => { + const summary = buildRuntimeSummary({ + k6Summary: flatK6Summary, + kubernetes: { + metricsServerAvailable: false, + restartCount: 0, + }, + scenario: 'fixed-medium', + workload: 'smoke', + }); + + assert.equal(summary.summary.latency_p95_ms, 150.25); + assert.equal(summary.summary.latency_p99_ms, 275.5); + assert.equal(summary.summary.error_rate, 0); + assert.equal(summary.summary.throughput_rps, 25.5); + }); + + it('marks missing HPA fields for the HPA scenario when no HPA state is present', () => { + const summary = buildRuntimeSummary({ + k6Summary: nestedK6Summary, + kubernetes: { + metricsServerAvailable: false, + restartCount: 0, + }, + scenario: 'hpa-cpu', + workload: 'baseline', + }); + + assert.ok(summary.missing.includes('hpa_current_replicas')); + assert.ok(summary.missing.includes('hpa_desired_replicas')); + }); + + it('uses measurement-window restart deltas instead of cumulative snapshots', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'lined-runner-')); + + try { + const result = runScenario( + { + allowRemoteBaseUrl: false, + apply: false, + baseUrl: 'http://localhost:8080', + k6Bin: 'k6', + k6Env: {}, + outputRoot: directory, + scenario: 'fixed-medium', + script: 'load-tests/k6/load-test-baseline.js', + skipHpaCleanup: false, + workload: 'smoke', + }, + fakeAdapters({ + k6ExitCode: 0, + k6Summary: nestedK6Summary, + restartCounts: [4, 6], + }) + ); + + assert.equal(result.summary.summary.restart_count, 2); + assert.equal(result.manifest.kubernetes.restart_count_before, 4); + assert.equal(result.manifest.kubernetes.restart_count_after, 6); + assert.equal(result.manifest.kubernetes.restart_count_delta, 2); + } finally { + fs.rmSync(directory, { force: true, recursive: true }); + } + }); + + it('does not emit a negative restart delta when pod counters reset', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'lined-runner-')); + + try { + const result = runScenario( + { + allowRemoteBaseUrl: false, + apply: false, + baseUrl: 'http://localhost:8080', + k6Bin: 'k6', + k6Env: {}, + outputRoot: directory, + scenario: 'fixed-medium', + script: 'load-tests/k6/load-test-baseline.js', + skipHpaCleanup: false, + workload: 'smoke', + }, + fakeAdapters({ + k6ExitCode: 0, + k6Summary: nestedK6Summary, + restartCounts: [4, 1], + }) + ); + + assert.equal(result.summary.summary.restart_count, 0); + assert.equal(result.manifest.kubernetes.restart_count_before, 4); + assert.equal(result.manifest.kubernetes.restart_count_after, 1); + } finally { + fs.rmSync(directory, { force: true, recursive: true }); + } + }); + + it('fails without writing a collector summary when k6 omits summary export', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'lined-runner-')); + + try { + let thrown; + assert.throws( + () => runScenario( + { + allowRemoteBaseUrl: false, + apply: false, + baseUrl: 'http://localhost:8080', + k6Bin: 'k6', + k6Env: {}, + outputRoot: directory, + scenario: 'fixed-medium', + script: 'load-tests/k6/load-test-baseline.js', + skipHpaCleanup: false, + workload: 'smoke', + }, + fakeAdapters({ + k6ExitCode: 0, + k6Summary: undefined, + }) + ), + (error) => { + thrown = error; + return /summary export/.test(error.message); + } + ); + + const runDirs = fs.readdirSync(directory); + const runDir = path.join(directory, runDirs[0]); + const manifest = JSON.parse( + fs.readFileSync(path.join(runDir, 'runtime-summary-manifest.json'), 'utf-8') + ); + + assert.equal(thrown instanceof Error, true); + assert.equal(fs.existsSync(path.join(runDir, 'runtime-summary-manifest.json')), true); + assert.equal(fs.existsSync(path.join(runDir, 'runtime-summary.json')), false); + assert.equal(manifest.collector_summary_written, false); + } finally { + fs.rmSync(directory, { force: true, recursive: true }); + } + }); +}); + +describe('manifest and runScenario', () => { + it('records sanitized provenance in the manifest', () => { + const manifest = buildManifest({ + appliedScenario: true, + finishedAt: '2026-06-01T10:00:10.000Z', + git: { + branch: 'bug/scenario-runner-seam', + commit: 'abc123', + }, + hpaCleanup: true, + k6: { + exitCode: 0, + summaryTrendStats: 'p(95),p(99),avg,min,max', + }, + kubernetes: { + metricsServerAvailable: false, + replicas: 1, + }, + options: { + baseUrl: 'http://localhost:8080', + fixtureProfileData: { + k6Env: { + USER_COUNT: '4', + VUS: '2', + }, + name: 'comparison-baseline', + schemaVersion: 1, + workload: 'baseline', + }, + k6Bin: 'k6', + k6Env: { + VUS: '2', + }, + scenario: 'fixed-medium', + script: 'load-tests/k6/load-test-baseline.js', + workload: 'smoke', + }, + scenario: { + path: 'k8s/kind/scenarios/fixed-medium', + }, + startedAt: '2026-06-01T10:00:00.000Z', + summaryExported: true, + summaryWritten: true, + }); + + assert.equal(manifest.kubernetes.applied_scenario, true); + assert.equal(manifest.kubernetes.hpa_cleanup, true); + assert.equal(manifest.collector_summary_written, true); + assert.deepEqual(manifest.fixture_profile, { + name: 'comparison-baseline', + schema_version: 1, + workload: 'baseline', + k6_env: { + USER_COUNT: '4', + VUS: '2', + }, + }); + assert.equal(manifest.workload_env.VUS, '2'); + assert.equal(manifest.git.branch, 'bug/scenario-runner-seam'); + }); + + it('writes a summary and manifest for successful runs', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'lined-runner-')); + + try { + const result = runScenario( + { + allowRemoteBaseUrl: false, + apply: false, + baseUrl: 'http://localhost:8080', + k6Bin: 'k6', + k6Env: {}, + outputRoot: directory, + scenario: 'fixed-medium', + script: 'load-tests/k6/load-test-baseline.js', + skipHpaCleanup: false, + workload: 'smoke', + }, + fakeAdapters({ + k6ExitCode: 0, + k6Summary: nestedK6Summary, + }) + ); + + assert.equal(fs.existsSync(result.summaryPath), true); + assert.equal(fs.existsSync(result.manifestPath), true); + assert.equal(result.summary.summary.latency_p95_ms, 250.5); + assert.equal(result.summary.fixture_profile, undefined); + assert.equal(result.manifest.collector_summary_written, true); + } finally { + fs.rmSync(directory, { force: true, recursive: true }); + } + }); + + it('applies fixture profiles when runScenario is called directly', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'lined-runner-')); + + try { + const result = runScenario( + { + allowRemoteBaseUrl: false, + apply: false, + baseUrl: 'http://localhost:8080', + fixtureProfile: 'comparison-read-heavy', + k6Bin: 'k6', + k6Env: { + VUS: '2', + }, + outputRoot: directory, + scenario: 'fixed-medium', + script: 'load-tests/k6/load-test-baseline.js', + skipHpaCleanup: false, + workload: 'baseline', + }, + fakeAdapters({ + k6ExitCode: 0, + k6Summary: nestedK6Summary, + }) + ); + + assert.equal(result.summary.workload, 'read-heavy'); + assert.equal(result.manifest.fixture_profile.name, 'comparison-read-heavy'); + assert.equal(result.manifest.workload_env.WORKLOAD, 'read-heavy'); + assert.equal(result.manifest.workload_env.USER_COUNT, '4'); + assert.equal(result.manifest.workload_env.VUS, '2'); + } finally { + fs.rmSync(directory, { force: true, recursive: true }); + } + }); + + it('lets direct runScenario workload overrides win over fixture defaults', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'lined-runner-')); + + try { + const result = runScenario( + { + allowRemoteBaseUrl: false, + apply: false, + baseUrl: 'http://localhost:8080', + fixtureProfile: 'comparison-baseline', + k6Bin: 'k6', + k6Env: {}, + outputRoot: directory, + scenario: 'fixed-medium', + script: 'load-tests/k6/load-test-baseline.js', + skipHpaCleanup: false, + workload: 'read-heavy', + }, + fakeAdapters({ + k6ExitCode: 0, + k6Summary: nestedK6Summary, + }) + ); + + assert.equal(result.summary.workload, 'read-heavy'); + assert.equal(result.manifest.fixture_profile.name, 'comparison-baseline'); + assert.equal(result.manifest.workload_env.WORKLOAD, 'read-heavy'); + assert.equal(result.manifest.workload_env.USER_COUNT, '4'); + } finally { + fs.rmSync(directory, { force: true, recursive: true }); + } + }); + + it('writes only a manifest when k6 fails', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'lined-runner-')); + + try { + assert.throws( + () => runScenario( + { + allowRemoteBaseUrl: false, + apply: false, + baseUrl: 'http://localhost:8080', + k6Bin: 'k6', + k6Env: {}, + outputRoot: directory, + scenario: 'fixed-medium', + script: 'load-tests/k6/load-test-baseline.js', + skipHpaCleanup: false, + workload: 'smoke', + }, + fakeAdapters({ + k6ExitCode: 1, + k6Summary: nestedK6Summary, + }) + ), + ScenarioRunError + ); + + const runDirs = fs.readdirSync(directory); + assert.equal(runDirs.length, 1); + const runDir = path.join(directory, runDirs[0]); + assert.equal(fs.existsSync(path.join(runDir, 'runtime-summary-manifest.json')), true); + assert.equal(fs.existsSync(path.join(runDir, 'runtime-summary.json')), false); + } finally { + fs.rmSync(directory, { force: true, recursive: true }); + } + }); +}); + +const fakeAdapters = ({ k6ExitCode, k6Summary, restartCounts = [0, 0] }) => ({ + clock: fakeClock(), + gitReader: () => ({ + branch: 'bug/scenario-runner-seam', + commit: 'abc123', + }), + k6Adapter: { + assertAvailable: () => {}, + run: () => ({ + exitCode: k6ExitCode, + summary: k6Summary, + }), + }, + kubernetesAdapter: { + applyScenarioIfNeeded: () => false, + cleanupHpaIfNeeded: () => true, + collectState: () => { + const restartCount = restartCounts.shift() ?? restartCounts.at(-1) ?? 0; + return { + metricsServerAvailable: false, + replicas: 1, + restartCount, + }; + }, + waitForRollout: () => {}, + }, +}); + +const fakeClock = () => { + const times = [ + '2026-06-01T10:00:00.000Z', + '2026-06-01T10:00:10.000Z', + ]; + return () => times.shift() ?? '2026-06-01T10:00:10.000Z'; +}; From 0f9dd3e51e414968a304cfc059b0b04bc70973f3 Mon Sep 17 00:00:00 2001 From: Oleksii Makieiev Date: Wed, 3 Jun 2026 11:22:58 +0300 Subject: [PATCH 2/3] Address fixture runner review comments --- .../runtime-scenarios/fixture-profiles.mjs | 40 +- .../runtime-scenarios/scenario-runner.mjs | 98 +++-- .../scenario-runner.test.mjs | 353 +++++++++++------- 3 files changed, 299 insertions(+), 192 deletions(-) diff --git a/backend/lined/load-tests/runtime-scenarios/fixture-profiles.mjs b/backend/lined/load-tests/runtime-scenarios/fixture-profiles.mjs index 013048c..36b4af9 100644 --- a/backend/lined/load-tests/runtime-scenarios/fixture-profiles.mjs +++ b/backend/lined/load-tests/runtime-scenarios/fixture-profiles.mjs @@ -78,26 +78,42 @@ const readFixtureArtifact = (cwd, file) => { }; const validateProfile = (name, profile) => { - if (!isRecord(profile)) { - throw new Error(`fixture profile ${name} must be an object`); + requireRecord(`fixture profile ${name}`, profile); + requireKnownProfileKeys(name, profile); + requireWorkload(name, profile.workload); + requireK6Env(name, profile.k6_env); +}; + +const requireRecord = (label, value) => { + if (!isRecord(value)) { + throw new Error(`${label} must be an object`); } +}; + +const requireKnownProfileKeys = (name, profile) => { const unknownKeys = Object.keys(profile).filter((key) => !PROFILE_KEYS.has(key)); if (unknownKeys.length > 0) { throw new Error(`fixture profile ${name} has unsupported keys: ${unknownKeys.join(', ')}`); } - if (typeof profile.workload !== 'string' || profile.workload.length === 0) { +}; + +const requireWorkload = (name, workload) => { + if (typeof workload !== 'string' || workload.length === 0) { throw new Error(`fixture profile ${name} must define workload`); } - if (!isRecord(profile.k6_env)) { - throw new Error(`fixture profile ${name} must define k6_env`); +}; + +const requireK6Env = (name, k6Env) => { + requireRecord(`fixture profile ${name} k6_env`, k6Env); + Object.entries(k6Env).forEach(([key, value]) => requireK6EnvEntry(name, key, value)); +}; + +const requireK6EnvEntry = (name, key, value) => { + if (!K6_ENV_KEYS.has(key)) { + throw new Error(`fixture profile ${name} has unsupported k6 env ${key}`); } - for (const [key, value] of Object.entries(profile.k6_env)) { - if (!K6_ENV_KEYS.has(key)) { - throw new Error(`fixture profile ${name} has unsupported k6 env ${key}`); - } - if (typeof value !== 'string') { - throw new Error(`fixture profile ${name} k6 env ${key} must be a string`); - } + if (typeof value !== 'string') { + throw new Error(`fixture profile ${name} k6 env ${key} must be a string`); } }; diff --git a/backend/lined/load-tests/runtime-scenarios/scenario-runner.mjs b/backend/lined/load-tests/runtime-scenarios/scenario-runner.mjs index 076f5e8..7ee448b 100644 --- a/backend/lined/load-tests/runtime-scenarios/scenario-runner.mjs +++ b/backend/lined/load-tests/runtime-scenarios/scenario-runner.mjs @@ -55,6 +55,34 @@ const DEFAULT_K6_BIN = 'k6'; const DEFAULT_OUTPUT_ROOT = 'load-tests/runtime-scenarios/output'; const DEFAULT_SCRIPT = 'load-tests/k6/load-test-baseline.js'; const DEFAULT_WORKLOAD = 'baseline'; +const HELP_OPTIONS = new Set(['--help', '-h']); + +const OPTION_HANDLERS = Object.freeze({ + '--scenario': readOptionInto('scenario'), + '--workload': (state, option) => { + state.options.workload = readNextOptionValue(state, option); + state.workloadExplicit = true; + }, + '--base-url': readOptionInto('baseUrl'), + '--fixture-profile': readOptionInto('fixtureProfile'), + '--fixture-profile-file': readOptionInto('fixtureProfileFile'), + '--output-root': readOptionInto('outputRoot'), + '--script': readOptionInto('script'), + '--k6-bin': readOptionInto('k6Bin'), + '--k6-env': (state, option) => addK6Env( + state.explicitK6Env, + readNextOptionValue(state, option) + ), + '--skip-apply': (state) => { + state.options.apply = false; + }, + '--skip-hpa-cleanup': (state) => { + state.options.skipHpaCleanup = true; + }, + '--allow-remote-base-url': (state) => { + state.options.allowRemoteBaseUrl = true; + }, +}); export class ScenarioRunError extends Error { constructor(message, result) { @@ -79,50 +107,31 @@ export const defaultOptions = () => ({ }); export const parseArgs = (argv) => { - const options = defaultOptions(); - const explicitK6Env = {}; - let workloadExplicit = false; - - for (let index = 0; index < argv.length; index += 1) { - const arg = argv[index]; - if (arg === '--help' || arg === '-h') { - return { ...options, help: true }; + const state = { + argv, + explicitK6Env: {}, + index: 0, + options: defaultOptions(), + workloadExplicit: false, + }; + + for (; state.index < argv.length; state.index += 1) { + const arg = argv[state.index]; + if (HELP_OPTIONS.has(arg)) { + return { ...state.options, help: true }; } - if (arg === '--scenario') { - options.scenario = readOptionValue(argv, ++index, arg); - } else if (arg === '--workload') { - options.workload = readOptionValue(argv, ++index, arg); - workloadExplicit = true; - } else if (arg === '--base-url') { - options.baseUrl = readOptionValue(argv, ++index, arg); - } else if (arg === '--fixture-profile') { - options.fixtureProfile = readOptionValue(argv, ++index, arg); - } else if (arg === '--fixture-profile-file') { - options.fixtureProfileFile = readOptionValue(argv, ++index, arg); - } else if (arg === '--output-root') { - options.outputRoot = readOptionValue(argv, ++index, arg); - } else if (arg === '--script') { - options.script = readOptionValue(argv, ++index, arg); - } else if (arg === '--k6-bin') { - options.k6Bin = readOptionValue(argv, ++index, arg); - } else if (arg === '--k6-env') { - addK6Env(explicitK6Env, readOptionValue(argv, ++index, arg)); - } else if (arg === '--skip-apply') { - options.apply = false; - } else if (arg === '--skip-hpa-cleanup') { - options.skipHpaCleanup = true; - } else if (arg === '--allow-remote-base-url') { - options.allowRemoteBaseUrl = true; - } else { + const handler = OPTION_HANDLERS[arg]; + if (!handler) { throw new Error(`Unknown option: ${arg}`); } + handler(state, arg); } - options.k6Env = { ...explicitK6Env }; - const resolvedOptions = applyFixtureProfileDefaults(options, { - explicitK6Env, - fixtureFile: options.fixtureProfileFile, - workloadExplicit, + state.options.k6Env = { ...state.explicitK6Env }; + const resolvedOptions = applyFixtureProfileDefaults(state.options, { + explicitK6Env: state.explicitK6Env, + fixtureFile: state.options.fixtureProfileFile, + workloadExplicit: state.workloadExplicit, }); validateOptions(resolvedOptions); return resolvedOptions; @@ -293,6 +302,17 @@ const readOptionValue = (argv, index, option) => { return value; }; +function readOptionInto(property) { + return (state, option) => { + state.options[property] = readNextOptionValue(state, option); + }; +} + +function readNextOptionValue(state, option) { + state.index += 1; + return readOptionValue(state.argv, state.index, option); +} + const addK6Env = (k6Env, assignment) => { const separator = assignment.indexOf('='); if (separator < 1) { diff --git a/backend/lined/load-tests/runtime-scenarios/scenario-runner.test.mjs b/backend/lined/load-tests/runtime-scenarios/scenario-runner.test.mjs index d168c77..46876e1 100644 --- a/backend/lined/load-tests/runtime-scenarios/scenario-runner.test.mjs +++ b/backend/lined/load-tests/runtime-scenarios/scenario-runner.test.mjs @@ -1,4 +1,3 @@ -import assert from 'node:assert/strict'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; @@ -101,100 +100,152 @@ const pods = { }], }; +const TEXTS = Object.freeze({ + env: { + stressThinkTime: 'THINK_TIME_SECONDS', + token: 'TOKEN', + userCount: 'USER_COUNT', + vus: 'VUS', + }, + fixture: { + baseline: 'comparison-baseline', + readHeavy: 'comparison-read-heavy', + unknown: 'unknown', + unsafe: 'unsafe', + }, + scenario: { + fixedMedium: 'fixed-medium', + hpaCpu: 'hpa-cpu', + unknown: 'unknown', + }, + workload: { + baseline: 'baseline', + readHeavy: 'read-heavy', + smoke: 'smoke', + unknown: 'unknown', + }, +}); + +const VALUES = Object.freeze({ + events: { + baselineSeedCount: '8', + }, + tasks: { + baselineSeedCount: '12', + }, + thinkTime: { + none: '0', + }, + users: { + baselineCount: '4', + }, + vus: { + baseline: '5', + override: '2', + }, +}); + describe('parseArgs', () => { - it('accepts valid scenario, workload, and allowlisted k6 env options', () => { + it('accepts valid scenario, workload, and allowlisted k6 env options', (t) => { + t.plan(3); const options = parseArgs([ '--scenario', - 'fixed-medium', + TEXTS.scenario.fixedMedium, '--workload', - 'smoke', + TEXTS.workload.smoke, '--k6-env', - 'VUS=2', + `${TEXTS.env.vus}=${VALUES.vus.override}`, ]); - assert.equal(options.scenario, 'fixed-medium'); - assert.equal(options.workload, 'smoke'); - assert.equal(options.k6Env.VUS, '2'); + t.assert.equal(options.scenario, TEXTS.scenario.fixedMedium); + t.assert.equal(options.workload, TEXTS.workload.smoke); + t.assert.equal(options.k6Env.VUS, VALUES.vus.override); }); - it('applies a fixture profile as workload and k6 env defaults', () => { + it('applies a fixture profile as workload and k6 env defaults', (t) => { + t.plan(5); const options = parseArgs([ '--scenario', - 'fixed-medium', + TEXTS.scenario.fixedMedium, '--fixture-profile', - 'comparison-baseline', + TEXTS.fixture.baseline, ]); - assert.equal(options.fixtureProfileData.name, 'comparison-baseline'); - assert.equal(options.workload, 'baseline'); - assert.equal(options.k6Env.USER_COUNT, '4'); - assert.equal(options.k6Env.SEED_TASK_COUNT, '12'); - assert.equal(options.k6Env.VUS, '5'); + t.assert.equal(options.fixtureProfileData.name, TEXTS.fixture.baseline); + t.assert.equal(options.workload, TEXTS.workload.baseline); + t.assert.equal(options.k6Env.USER_COUNT, VALUES.users.baselineCount); + t.assert.equal(options.k6Env.SEED_TASK_COUNT, VALUES.tasks.baselineSeedCount); + t.assert.equal(options.k6Env.VUS, VALUES.vus.baseline); }); - it('lets explicit workload and k6 env override fixture defaults', () => { + it('lets explicit workload and k6 env override fixture defaults', (t) => { + t.plan(4); const options = parseArgs([ '--scenario', - 'fixed-medium', + TEXTS.scenario.fixedMedium, '--fixture-profile', - 'comparison-baseline', + TEXTS.fixture.baseline, '--workload', - 'read-heavy', + TEXTS.workload.readHeavy, '--k6-env', - 'VUS=2', + `${TEXTS.env.vus}=${VALUES.vus.override}`, '--k6-env', - 'THINK_TIME_SECONDS=0', + `${TEXTS.env.stressThinkTime}=${VALUES.thinkTime.none}`, ]); - assert.equal(options.workload, 'read-heavy'); - assert.equal(options.k6Env.USER_COUNT, '4'); - assert.equal(options.k6Env.VUS, '2'); - assert.equal(options.k6Env.THINK_TIME_SECONDS, '0'); + t.assert.equal(options.workload, TEXTS.workload.readHeavy); + t.assert.equal(options.k6Env.USER_COUNT, VALUES.users.baselineCount); + t.assert.equal(options.k6Env.VUS, VALUES.vus.override); + t.assert.equal(options.k6Env.THINK_TIME_SECONDS, VALUES.thinkTime.none); }); - it('rejects unknown scenarios and workloads', () => { - assert.throws( - () => parseArgs(['--scenario', 'unknown']), + it('rejects unknown scenarios and workloads', (t) => { + t.plan(2); + t.assert.throws( + () => parseArgs(['--scenario', TEXTS.scenario.unknown]), /--scenario must be one of/ ); - assert.throws( - () => parseArgs(['--scenario', 'fixed-medium', '--workload', 'unknown']), + t.assert.throws( + () => parseArgs(['--scenario', TEXTS.scenario.fixedMedium, '--workload', TEXTS.workload.unknown]), /--workload must be one of/ ); }); - it('rejects unknown fixture profiles', () => { - assert.throws( - () => parseArgs(['--scenario', 'fixed-medium', '--fixture-profile', 'unknown']), + it('rejects unknown fixture profiles', (t) => { + t.plan(1); + t.assert.throws( + () => parseArgs(['--scenario', TEXTS.scenario.fixedMedium, '--fixture-profile', TEXTS.fixture.unknown]), /--fixture-profile must be one of/ ); }); - it('rejects unsupported k6 env keys so secrets are not forwarded', () => { - assert.throws( - () => parseArgs(['--scenario', 'fixed-medium', '--k6-env', 'TOKEN=secret']), + it('rejects unsupported k6 env keys so secrets are not forwarded', (t) => { + t.plan(1); + t.assert.throws( + () => parseArgs(['--scenario', TEXTS.scenario.fixedMedium, '--k6-env', `${TEXTS.env.token}=secret`]), /Unsupported k6 env TOKEN/ ); }); - it('rejects unsupported fixture k6 env keys', () => { + it('rejects unsupported fixture k6 env keys', (t) => { + t.plan(1); const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'lined-fixtures-')); const fixtureFile = path.join(directory, 'fixtures.json'); fs.writeFileSync(fixtureFile, JSON.stringify({ schema_version: 1, profiles: { - unsafe: { - workload: 'baseline', + [TEXTS.fixture.unsafe]: { + workload: TEXTS.workload.baseline, k6_env: { - TOKEN: 'secret', + [TEXTS.env.token]: 'secret', }, }, }, }), 'utf-8'); try { - assert.throws( - () => loadFixtureProfile('unsafe', { file: fixtureFile }), + t.assert.throws( + () => loadFixtureProfile(TEXTS.fixture.unsafe, { file: fixtureFile }), /unsupported k6 env TOKEN/ ); } finally { @@ -204,24 +255,27 @@ describe('parseArgs', () => { }); describe('ensureLocalBaseUrl', () => { - it('accepts local targets by default', () => { - assert.doesNotThrow(() => ensureLocalBaseUrl('http://localhost:8080', false)); - assert.doesNotThrow(() => ensureLocalBaseUrl('http://127.0.0.1:8080', false)); - assert.doesNotThrow(() => ensureLocalBaseUrl('http://[::1]:8080', false)); + it('accepts local targets by default', (t) => { + t.plan(3); + t.assert.doesNotThrow(() => ensureLocalBaseUrl('http://localhost:8080', false)); + t.assert.doesNotThrow(() => ensureLocalBaseUrl('http://127.0.0.1:8080', false)); + t.assert.doesNotThrow(() => ensureLocalBaseUrl('http://[::1]:8080', false)); }); - it('rejects remote targets unless explicitly allowed', () => { - assert.throws( + it('rejects remote targets unless explicitly allowed', (t) => { + t.plan(2); + t.assert.throws( () => ensureLocalBaseUrl('http://example.com', false), /BASE_URL must point to localhost/ ); - assert.doesNotThrow(() => ensureLocalBaseUrl('http://example.com', true)); + t.assert.doesNotThrow(() => ensureLocalBaseUrl('http://example.com', true)); }); }); describe('runK6', () => { - it('reports a clear install hint when k6 is missing', () => { - assert.throws( + it('reports a clear install hint when k6 is missing', (t) => { + t.plan(1); + t.assert.throws( () => assertK6Available('missing-k6', { commandRunner: () => ({ error: Object.assign(new Error('spawn missing-k6 ENOENT'), { @@ -234,7 +288,8 @@ describe('runK6', () => { ); }); - it('builds argv arrays instead of shell command strings', () => { + it('builds argv arrays instead of shell command strings', (t) => { + t.plan(5); const calls = []; const commandRunner = (command, args) => { calls.push({ args, command }); @@ -255,16 +310,17 @@ describe('runK6', () => { { commandRunner } ); - assert.equal(calls[0].command, 'k6'); - assert.ok(Array.isArray(calls[0].args)); - assert.ok(calls[0].args.includes('--summary-export')); - assert.ok(calls[0].args.includes('VUS=2')); - assert.ok(calls[0].args.includes('ALLOW_REMOTE_BASE_URL=true')); + t.assert.equal(calls[0].command, 'k6'); + t.assert.ok(Array.isArray(calls[0].args)); + t.assert.ok(calls[0].args.includes('--summary-export')); + t.assert.ok(calls[0].args.includes('VUS=2')); + t.assert.ok(calls[0].args.includes('ALLOW_REMOTE_BASE_URL=true')); }); }); describe('Kubernetes state adapter helpers', () => { - it('deletes stale HPA for fixed scenarios unless skipped', () => { + it('deletes stale HPA for fixed scenarios unless skipped', (t) => { + t.plan(2); const calls = []; const commandRunner = (command, args) => { calls.push({ args, command }); @@ -277,8 +333,8 @@ describe('Kubernetes state adapter helpers', () => { { commandRunner } ); - assert.equal(cleaned, true); - assert.deepEqual(calls[0], { + t.assert.equal(cleaned, true); + t.assert.deepEqual(calls[0], { command: 'kubectl', args: [ '-n', @@ -291,7 +347,8 @@ describe('Kubernetes state adapter helpers', () => { }); }); - it('skips HPA cleanup when requested', () => { + it('skips HPA cleanup when requested', (t) => { + t.plan(2); const calls = []; const cleaned = cleanupHpaIfNeeded( { skipHpaCleanup: true }, @@ -304,51 +361,55 @@ describe('Kubernetes state adapter helpers', () => { } ); - assert.equal(cleaned, false); - assert.deepEqual(calls, []); + t.assert.equal(cleaned, false); + t.assert.deepEqual(calls, []); }); - it('parses Kubernetes resource quantities and pod top output', () => { - assert.equal(parseCpuQuantity('500m'), 500); - assert.equal(parseCpuQuantity('1'), 1000); - assert.equal(parseCpuQuantity('250u'), 0.25); - assert.equal(parseMemoryQuantity('1Gi'), 1024 ** 3); - assert.equal(parseMemoryQuantity('512Mi'), 512 * 1024 ** 2); - assert.deepEqual(parseTopPods('lined-backend-a 250m 512Mi\n'), [{ + it('parses Kubernetes resource quantities and pod top output', (t) => { + t.plan(6); + t.assert.equal(parseCpuQuantity('500m'), 500); + t.assert.equal(parseCpuQuantity('1'), 1000); + t.assert.equal(parseCpuQuantity('250u'), 0.25); + t.assert.equal(parseMemoryQuantity('1Gi'), 1024 ** 3); + t.assert.equal(parseMemoryQuantity('512Mi'), 512 * 1024 ** 2); + t.assert.deepEqual(parseTopPods('lined-backend-a 250m 512Mi\n'), [{ cpuMillicores: 250, memoryBytes: 512 * 1024 ** 2, name: 'lined-backend-a', }]); }); - it('summarizes Kubernetes utilization and restarts', () => { + it('summarizes Kubernetes utilization and restarts', (t) => { + t.plan(4); const result = summarizeKubernetesState({ deployment, pods, topOutput: 'lined-backend-a 250m 512Mi\nlined-backend-b 250m 512Mi\n', }); - assert.equal(result.cpuUtilization, 0.5); - assert.equal(result.memoryUtilization, 0.5); - assert.equal(result.restartCount, 3); - assert.equal(result.metricsServerAvailable, true); + t.assert.equal(result.cpuUtilization, 0.5); + t.assert.equal(result.memoryUtilization, 0.5); + t.assert.equal(result.restartCount, 3); + t.assert.equal(result.metricsServerAvailable, true); }); - it('omits utilization when metrics-server data is missing', () => { + it('omits utilization when metrics-server data is missing', (t) => { + t.plan(3); const result = summarizeKubernetesState({ deployment, pods, topOutput: '', }); - assert.equal(result.cpuUtilization, undefined); - assert.equal(result.memoryUtilization, undefined); - assert.equal(result.metricsServerAvailable, false); + t.assert.equal(result.cpuUtilization, undefined); + t.assert.equal(result.memoryUtilization, undefined); + t.assert.equal(result.metricsServerAvailable, false); }); }); describe('buildRuntimeSummary', () => { - it('builds a collector-compatible summary and records missing optional metrics', () => { + it('builds a collector-compatible summary and records missing optional metrics', (t) => { + t.plan(1); const summary = buildRuntimeSummary({ k6Summary: nestedK6Summary, kubernetes: { @@ -359,7 +420,7 @@ describe('buildRuntimeSummary', () => { workload: 'smoke', }); - assert.deepEqual(summary, { + t.assert.deepEqual(summary, { schema_version: 1, scenario: 'fixed-medium', workload: 'smoke', @@ -379,7 +440,8 @@ describe('buildRuntimeSummary', () => { }); }); - it('reads flat k6 v2 summary exports', () => { + it('reads flat k6 v2 summary exports', (t) => { + t.plan(4); const summary = buildRuntimeSummary({ k6Summary: flatK6Summary, kubernetes: { @@ -390,13 +452,14 @@ describe('buildRuntimeSummary', () => { workload: 'smoke', }); - assert.equal(summary.summary.latency_p95_ms, 150.25); - assert.equal(summary.summary.latency_p99_ms, 275.5); - assert.equal(summary.summary.error_rate, 0); - assert.equal(summary.summary.throughput_rps, 25.5); + t.assert.equal(summary.summary.latency_p95_ms, 150.25); + t.assert.equal(summary.summary.latency_p99_ms, 275.5); + t.assert.equal(summary.summary.error_rate, 0); + t.assert.equal(summary.summary.throughput_rps, 25.5); }); - it('marks missing HPA fields for the HPA scenario when no HPA state is present', () => { + it('marks missing HPA fields for the HPA scenario when no HPA state is present', (t) => { + t.plan(2); const summary = buildRuntimeSummary({ k6Summary: nestedK6Summary, kubernetes: { @@ -407,11 +470,12 @@ describe('buildRuntimeSummary', () => { workload: 'baseline', }); - assert.ok(summary.missing.includes('hpa_current_replicas')); - assert.ok(summary.missing.includes('hpa_desired_replicas')); + t.assert.ok(summary.missing.includes('hpa_current_replicas')); + t.assert.ok(summary.missing.includes('hpa_desired_replicas')); }); - it('uses measurement-window restart deltas instead of cumulative snapshots', () => { + it('uses measurement-window restart deltas instead of cumulative snapshots', (t) => { + t.plan(4); const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'lined-runner-')); try { @@ -435,16 +499,17 @@ describe('buildRuntimeSummary', () => { }) ); - assert.equal(result.summary.summary.restart_count, 2); - assert.equal(result.manifest.kubernetes.restart_count_before, 4); - assert.equal(result.manifest.kubernetes.restart_count_after, 6); - assert.equal(result.manifest.kubernetes.restart_count_delta, 2); + t.assert.equal(result.summary.summary.restart_count, 2); + t.assert.equal(result.manifest.kubernetes.restart_count_before, 4); + t.assert.equal(result.manifest.kubernetes.restart_count_after, 6); + t.assert.equal(result.manifest.kubernetes.restart_count_delta, 2); } finally { fs.rmSync(directory, { force: true, recursive: true }); } }); - it('does not emit a negative restart delta when pod counters reset', () => { + it('does not emit a negative restart delta when pod counters reset', (t) => { + t.plan(3); const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'lined-runner-')); try { @@ -468,20 +533,21 @@ describe('buildRuntimeSummary', () => { }) ); - assert.equal(result.summary.summary.restart_count, 0); - assert.equal(result.manifest.kubernetes.restart_count_before, 4); - assert.equal(result.manifest.kubernetes.restart_count_after, 1); + t.assert.equal(result.summary.summary.restart_count, 0); + t.assert.equal(result.manifest.kubernetes.restart_count_before, 4); + t.assert.equal(result.manifest.kubernetes.restart_count_after, 1); } finally { fs.rmSync(directory, { force: true, recursive: true }); } }); - it('fails without writing a collector summary when k6 omits summary export', () => { + it('fails without writing a collector summary when k6 omits summary export', (t) => { + t.plan(5); const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'lined-runner-')); try { let thrown; - assert.throws( + t.assert.throws( () => runScenario( { allowRemoteBaseUrl: false, @@ -512,10 +578,10 @@ describe('buildRuntimeSummary', () => { fs.readFileSync(path.join(runDir, 'runtime-summary-manifest.json'), 'utf-8') ); - assert.equal(thrown instanceof Error, true); - assert.equal(fs.existsSync(path.join(runDir, 'runtime-summary-manifest.json')), true); - assert.equal(fs.existsSync(path.join(runDir, 'runtime-summary.json')), false); - assert.equal(manifest.collector_summary_written, false); + t.assert.equal(thrown instanceof Error, true); + t.assert.equal(fs.existsSync(path.join(runDir, 'runtime-summary-manifest.json')), true); + t.assert.equal(fs.existsSync(path.join(runDir, 'runtime-summary.json')), false); + t.assert.equal(manifest.collector_summary_written, false); } finally { fs.rmSync(directory, { force: true, recursive: true }); } @@ -523,7 +589,8 @@ describe('buildRuntimeSummary', () => { }); describe('manifest and runScenario', () => { - it('records sanitized provenance in the manifest', () => { + it('records sanitized provenance in the manifest', (t) => { + t.plan(6); const manifest = buildManifest({ appliedScenario: true, finishedAt: '2026-06-01T10:00:10.000Z', @@ -567,10 +634,10 @@ describe('manifest and runScenario', () => { summaryWritten: true, }); - assert.equal(manifest.kubernetes.applied_scenario, true); - assert.equal(manifest.kubernetes.hpa_cleanup, true); - assert.equal(manifest.collector_summary_written, true); - assert.deepEqual(manifest.fixture_profile, { + t.assert.equal(manifest.kubernetes.applied_scenario, true); + t.assert.equal(manifest.kubernetes.hpa_cleanup, true); + t.assert.equal(manifest.collector_summary_written, true); + t.assert.deepEqual(manifest.fixture_profile, { name: 'comparison-baseline', schema_version: 1, workload: 'baseline', @@ -579,11 +646,12 @@ describe('manifest and runScenario', () => { VUS: '2', }, }); - assert.equal(manifest.workload_env.VUS, '2'); - assert.equal(manifest.git.branch, 'bug/scenario-runner-seam'); + t.assert.equal(manifest.workload_env.VUS, '2'); + t.assert.equal(manifest.git.branch, 'bug/scenario-runner-seam'); }); - it('writes a summary and manifest for successful runs', () => { + it('writes a summary and manifest for successful runs', (t) => { + t.plan(5); const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'lined-runner-')); try { @@ -606,17 +674,18 @@ describe('manifest and runScenario', () => { }) ); - assert.equal(fs.existsSync(result.summaryPath), true); - assert.equal(fs.existsSync(result.manifestPath), true); - assert.equal(result.summary.summary.latency_p95_ms, 250.5); - assert.equal(result.summary.fixture_profile, undefined); - assert.equal(result.manifest.collector_summary_written, true); + t.assert.equal(fs.existsSync(result.summaryPath), true); + t.assert.equal(fs.existsSync(result.manifestPath), true); + t.assert.equal(result.summary.summary.latency_p95_ms, 250.5); + t.assert.equal(result.summary.fixture_profile, undefined); + t.assert.equal(result.manifest.collector_summary_written, true); } finally { fs.rmSync(directory, { force: true, recursive: true }); } }); - it('applies fixture profiles when runScenario is called directly', () => { + it('applies fixture profiles when runScenario is called directly', (t) => { + t.plan(5); const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'lined-runner-')); try { @@ -625,16 +694,16 @@ describe('manifest and runScenario', () => { allowRemoteBaseUrl: false, apply: false, baseUrl: 'http://localhost:8080', - fixtureProfile: 'comparison-read-heavy', + fixtureProfile: TEXTS.fixture.readHeavy, k6Bin: 'k6', k6Env: { - VUS: '2', + VUS: VALUES.vus.override, }, outputRoot: directory, - scenario: 'fixed-medium', + scenario: TEXTS.scenario.fixedMedium, script: 'load-tests/k6/load-test-baseline.js', skipHpaCleanup: false, - workload: 'baseline', + workload: TEXTS.workload.baseline, }, fakeAdapters({ k6ExitCode: 0, @@ -642,17 +711,18 @@ describe('manifest and runScenario', () => { }) ); - assert.equal(result.summary.workload, 'read-heavy'); - assert.equal(result.manifest.fixture_profile.name, 'comparison-read-heavy'); - assert.equal(result.manifest.workload_env.WORKLOAD, 'read-heavy'); - assert.equal(result.manifest.workload_env.USER_COUNT, '4'); - assert.equal(result.manifest.workload_env.VUS, '2'); + t.assert.equal(result.summary.workload, TEXTS.workload.readHeavy); + t.assert.equal(result.manifest.fixture_profile.name, TEXTS.fixture.readHeavy); + t.assert.equal(result.manifest.workload_env.WORKLOAD, TEXTS.workload.readHeavy); + t.assert.equal(result.manifest.workload_env.USER_COUNT, VALUES.users.baselineCount); + t.assert.equal(result.manifest.workload_env.VUS, VALUES.vus.override); } finally { fs.rmSync(directory, { force: true, recursive: true }); } }); - it('lets direct runScenario workload overrides win over fixture defaults', () => { + it('lets direct runScenario workload overrides win over fixture defaults', (t) => { + t.plan(4); const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'lined-runner-')); try { @@ -661,14 +731,14 @@ describe('manifest and runScenario', () => { allowRemoteBaseUrl: false, apply: false, baseUrl: 'http://localhost:8080', - fixtureProfile: 'comparison-baseline', + fixtureProfile: TEXTS.fixture.baseline, k6Bin: 'k6', k6Env: {}, outputRoot: directory, - scenario: 'fixed-medium', + scenario: TEXTS.scenario.fixedMedium, script: 'load-tests/k6/load-test-baseline.js', skipHpaCleanup: false, - workload: 'read-heavy', + workload: TEXTS.workload.readHeavy, }, fakeAdapters({ k6ExitCode: 0, @@ -676,20 +746,21 @@ describe('manifest and runScenario', () => { }) ); - assert.equal(result.summary.workload, 'read-heavy'); - assert.equal(result.manifest.fixture_profile.name, 'comparison-baseline'); - assert.equal(result.manifest.workload_env.WORKLOAD, 'read-heavy'); - assert.equal(result.manifest.workload_env.USER_COUNT, '4'); + t.assert.equal(result.summary.workload, TEXTS.workload.readHeavy); + t.assert.equal(result.manifest.fixture_profile.name, TEXTS.fixture.baseline); + t.assert.equal(result.manifest.workload_env.WORKLOAD, TEXTS.workload.readHeavy); + t.assert.equal(result.manifest.workload_env.USER_COUNT, VALUES.users.baselineCount); } finally { fs.rmSync(directory, { force: true, recursive: true }); } }); - it('writes only a manifest when k6 fails', () => { + it('writes only a manifest when k6 fails', (t) => { + t.plan(4); const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'lined-runner-')); try { - assert.throws( + t.assert.throws( () => runScenario( { allowRemoteBaseUrl: false, @@ -712,10 +783,10 @@ describe('manifest and runScenario', () => { ); const runDirs = fs.readdirSync(directory); - assert.equal(runDirs.length, 1); + t.assert.equal(runDirs.length, 1); const runDir = path.join(directory, runDirs[0]); - assert.equal(fs.existsSync(path.join(runDir, 'runtime-summary-manifest.json')), true); - assert.equal(fs.existsSync(path.join(runDir, 'runtime-summary.json')), false); + t.assert.equal(fs.existsSync(path.join(runDir, 'runtime-summary-manifest.json')), true); + t.assert.equal(fs.existsSync(path.join(runDir, 'runtime-summary.json')), false); } finally { fs.rmSync(directory, { force: true, recursive: true }); } From 1707f437a9392c2e5ebde1097def32efdb80066d Mon Sep 17 00:00:00 2001 From: Oleksii Makieiev Date: Wed, 3 Jun 2026 11:51:18 +0300 Subject: [PATCH 3/3] Harden runtime scenario runner review paths --- backend/lined/.beads/interactions.jsonl | 1 - backend/lined/.beads/issues.jsonl | 16 +- .../runtime-scenarios/command-runner.mjs | 12 +- .../runtime-scenarios/fixture-profiles.mjs | 23 ++- .../runtime-scenarios/k6-adapter.mjs | 10 +- .../runtime-scenarios/kubernetes-adapter.mjs | 41 +++-- .../runtime-scenarios/runtime-summary.mjs | 1 + .../runtime-scenarios/scenario-runner.mjs | 19 +- .../scenario-runner.test.mjs | 166 +++++++++++++++++- 9 files changed, 262 insertions(+), 27 deletions(-) diff --git a/backend/lined/.beads/interactions.jsonl b/backend/lined/.beads/interactions.jsonl index 7ca776c..0e967a5 100644 --- a/backend/lined/.beads/interactions.jsonl +++ b/backend/lined/.beads/interactions.jsonl @@ -4,4 +4,3 @@ {"id":"int-4377e589","kind":"field_change","created_at":"2026-05-29T08:40:48.238866Z","actor":"Oleksii Makieiev","issue_id":"lined-vbf","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Implemented runtime scenario summary docs and CLI with critic review, verification, and Notion write-back."}} {"id":"int-5e657a3c","kind":"field_change","created_at":"2026-05-30T16:21:49.590801Z","actor":"Oleksii Makieiev","issue_id":"lined-z7y","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Implemented account provisioning policy seam and shared role resolver; focused tests and ./gradlew check pass."}} {"id":"int-7ae524e7","kind":"field_change","created_at":"2026-06-01T14:19:13.525997Z","actor":"Oleksii Makieiev","issue_id":"lined-0zb","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Completed calendar time-window refactor with validated window seam, conflict analyzer, header-bound requester checks, critic review, and ./gradlew check."}} -{"id":"int-be0a8ce1","kind":"field_change","created_at":"2026-06-03T08:15:00.608728Z","actor":"Oleksii Makieiev","issue_id":"lined-6cx","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Implemented versioned scenario fixture profiles, runner integration, docs, tests, and verified runtime scenario render checks."}} diff --git a/backend/lined/.beads/issues.jsonl b/backend/lined/.beads/issues.jsonl index 9147be9..3bc20ba 100644 --- a/backend/lined/.beads/issues.jsonl +++ b/backend/lined/.beads/issues.jsonl @@ -1 +1,15 @@ -{"_type":"issue","id":"lined-62z","title":"Add backend Docker image","description":"Add reproducible Docker image support and documented build/run flow for the Spring Boot backend as the experiment/backend-containerization task.","acceptance_criteria":"Dockerfile builds the backend image reproducibly; container run flow is documented; docs index routes to the containerization guide; backend behavior is unchanged.","status":"closed","priority":2,"issue_type":"task","assignee":"Oleksii Makieiev","owner":"alexmakeev2703@gmail.com","created_at":"2026-05-23T17:35:06Z","created_by":"Oleksii Makieiev","updated_at":"2026-05-23T17:41:10Z","started_at":"2026-05-23T17:35:17Z","closed_at":"2026-05-23T17:41:10Z","close_reason":"Implemented Dockerfile, .dockerignore, and containerization docs; Gradle check and JaCoCo passed. Docker daemon-backed build/run verification is tracked separately in lined-azh because the local Docker socket was unavailable.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"lined-51s","title":"Add scenario runner seam","description":"Architecture review bug bug/scenario-runner-seam: runtime evidence generation is split across procedural docs, kubectl ordering, k6 execution, telemetry collection, and hand-built summaries. Add a narrow runner seam that accepts scenario, workload, and output root, coordinates Kubernetes/k6/state adapters, and writes sanitized collector-ready runtime-summary artifacts.","acceptance_criteria":"A CLI under load-tests/runtime-scenarios accepts allowlisted scenario/workload inputs, coordinates kubectl and k6 through testable adapters, writes runtime-summary.json only for successful runs, writes sanitized manifests, includes unit tests, and documents the workflow.","status":"in_progress","priority":2,"issue_type":"bug","assignee":"Oleksii Makieiev","owner":"alexmakeev2703@gmail.com","created_at":"2026-06-01T15:08:34Z","created_by":"Oleksii Makieiev","updated_at":"2026-06-01T15:08:49Z","started_at":"2026-06-01T15:08:49Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"lined-0zb","title":"Deepen calendar time-window handling","description":"Architecture review bug: event flows pass raw OffsetDateTime pairs and repeat start/end validation across create, update, list, conflict, and user-conflict paths. Add one validated calendar time-window path and keep overlap/conflict rules local to the event scheduling module without changing public API shape.","status":"closed","priority":2,"issue_type":"bug","assignee":"Oleksii Makieiev","owner":"alexmakeev2703@gmail.com","created_at":"2026-06-01T14:06:34Z","created_by":"Oleksii Makieiev","updated_at":"2026-06-01T14:19:13Z","started_at":"2026-06-01T14:06:46Z","closed_at":"2026-06-01T14:19:13Z","close_reason":"Completed calendar time-window refactor with validated window seam, conflict analyzer, header-bound requester checks, critic review, and ./gradlew check.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"lined-z7y","title":"Fix account provisioning policy seam","description":"Architecture review bug from docs/experiment-tasks.md: AccountApplicationService exposes provisioning booleans and hard-coded default role/free-plan policy, while role resolution knowledge is duplicated between user and role modules. Implement one clear registration provisioning path, centralize default role/plan policy, and share role resolution.","status":"closed","priority":2,"issue_type":"bug","assignee":"Oleksii Makieiev","owner":"alexmakeev2703@gmail.com","created_at":"2026-05-30T16:16:47Z","created_by":"Oleksii Makieiev","updated_at":"2026-05-30T16:21:50Z","started_at":"2026-05-30T16:16:55Z","closed_at":"2026-05-30T16:21:50Z","close_reason":"Implemented account provisioning policy seam and shared role resolver; focused tests and ./gradlew check pass.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"lined-9pq","title":"Define SLO constraint thresholds","description":"Define initial local experiment SLO and constraint thresholds for classifying runtime-summary evidence for experiment/slo-constraint-thresholds. Keep backend behavior and scoring unchanged; add docs/config only.","status":"closed","priority":2,"issue_type":"task","assignee":"Oleksii Makieiev","owner":"alexmakeev2703@gmail.com","created_at":"2026-05-30T05:18:03Z","created_by":"Oleksii Makieiev","updated_at":"2026-05-30T05:24:19Z","started_at":"2026-05-30T05:18:18Z","closed_at":"2026-05-30T05:24:19Z","close_reason":"Completed SLO threshold docs and versioned config","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"lined-vbf","title":"Add runtime scenario summary workflow","description":"Implement experiment/runtime-scenario-summaries: add a repeatable local workflow for running kind deployment scenarios under k6 and producing sanitized runtime-summary.json artifacts for collector ingestion.","status":"closed","priority":2,"issue_type":"task","assignee":"Oleksii Makieiev","owner":"alexmakeev2703@gmail.com","created_at":"2026-05-29T08:25:25Z","created_by":"Oleksii Makieiev","updated_at":"2026-05-29T08:40:48Z","started_at":"2026-05-29T08:25:33Z","closed_at":"2026-05-29T08:40:48Z","close_reason":"Implemented runtime scenario summary docs and CLI with critic review, verification, and Notion write-back.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"lined-8jq","title":"Add Prometheus telemetry pipeline","description":"Implement experiment/prometheus-telemetry-pipeline from docs/experiment-tasks.md by adding local kind Prometheus manifests and documentation for collecting Actuator runtime metrics.","status":"closed","priority":2,"issue_type":"task","assignee":"Oleksii Makieiev","owner":"alexmakeev2703@gmail.com","created_at":"2026-05-28T20:30:07Z","created_by":"Oleksii Makieiev","updated_at":"2026-05-28T20:36:12Z","started_at":"2026-05-28T20:30:16Z","closed_at":"2026-05-28T20:36:12Z","close_reason":"Implemented Prometheus telemetry pipeline manifests and documentation","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"lined-hgb","title":"Add runtime fitness extension","description":"Implement experiment/fitness-runtime-extension by documenting runtime-aware fitness inputs and adding optional summarized runtime metrics ingestion without changing the existing structural fitnessScore.","acceptance_criteria":"Runtime fitness design is documented and linked; collector optionally reads summarized runtime metrics without failing when absent; existing fitnessScore semantics and analyzer compatibility are preserved; Notion research pages are updated and verified.","status":"closed","priority":2,"issue_type":"task","assignee":"Oleksii Makieiev","owner":"alexmakeev2703@gmail.com","created_at":"2026-05-28T06:21:08Z","created_by":"Oleksii Makieiev","updated_at":"2026-05-28T06:31:02Z","started_at":"2026-05-28T06:21:22Z","closed_at":"2026-05-28T06:31:02Z","close_reason":"Implemented runtime fitness design docs, optional collector runtime summary ingestion, critic review loop, verification, and Notion write-back.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"lined-byj","title":"Add load-test baseline","description":"Implement experiment/load-test-baseline by adding a repeatable k6 workload for users, lobbies, tasks, and events against the local kind backend baseline.","acceptance_criteria":"k6 baseline script creates synthetic data and exercises valid workflows; documentation explains kind port-forward, k6/Docker commands, synthetic data, and Prometheus metrics verification; no Spring business behavior changes.","status":"closed","priority":2,"issue_type":"task","assignee":"Oleksii Makieiev","owner":"alexmakeev2703@gmail.com","created_at":"2026-05-25T19:40:17Z","created_by":"Oleksii Makieiev","updated_at":"2026-05-26T04:33:19Z","started_at":"2026-05-25T19:40:27Z","closed_at":"2026-05-26T04:33:19Z","close_reason":"Completed: added bounded k6 load-test baseline script, documented smoke/baseline workflows, Docker fallback, synthetic data behavior, and runtime metrics verification; Gradle test/check and static script checks passed.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"lined-cy8","title":"Runtime metrics baseline","description":"Implement experiment/runtime-metrics-baseline from docs/experiment-tasks.md: expose Prometheus-compatible backend metrics and document key runtime signals so /actuator/prometheus can be collected for latency, error, and resource analysis.","status":"closed","priority":2,"issue_type":"task","assignee":"Oleksii Makieiev","owner":"alexmakeev2703@gmail.com","created_at":"2026-05-25T19:11:10Z","created_by":"Oleksii Makieiev","updated_at":"2026-05-25T19:22:23Z","started_at":"2026-05-25T19:11:18Z","closed_at":"2026-05-25T19:22:23Z","close_reason":"Implemented runtime metrics baseline with Prometheus scrape metadata, documentation, and local verification.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"lined-stn","title":"Add backend Kubernetes health probes","description":"Implement experiment/backend-health-probes by configuring the kind backend Deployment with Spring Boot Actuator readiness and liveness probes, documenting verification steps, and preserving backend business behavior.","acceptance_criteria":"Backend Deployment has readinessProbe and livenessProbe using Actuator endpoints; readiness and liveness health groups are explicit; kind baseline docs explain probe verification; Gradle quality gates and kind rollout verification pass.","status":"closed","priority":2,"issue_type":"task","assignee":"Oleksii Makieiev","owner":"alexmakeev2703@gmail.com","created_at":"2026-05-25T18:08:23Z","created_by":"Oleksii Makieiev","updated_at":"2026-05-25T18:14:45Z","started_at":"2026-05-25T18:08:35Z","closed_at":"2026-05-25T18:14:45Z","close_reason":"Completed: configured Actuator readiness and liveness health groups, added Kubernetes probes to the kind backend Deployment, documented verification steps, and verified Gradle test/check/JaCoCo plus kind rollout and Actuator smoke tests.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"lined-wq2","title":"Deploy backend baseline to kind","description":"Implement experiment/kind-postgres-backend-baseline by adding local kind Kubernetes manifests for PostgreSQL and the Spring Boot backend, plus documentation for building, loading, applying, port-forwarding, and verifying the Actuator health endpoint.","acceptance_criteria":"PostgreSQL and backend Kubernetes manifests exist under k8s/kind; docs explain the reproducible kind workflow; backend health can be verified at /actuator/health after port-forwarding; Java business behavior is unchanged.","status":"closed","priority":2,"issue_type":"task","assignee":"Oleksii Makieiev","owner":"alexmakeev2703@gmail.com","created_at":"2026-05-23T17:57:50Z","created_by":"Oleksii Makieiev","updated_at":"2026-05-23T18:03:39Z","started_at":"2026-05-23T17:57:59Z","closed_at":"2026-05-23T18:03:39Z","close_reason":"Completed: added k8s/kind manifests, documented the kind baseline workflow, and verified Docker build, kind load, rollout status, and /actuator/health smoke test.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"lined-62z","title":"Add backend Docker image","description":"Add reproducible Docker image support and documented build/run flow for the Spring Boot backend as the experiment/backend-containerization task.","acceptance_criteria":"Dockerfile builds the backend image reproducibly; container run flow is documented; docs index routes to the containerization guide; backend behavior is unchanged.","status":"open","priority":2,"issue_type":"task","owner":"alexmakeev2703@gmail.com","created_at":"2026-05-23T17:35:06Z","created_by":"Oleksii Makieiev","updated_at":"2026-05-23T17:35:06Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"lined-15y","title":"Remove redundant calendar requesterId query parameter","description":"PR #43 review follow-up: conflict endpoints now bind requester identity to X-User-Id, making the requesterId query parameter redundant. Deprecate and remove requesterId from /api/calendar/conflicts and /api/calendar/user-conflict in a compatibility-focused cycle.","status":"open","priority":3,"issue_type":"task","owner":"alexmakeev2703@gmail.com","created_at":"2026-06-01T14:40:15Z","created_by":"Oleksii Makieiev","updated_at":"2026-06-01T14:40:15Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"lined-oss","title":"Document Notion knowledge-base workflow","description":"Add backend documentation for using Notion as the durable knowledge base, including write-back checklist, verification after write, fallback policy, and entry template.","status":"closed","priority":3,"issue_type":"task","owner":"alexmakeev2703@gmail.com","created_at":"2026-05-26T20:53:14Z","created_by":"Oleksii Makieiev","updated_at":"2026-05-26T20:53:24Z","closed_at":"2026-05-26T20:53:24Z","close_reason":"Completed: added Notion knowledge-base workflow doc, linked it from backend docs index and AGENTS routing, and verified the documentation diff.","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"lined-azh","title":"Verify backend Docker image with running daemon","description":"Docker build and container smoke checks for experiment/backend-containerization could not run in this session because Docker daemon socket /Users/oleksii_makieiev/.docker/run/docker.sock was missing. Start Docker Desktop or another daemon, run docker build -t lined-backend:local ., then run the image against PostgreSQL and verify /actuator/health and /swagger-ui.html.","acceptance_criteria":"docker build succeeds; container starts against PostgreSQL; actuator health and Swagger UI are reachable from localhost.","status":"open","priority":3,"issue_type":"task","owner":"alexmakeev2703@gmail.com","created_at":"2026-05-23T17:39:18Z","created_by":"Oleksii Makieiev","updated_at":"2026-05-23T17:39:18Z","dependency_count":0,"dependent_count":0,"comment_count":0} diff --git a/backend/lined/load-tests/runtime-scenarios/command-runner.mjs b/backend/lined/load-tests/runtime-scenarios/command-runner.mjs index 5f8c6d6..1933375 100644 --- a/backend/lined/load-tests/runtime-scenarios/command-runner.mjs +++ b/backend/lined/load-tests/runtime-scenarios/command-runner.mjs @@ -3,20 +3,30 @@ import { spawnSync } from 'node:child_process'; export const runCommand = ( command, args, - { allowFailure = false, capture = false, cwd } = {} + { allowFailure = false, capture = false, cwd, timeoutMs } = {} ) => { const result = spawnSync(command, args, { cwd, encoding: capture ? 'utf-8' : undefined, stdio: capture ? ['ignore', 'pipe', 'pipe'] : 'inherit', + timeout: timeoutMs, }); + if (isTimeout(result, timeoutMs) && !allowFailure) { + throw new Error(`${command} timed out after ${timeoutMs}ms`); + } if (result.error && !allowFailure) { throw result.error; } + if (result.signal && !allowFailure) { + throw new Error(`${command} was killed by signal ${result.signal}`); + } if (!allowFailure && result.status !== 0) { throw new Error(`${command} ${args.join(' ')} failed with exit code ${result.status}`); } return result; }; + +const isTimeout = (result, timeoutMs) => timeoutMs !== undefined + && (result.error?.code === 'ETIMEDOUT' || result.signal === 'SIGTERM'); diff --git a/backend/lined/load-tests/runtime-scenarios/fixture-profiles.mjs b/backend/lined/load-tests/runtime-scenarios/fixture-profiles.mjs index 36b4af9..428ba93 100644 --- a/backend/lined/load-tests/runtime-scenarios/fixture-profiles.mjs +++ b/backend/lined/load-tests/runtime-scenarios/fixture-profiles.mjs @@ -23,14 +23,14 @@ export const fixtureProfileNames = ({ cwd = process.cwd(), file = FIXTURE_PROFIL export const loadFixtureProfile = ( name, - { cwd = process.cwd(), file = FIXTURE_PROFILES_PATH } = {} + { allowedWorkloads, cwd = process.cwd(), file = FIXTURE_PROFILES_PATH } = {} ) => { const artifact = readFixtureArtifact(cwd, file); const profile = artifact.profiles[name]; if (!profile) { throw new Error(`--fixture-profile must be one of: ${Object.keys(artifact.profiles).sort().join(', ')}`); } - validateProfile(name, profile); + validateProfile(name, profile, allowedWorkloads); return { description: profile.description, k6Env: { ...profile.k6_env }, @@ -46,6 +46,7 @@ export const applyFixtureProfileDefaults = ( cwd = process.cwd(), explicitK6Env = {}, fixtureFile = FIXTURE_PROFILES_PATH, + allowedWorkloads, workloadExplicit = false, } = {} ) => { @@ -53,7 +54,11 @@ export const applyFixtureProfileDefaults = ( return options; } - const profile = loadFixtureProfile(options.fixtureProfile, { cwd, file: fixtureFile }); + const profile = loadFixtureProfile(options.fixtureProfile, { + allowedWorkloads, + cwd, + file: fixtureFile, + }); const merged = { ...options, fixtureProfileData: profile, @@ -77,10 +82,10 @@ const readFixtureArtifact = (cwd, file) => { return parsed; }; -const validateProfile = (name, profile) => { +const validateProfile = (name, profile, allowedWorkloads) => { requireRecord(`fixture profile ${name}`, profile); requireKnownProfileKeys(name, profile); - requireWorkload(name, profile.workload); + requireWorkload(name, profile.workload, allowedWorkloads); requireK6Env(name, profile.k6_env); }; @@ -97,10 +102,16 @@ const requireKnownProfileKeys = (name, profile) => { } }; -const requireWorkload = (name, workload) => { +const requireWorkload = (name, workload, allowedWorkloads) => { if (typeof workload !== 'string' || workload.length === 0) { throw new Error(`fixture profile ${name} must define workload`); } + if (allowedWorkloads && !allowedWorkloads.has(workload)) { + throw new Error( + `fixture profile ${name} has unsupported workload ${workload}; ` + + `allowed: ${Array.from(allowedWorkloads).join(', ')}` + ); + } }; const requireK6Env = (name, k6Env) => { diff --git a/backend/lined/load-tests/runtime-scenarios/k6-adapter.mjs b/backend/lined/load-tests/runtime-scenarios/k6-adapter.mjs index 94da01c..7522c87 100644 --- a/backend/lined/load-tests/runtime-scenarios/k6-adapter.mjs +++ b/backend/lined/load-tests/runtime-scenarios/k6-adapter.mjs @@ -6,6 +6,8 @@ import { runCommand } from './command-runner.mjs'; import { parseK6Summary } from './runtime-summary.mjs'; export const SUMMARY_TREND_STATS = 'p(95),p(99),avg,min,max'; +export const K6_PREFLIGHT_TIMEOUT_MS = 30_000; +export const K6_RUN_TIMEOUT_MS = 600_000; export const assertK6Available = ( k6Bin, @@ -15,6 +17,7 @@ export const assertK6Available = ( allowFailure: true, capture: true, cwd, + timeoutMs: K6_PREFLIGHT_TIMEOUT_MS, }); if (result.error?.code === 'ENOENT') { @@ -27,6 +30,9 @@ export const assertK6Available = ( if (result.error) { throw result.error; } + if (result.signal) { + throw new Error(`k6 preflight was killed by signal ${result.signal}`); + } if (result.status !== 0) { throw new Error(`k6 preflight failed with exit code ${result.status}`); } @@ -61,12 +67,14 @@ export const runK6 = ( const result = commandRunner(options.k6Bin, args, { allowFailure: true, cwd, + timeoutMs: K6_RUN_TIMEOUT_MS, }); try { return { args, - exitCode: result.status, + exitCode: result.signal ? null : result.status, + signal: result.signal ?? undefined, summary: fs.existsSync(summaryPath) ? parseK6Summary(fs.readFileSync(summaryPath, 'utf-8')) : undefined, diff --git a/backend/lined/load-tests/runtime-scenarios/kubernetes-adapter.mjs b/backend/lined/load-tests/runtime-scenarios/kubernetes-adapter.mjs index 41e26fb..0da16a2 100644 --- a/backend/lined/load-tests/runtime-scenarios/kubernetes-adapter.mjs +++ b/backend/lined/load-tests/runtime-scenarios/kubernetes-adapter.mjs @@ -3,6 +3,9 @@ import { runCommand } from './command-runner.mjs'; export const NAMESPACE = 'lined'; export const BACKEND_DEPLOYMENT = 'lined-backend'; export const BACKEND_LABEL = 'app.kubernetes.io/name=lined-backend'; +export const KUBECTL_TIMEOUT_MS = 60_000; +export const KUBECTL_TOP_TIMEOUT_MS = 30_000; +export const KUBECTL_ROLLOUT_TIMEOUT_MS = 600_000; export const cleanupHpaIfNeeded = ( options, @@ -20,7 +23,7 @@ export const cleanupHpaIfNeeded = ( 'hpa', BACKEND_DEPLOYMENT, '--ignore-not-found', - ], { cwd }); + ], { cwd, timeoutMs: KUBECTL_TIMEOUT_MS }); return true; }; @@ -34,7 +37,10 @@ export const applyScenarioIfNeeded = ( return false; } - commandRunner('kubectl', ['apply', '-k', scenario.path], { cwd }); + commandRunner('kubectl', ['apply', '-k', scenario.path], { + cwd, + timeoutMs: KUBECTL_TIMEOUT_MS, + }); return true; }; @@ -47,7 +53,7 @@ export const waitForRollout = ( 'rollout', 'status', `deployment/${BACKEND_DEPLOYMENT}`, - ], { cwd }); + ], { cwd, timeoutMs: KUBECTL_ROLLOUT_TIMEOUT_MS }); }; export const collectKubernetesState = ( @@ -62,7 +68,7 @@ export const collectKubernetesState = ( BACKEND_DEPLOYMENT, '-o', 'json', - ], cwd); + ], cwd, KUBECTL_TIMEOUT_MS); const pods = readKubectlJson(commandRunner, [ '-n', NAMESPACE, @@ -72,7 +78,7 @@ export const collectKubernetesState = ( BACKEND_LABEL, '-o', 'json', - ], cwd); + ], cwd, KUBECTL_TIMEOUT_MS); const hpa = readOptionalKubectlJson(commandRunner, [ '-n', NAMESPACE, @@ -81,7 +87,7 @@ export const collectKubernetesState = ( BACKEND_DEPLOYMENT, '-o', 'json', - ], cwd); + ], cwd, KUBECTL_TIMEOUT_MS); const top = readOptionalKubectlText(commandRunner, [ '-n', NAMESPACE, @@ -90,7 +96,7 @@ export const collectKubernetesState = ( '-l', BACKEND_LABEL, '--no-headers', - ], cwd); + ], cwd, KUBECTL_TOP_TIMEOUT_MS); return { ...summarizeKubernetesState({ @@ -186,7 +192,11 @@ export const parseTopPods = (content = '') => content .map((line) => line.trim()) .filter(Boolean) .map((line) => { - const [name, cpu, memory] = line.split(/\s+/); + const parts = line.split(/\s+/); + if (parts.length < 3) { + throw new Error(`Malformed kubectl top pods line: ${line}`); + } + const [name, cpu, memory] = parts; return { cpuMillicores: parseCpuQuantity(cpu), memoryBytes: parseMemoryQuantity(memory), @@ -194,20 +204,22 @@ export const parseTopPods = (content = '') => content }; }); -const readKubectlJson = (commandRunner, args, cwd) => JSON.parse(commandRunner( +const readKubectlJson = (commandRunner, args, cwd, timeoutMs) => JSON.parse(commandRunner( 'kubectl', args, { capture: true, cwd, + timeoutMs, } ).stdout); -const readOptionalKubectlJson = (commandRunner, args, cwd) => { +const readOptionalKubectlJson = (commandRunner, args, cwd, timeoutMs) => { const result = commandRunner('kubectl', args, { allowFailure: true, capture: true, cwd, + timeoutMs, }); const output = String(result.stdout ?? ''); if (result.error || result.status !== 0 || output.trim() === '') { @@ -216,11 +228,12 @@ const readOptionalKubectlJson = (commandRunner, args, cwd) => { return JSON.parse(output); }; -const readOptionalKubectlText = (commandRunner, args, cwd) => { +const readOptionalKubectlText = (commandRunner, args, cwd, timeoutMs) => { const result = commandRunner('kubectl', args, { allowFailure: true, capture: true, cwd, + timeoutMs, }); return result.error || result.status !== 0 ? '' : String(result.stdout ?? ''); }; @@ -263,7 +276,11 @@ const sumPodUsage = (usage, field) => { }; const ratioOrUndefined = (numerator, denominator) => { - if (numerator === undefined || denominator === undefined || denominator <= 0) { + if ( + !Number.isFinite(numerator) + || !Number.isFinite(denominator) + || denominator <= 0 + ) { return undefined; } return Number((numerator / denominator).toFixed(6)); diff --git a/backend/lined/load-tests/runtime-scenarios/runtime-summary.mjs b/backend/lined/load-tests/runtime-scenarios/runtime-summary.mjs index b9a86fa..271d95a 100644 --- a/backend/lined/load-tests/runtime-scenarios/runtime-summary.mjs +++ b/backend/lined/load-tests/runtime-scenarios/runtime-summary.mjs @@ -94,6 +94,7 @@ export const buildManifest = ({ exit_code: k6.exitCode, executable: options.k6Bin, script: options.script, + signal: k6.signal, summary_exported: summaryExported, summary_trend_stats: k6.summaryTrendStats, }, diff --git a/backend/lined/load-tests/runtime-scenarios/scenario-runner.mjs b/backend/lined/load-tests/runtime-scenarios/scenario-runner.mjs index 7ee448b..3963db4 100644 --- a/backend/lined/load-tests/runtime-scenarios/scenario-runner.mjs +++ b/backend/lined/load-tests/runtime-scenarios/scenario-runner.mjs @@ -129,6 +129,7 @@ export const parseArgs = (argv) => { state.options.k6Env = { ...state.explicitK6Env }; const resolvedOptions = applyFixtureProfileDefaults(state.options, { + allowedWorkloads: new Set(WORKLOADS), explicitK6Env: state.explicitK6Env, fixtureFile: state.options.fixtureProfileFile, workloadExplicit: state.workloadExplicit, @@ -189,6 +190,7 @@ export const runScenario = ( hpaCleanup, k6: { exitCode: k6Result.exitCode, + signal: k6Result.signal, summaryTrendStats: SUMMARY_TREND_STATS, }, kubernetes, @@ -203,6 +205,17 @@ export const runScenario = ( fs.mkdirSync(outputDir, { recursive: true }); writeJson(manifestPath, manifest); + if (k6Result.signal) { + throw new ScenarioRunError( + `k6 was killed by signal ${k6Result.signal}; ` + + `wrote manifest ${manifestPath} but did not write collector summary`, + { + manifest, + manifestPath, + } + ); + } + if (k6Result.exitCode !== 0) { throw new ScenarioRunError( `k6 failed with exit code ${k6Result.exitCode}; ` @@ -246,13 +259,17 @@ const resolveFixtureOptions = (options) => { k6Env: options.k6Env ?? {}, }, { + allowedWorkloads: new Set(WORKLOADS), explicitK6Env: options.k6Env ?? {}, fixtureFile: options.fixtureProfileFile ?? FIXTURE_PROFILES_PATH, - workloadExplicit: options.workload !== undefined && options.workload !== DEFAULT_WORKLOAD, + workloadExplicit: isProgrammaticWorkloadExplicit(options), } ); }; +const isProgrammaticWorkloadExplicit = (options) => options.workloadExplicit === true + || (options.workload !== undefined && options.workload !== DEFAULT_WORKLOAD); + export const ensureLocalBaseUrl = (baseUrl, allowRemoteBaseUrl) => { if (allowRemoteBaseUrl) { return; diff --git a/backend/lined/load-tests/runtime-scenarios/scenario-runner.test.mjs b/backend/lined/load-tests/runtime-scenarios/scenario-runner.test.mjs index 46876e1..04372f0 100644 --- a/backend/lined/load-tests/runtime-scenarios/scenario-runner.test.mjs +++ b/backend/lined/load-tests/runtime-scenarios/scenario-runner.test.mjs @@ -3,6 +3,7 @@ import os from 'node:os'; import path from 'node:path'; import { describe, it } from 'node:test'; +import { runCommand } from './command-runner.mjs'; import { assertK6Available, runK6 } from './k6-adapter.mjs'; import { cleanupHpaIfNeeded, @@ -120,6 +121,7 @@ const TEXTS = Object.freeze({ }, workload: { baseline: 'baseline', + typo: 'basline', readHeavy: 'read-heavy', smoke: 'smoke', unknown: 'unknown', @@ -252,6 +254,50 @@ describe('parseArgs', () => { fs.rmSync(directory, { force: true, recursive: true }); } }); + + it('rejects unsupported fixture workloads before option validation', (t) => { + t.plan(1); + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'lined-fixtures-')); + const fixtureFile = path.join(directory, 'fixtures.json'); + fs.writeFileSync(fixtureFile, JSON.stringify({ + schema_version: 1, + profiles: { + [TEXTS.fixture.unsafe]: { + workload: TEXTS.workload.typo, + k6_env: {}, + }, + }, + }), 'utf-8'); + + try { + t.assert.throws( + () => parseArgs([ + '--scenario', + TEXTS.scenario.fixedMedium, + '--fixture-profile', + TEXTS.fixture.unsafe, + '--fixture-profile-file', + fixtureFile, + ]), + /fixture profile unsafe has unsupported workload basline/ + ); + } finally { + fs.rmSync(directory, { force: true, recursive: true }); + } + }); +}); + +describe('runCommand', () => { + it('fails with a timeout-specific message', (t) => { + t.plan(1); + t.assert.throws( + () => runCommand(process.execPath, ['-e', 'setTimeout(() => {}, 1000)'], { + capture: true, + timeoutMs: 10, + }), + /timed out after 10ms/ + ); + }); }); describe('ensureLocalBaseUrl', () => { @@ -289,10 +335,10 @@ describe('runK6', () => { }); it('builds argv arrays instead of shell command strings', (t) => { - t.plan(5); + t.plan(6); const calls = []; - const commandRunner = (command, args) => { - calls.push({ args, command }); + const commandRunner = (command, args, options) => { + calls.push({ args, command, options }); return { status: 0 }; }; @@ -315,6 +361,30 @@ describe('runK6', () => { t.assert.ok(calls[0].args.includes('--summary-export')); t.assert.ok(calls[0].args.includes('VUS=2')); t.assert.ok(calls[0].args.includes('ALLOW_REMOTE_BASE_URL=true')); + t.assert.equal(typeof calls[0].options.timeoutMs, 'number'); + }); + + it('reports a signal separately from exit code', (t) => { + t.plan(2); + const result = runK6( + { + allowRemoteBaseUrl: false, + baseUrl: 'http://localhost:8080', + k6Bin: 'k6', + k6Env: {}, + script: 'load-tests/k6/load-test-baseline.js', + workload: 'smoke', + }, + { + commandRunner: () => ({ + signal: 'SIGTERM', + status: null, + }), + } + ); + + t.assert.equal(result.exitCode, null); + t.assert.equal(result.signal, 'SIGTERM'); }); }); @@ -379,6 +449,14 @@ describe('Kubernetes state adapter helpers', () => { }]); }); + it('rejects malformed pod top output', (t) => { + t.plan(1); + t.assert.throws( + () => parseTopPods('lined-backend-a 250m\n'), + /Malformed kubectl top pods line/ + ); + }); + it('summarizes Kubernetes utilization and restarts', (t) => { t.plan(4); const result = summarizeKubernetesState({ @@ -601,6 +679,7 @@ describe('manifest and runScenario', () => { hpaCleanup: true, k6: { exitCode: 0, + signal: undefined, summaryTrendStats: 'p(95),p(99),avg,min,max', }, kubernetes: { @@ -755,6 +834,79 @@ describe('manifest and runScenario', () => { } }); + it('lets direct runScenario default workload overrides win when marked explicit', (t) => { + t.plan(4); + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'lined-runner-')); + + try { + const result = runScenario( + { + allowRemoteBaseUrl: false, + apply: false, + baseUrl: 'http://localhost:8080', + fixtureProfile: TEXTS.fixture.readHeavy, + k6Bin: 'k6', + k6Env: {}, + outputRoot: directory, + scenario: TEXTS.scenario.fixedMedium, + script: 'load-tests/k6/load-test-baseline.js', + skipHpaCleanup: false, + workload: TEXTS.workload.baseline, + workloadExplicit: true, + }, + fakeAdapters({ + k6ExitCode: 0, + k6Summary: nestedK6Summary, + }) + ); + + t.assert.equal(result.summary.workload, TEXTS.workload.baseline); + t.assert.equal(result.manifest.fixture_profile.name, TEXTS.fixture.readHeavy); + t.assert.equal(result.manifest.workload_env.WORKLOAD, TEXTS.workload.baseline); + t.assert.equal(result.manifest.workload_env.USER_COUNT, VALUES.users.baselineCount); + } finally { + fs.rmSync(directory, { force: true, recursive: true }); + } + }); + + it('uses signal-specific errors when k6 is killed', (t) => { + t.plan(2); + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'lined-runner-')); + + try { + t.assert.throws( + () => runScenario( + { + allowRemoteBaseUrl: false, + apply: false, + baseUrl: 'http://localhost:8080', + k6Bin: 'k6', + k6Env: {}, + outputRoot: directory, + scenario: 'fixed-medium', + script: 'load-tests/k6/load-test-baseline.js', + skipHpaCleanup: false, + workload: 'smoke', + }, + fakeAdapters({ + k6ExitCode: null, + k6Signal: 'SIGTERM', + k6Summary: nestedK6Summary, + }) + ), + /k6 was killed by signal SIGTERM/ + ); + + const runDir = path.join(directory, fs.readdirSync(directory)[0]); + const manifest = JSON.parse( + fs.readFileSync(path.join(runDir, 'runtime-summary-manifest.json'), 'utf-8') + ); + t.assert.equal(manifest.k6.signal, 'SIGTERM'); + } finally { + fs.rmSync(directory, { force: true, recursive: true }); + } + }); + it('writes only a manifest when k6 fails', (t) => { t.plan(4); const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'lined-runner-')); @@ -793,7 +945,12 @@ describe('manifest and runScenario', () => { }); }); -const fakeAdapters = ({ k6ExitCode, k6Summary, restartCounts = [0, 0] }) => ({ +const fakeAdapters = ({ + k6ExitCode, + k6Signal, + k6Summary, + restartCounts = [0, 0], +}) => ({ clock: fakeClock(), gitReader: () => ({ branch: 'bug/scenario-runner-seam', @@ -803,6 +960,7 @@ const fakeAdapters = ({ k6ExitCode, k6Summary, restartCounts = [0, 0] }) => ({ assertAvailable: () => {}, run: () => ({ exitCode: k6ExitCode, + signal: k6Signal, summary: k6Summary, }), },