diff --git a/dashboard/src/engine/__tests__/releaseCert.test.ts b/dashboard/src/engine/__tests__/releaseCert.test.ts index d03249b..c833cb0 100644 --- a/dashboard/src/engine/__tests__/releaseCert.test.ts +++ b/dashboard/src/engine/__tests__/releaseCert.test.ts @@ -1,11 +1,14 @@ import { checkBranch, checkPageForOverlay, + checkServerLog, runCert, CERT_URL_MATRIX, OVERLAY_MARKERS, + STDERR_SIGNATURES, } from "../releaseCert"; import type { ExecFn } from "../releaseCert"; +import { writeFileSync, unlinkSync } from "node:fs"; function fakeExec(branch: string, porcelain: string): ExecFn { return (cmd: string) => { @@ -115,6 +118,91 @@ describe("OVERLAY_MARKERS", () => { }); }); +// --- STDERR_SIGNATURES --- + +describe("STDERR_SIGNATURES", () => { + it("includes critical server crash patterns", () => { + expect(STDERR_SIGNATURES).toContain("⨯ Error"); + expect(STDERR_SIGNATURES).toContain("TypeError:"); + expect(STDERR_SIGNATURES).toContain("Hydration failed"); + expect(STDERR_SIGNATURES).toContain("digest:"); + }); +}); + +// --- checkServerLog --- + +describe("checkServerLog", () => { + const tmpLog = "/tmp/s24-test-server.log"; + + afterEach(() => { + try { unlinkSync(tmpLog); } catch { /* ignore */ } + }); + + it("returns ok:true when logPath is null", () => { + const result = checkServerLog(null); + expect(result.ok).toBe(true); + expect(result.signatures).toEqual([]); + }); + + it("returns ok:true for clean log file", () => { + writeFileSync(tmpLog, "Ready in 200ms\nListening on port 3000\n"); + const result = checkServerLog(tmpLog); + expect(result.ok).toBe(true); + expect(result.signatures).toEqual([]); + }); + + it("detects stderr crash signatures", () => { + writeFileSync(tmpLog, "Ready in 200ms\n⨯ Error: component threw during render\n digest: '123'\n"); + const result = checkServerLog(tmpLog); + expect(result.ok).toBe(false); + expect(result.signatures).toContain("⨯ Error"); + expect(result.signatures).toContain("digest:"); + }); + + it("AT-S24-06: fails closed when log file does not exist", () => { + const result = checkServerLog("/tmp/nonexistent-s24.log"); + expect(result.ok).toBe(false); + expect(result.signatures).toContain("LOG_READ_FAILED"); + }); +}); + +// --- S24 Harness: composed overlay-fallacy proof (AT-S24-02/03/04) --- + +describe("S24 overlay-fallacy harness (DI + fixtures)", () => { + const tmpLog = "/tmp/s24-harness-server.log"; + afterEach(() => { try { unlinkSync(tmpLog); } catch { /* noop */ } }); + + const CLEAN_HTML = '
OK
'; + const CRASH_LOG = + "✓ Ready in 200ms\n⨯ Error: component threw during server render\n" + + " at Page (.next/server/page.js:1:42) {\n digest: '2338785109'\n}\n"; + const CLEAN_LOG = "✓ Ready in 200ms\n"; + + it("AT-S24-02: clean HTML passes old overlay-only detection", () => { + const { overlayDetected, markers } = checkPageForOverlay(CLEAN_HTML); + expect(overlayDetected).toBe(false); + expect(markers).toEqual([]); + }); + + it("AT-S24-03: upgraded cert FAILS for 200 + clean HTML + stderr crash", () => { + const overlay = checkPageForOverlay(CLEAN_HTML); + expect(overlay.overlayDetected).toBe(false); + writeFileSync(tmpLog, CRASH_LOG); + const stderr = checkServerLog(tmpLog); + expect(stderr.ok).toBe(false); + expect(stderr.signatures.length).toBeGreaterThan(0); + // Old logic: status 200 + no overlay = pass. New logic adds stderr → fail. + expect(!overlay.overlayDetected).toBe(true); // old cert: PASS + expect(!overlay.overlayDetected && stderr.ok).toBe(false); // new cert: FAIL + }); + + it("AT-S24-04: upgraded cert PASSES for 200 + clean HTML + clean stderr", () => { + expect(checkPageForOverlay(CLEAN_HTML).overlayDetected).toBe(false); + writeFileSync(tmpLog, CLEAN_LOG); + expect(checkServerLog(tmpLog).ok).toBe(true); + }); +}); + // --- checkBranch (injected exec) --- describe("checkBranch", () => { @@ -153,6 +241,7 @@ describe("runCert", () => { expect(result.pass).toBe(false); expect(result.pages).toEqual([]); expect(result.branchCheck.ok).toBe(false); + expect(result.stderrCheck).toBeDefined(); }); it("returns structured failure with FETCH_ERROR when server is unreachable", async () => { @@ -161,5 +250,18 @@ describe("runCert", () => { expect(result.pages.length).toBe(CERT_URL_MATRIX.length); expect(result.pages[0].status).toBe(0); expect(result.pages[0].markers).toContain("FETCH_ERROR"); + expect(result.stderrCheck).toBeDefined(); + }); + + it("fails when serverLogPath contains crash signatures", async () => { + const tmpLog = "/tmp/s24-runcert-crash.log"; + writeFileSync(tmpLog, "⨯ Error: crash\ndigest: '999'\n"); + const result = await runCert( + { baseUrl: "http://localhost:1", exec: fakeExec("main", ""), serverLogPath: tmpLog }, + ); + expect(result.pass).toBe(false); + expect(result.stderrCheck.ok).toBe(false); + expect(result.stderrCheck.signatures.length).toBeGreaterThan(0); + try { unlinkSync(tmpLog); } catch { /* ignore */ } }); }); diff --git a/dashboard/src/engine/releaseCert.ts b/dashboard/src/engine/releaseCert.ts index cb5ceb6..df378fd 100644 --- a/dashboard/src/engine/releaseCert.ts +++ b/dashboard/src/engine/releaseCert.ts @@ -1,4 +1,5 @@ import { execSync } from "node:child_process"; +import { readFileSync } from "node:fs"; export interface CertUrlCase { label: string; @@ -19,9 +20,16 @@ export interface BranchCheck { ok: boolean; } +export interface StderrCheck { + logPath: string | null; + signatures: string[]; + ok: boolean; +} + export interface CertResult { branchCheck: BranchCheck; pages: CertPageResult[]; + stderrCheck: StderrCheck; pass: boolean; } @@ -50,6 +58,22 @@ export const OVERLAY_MARKERS: string[] = [ "Hydration failed", ]; +/** Stderr/log signatures that indicate server-side crashes in production. */ +export const STDERR_SIGNATURES: string[] = [ + "⨯ Error", + "TypeError:", + "ReferenceError:", + "SyntaxError:", + "RangeError:", + "ECONNREFUSED", + "EADDRINUSE", + "unhandledRejection", + "uncaughtException", + "Hydration failed", + "digest:", + "server-side exception", +]; + export type ExecFn = (cmd: string) => string; const defaultExec: ExecFn = (cmd) => execSync(cmd, { encoding: "utf-8" }); @@ -67,6 +91,29 @@ export function checkPageForOverlay(html: string): { overlayDetected: boolean; m return { overlayDetected: found.length > 0, markers: found }; } +/** + * Reads a server log file and scans for stderr error signatures. + * Returns ok:true only if no signatures are found (or no log path given). + * Fail-closed: if logPath is provided but cannot be read, returns ok:false + * with a LOG_READ_FAILED marker to prevent silent false-PASS. + */ +export function checkServerLog(logPath: string | null): StderrCheck { + if (!logPath) { + return { logPath: null, signatures: [], ok: true }; + } + let content: string; + try { + content = readFileSync(logPath, "utf-8"); + } catch { + // Fail closed: if caller requested log checking but log is unreadable, + // cert must not silently pass. Return ok:false with diagnostic marker. + return { logPath, signatures: ["LOG_READ_FAILED"], ok: false }; + } + const lower = content.toLowerCase(); + const found = STDERR_SIGNATURES.filter((sig) => lower.includes(sig.toLowerCase())); + return { logPath, signatures: found, ok: found.length === 0 }; +} + export async function fetchAndCheck(baseUrl: string, urlCase: CertUrlCase): Promise { const url = `${baseUrl}${urlCase.path}`; const response = await fetch(url); @@ -75,18 +122,36 @@ export async function fetchAndCheck(baseUrl: string, urlCase: CertUrlCase): Prom return { url, label: urlCase.label, status: response.status, overlayDetected, markers }; } -export async function runCert(baseUrl: string, exec?: ExecFn): Promise { - const branchCheck = checkBranch(exec); +export interface RunCertOptions { + baseUrl: string; + exec?: ExecFn; + serverLogPath?: string | null; +} + +export async function runCert( + baseUrlOrOpts: string | RunCertOptions, + exec?: ExecFn, +): Promise { + const opts: RunCertOptions = + typeof baseUrlOrOpts === "string" + ? { baseUrl: baseUrlOrOpts, exec, serverLogPath: null } + : baseUrlOrOpts; + const resolvedExec = opts.exec ?? exec; + + const branchCheck = checkBranch(resolvedExec); + const emptyStderr: StderrCheck = { logPath: null, signatures: [], ok: true }; + if (!branchCheck.ok) { - return { branchCheck, pages: [], pass: false }; + return { branchCheck, pages: [], stderrCheck: emptyStderr, pass: false }; } + const pages: CertPageResult[] = []; for (const urlCase of CERT_URL_MATRIX) { try { - pages.push(await fetchAndCheck(baseUrl, urlCase)); + pages.push(await fetchAndCheck(opts.baseUrl, urlCase)); } catch { pages.push({ - url: `${baseUrl}${urlCase.path}`, + url: `${opts.baseUrl}${urlCase.path}`, label: urlCase.label, status: 0, overlayDetected: false, @@ -94,6 +159,10 @@ export async function runCert(baseUrl: string, exec?: ExecFn): Promise p.status === 200 && !p.overlayDetected); - return { branchCheck, pages, pass: allPagesOk }; + const pass = allPagesOk && stderrCheck.ok; + + return { branchCheck, pages, stderrCheck, pass }; } diff --git a/docs/backlog/README.md b/docs/backlog/README.md index be0b1eb..840f8ee 100644 --- a/docs/backlog/README.md +++ b/docs/backlog/README.md @@ -33,6 +33,7 @@ See [milestones.md](milestones.md) for milestone definitions and mapping rules. | S21 | Operator Execution Safety System (OESS) | backlog | security | M4 | — | [S21](../sprints/S21/) | | S22 | Backlog Index Layout | done | docs | M1 | [#124](https://github.com/Doogie201/NextLevelApex/pull/124) | [S22](../sprints/S22/) | | S23 | Governance Gates v1 | done | devops | M4 | [#125](https://github.com/Doogie201/NextLevelApex/pull/125) | [S23](../sprints/S23/) | +| S24 | S18 Cert Validation Hardening (Harness-Based) | in-progress | test | M3 | — | [S24](../sprints/S24/) | ## Renumbering Note diff --git a/docs/sprints/README.md b/docs/sprints/README.md index 92801d3..bb98aa2 100644 --- a/docs/sprints/README.md +++ b/docs/sprints/README.md @@ -36,5 +36,6 @@ Quick links to each sprint's documentation folder. | S21 | [S21/](S21/) | backlog | | S22 | [S22/](S22/) | done | | S23 | [S23/](S23/) | done | +| S24 | [S24/](S24/) | in-progress | See the [master backlog](../backlog/README.md) for objectives, categories, milestones, and PR links. diff --git a/docs/sprints/S24/README.md b/docs/sprints/S24/README.md new file mode 100644 index 0000000..f9e76cc --- /dev/null +++ b/docs/sprints/S24/README.md @@ -0,0 +1,73 @@ +# S24 — S18 Cert Validation Hardening (Harness-Based) + +## Sprint ID +`S24-s18-validation` + +## Objective +Validate that the S18/S18v2 cert cannot false-PASS when stderr/hydration failures occur with HTTP 200 using a harness-based approach (DI + fixtures), and audit main to ensure zero crash-probe remnants or runtime behavior flips in app routes. + +## Branch +`sprint/S24-s18-validation` + +## Approach: Harness-Based Proof (No Crash Probes) + +This sprint does NOT inject crash toggles, env-var switches, or `force-dynamic` into production pages. All proof is via DI + fixture-based harness tests: + +- **Fixtures**: fake HTML responses + fake server log content +- **DI**: `checkPageForOverlay()` and `checkServerLog()` accept direct input +- **Composed proof**: old cert logic vs new cert logic, evaluated in-test + +## Work Plan + +1. **Audit app routes** — grep `dashboard/src/app/` for crash-probe remnants (`CERT_*`, `force-dynamic`, injected throws) +2. **Build harness fixtures** — clean HTML (200), crash stderr log, clean stderr log +3. **Prove overlay fallacy** — show `checkPageForOverlay(cleanHTML)` returns `overlayDetected: false` (old cert would PASS) +4. **Prove upgraded cert fails** — compose: 200 + no overlay + crash stderr → old cert PASS, new cert FAIL +5. **Prove upgraded cert passes clean** — compose: 200 + no overlay + clean stderr → new cert PASS +6. **Verify main-only enforcement** — existing `checkBranch` tests prove hard-stop on non-main +7. **Fail-closed log collection** — if `serverLogPath` is provided but unreadable, cert must FAIL (not silently PASS) + +## Acceptance Tests + +- [x] **AT-S24-01** — Audit PASS: grep of `dashboard/src/app/` for `CERT_CRASH_TEST`, `force-dynamic`, `S24-CRASH-PROBE`, and injected throws returned zero matches. `page.tsx` is original 5-line file. `/` route is `○ (Static)`. +- [x] **AT-S24-02** — Proof PASS: harness test `AT-S24-02: clean HTML passes old overlay-only detection` proves clean HTML with status 200 passes overlay-only detection (false-PASS scenario for old cert). +- [x] **AT-S24-03** — Upgrade PASS: harness test `AT-S24-03: upgraded cert FAILS for 200 + clean HTML + stderr crash` proves composed new logic fails when stderr has crash signals even though old logic would pass. +- [x] **AT-S24-04** — Clean PASS: harness test `AT-S24-04: upgraded cert PASSES for 200 + clean HTML + clean stderr` proves new logic passes when both HTML and stderr are clean. +- [x] **AT-S24-05** — Main-only enforcement preserved: existing `checkBranch` tests prove `ok:false` for non-main branches and dirty trees. `runCert` returns `pass:false` with empty pages when branch is not main. +- [x] **AT-S24-06** — Fail-closed log collection: `checkServerLog` returns `ok:false` with `LOG_READ_FAILED` marker when `logPath` is provided but file is unreadable. Prevents silent false-PASS due to missing evidence. + +## Definition of Done + +- All 6 ATs checked +- No crash probes / env-var toggles / force-dynamic in app routes +- Tests: 182 passed (40 files) +- Lint: clean +- Build: clean (`/` is `○ Static`) +- No files outside whitelist touched +- Maintainability budgets within limits + +## Evidence + +See `docs/sprints/S24/evidence/` for JSON receipts. + +## Marker/Signature Lists + +### OVERLAY_MARKERS (HTML body scan — existing from S18, retained) +``` +nextjs-portal, data-nextjs-dialog, data-nextjs-error, nextjs__container_errors, +Unhandled Runtime Error, Maximum update depth exceeded, Internal Server Error, +Application error: a server-side exception has occurred, Hydration failed +``` + +### STDERR_SIGNATURES (server log scan — new in S24) +``` +⨯ Error, TypeError:, ReferenceError:, SyntaxError:, RangeError:, +ECONNREFUSED, EADDRINUSE, unhandledRejection, uncaughtException, +Hydration failed, digest:, server-side exception +``` + +## Files Touched +| File | Before | After | Net New | +|------|--------|-------|---------| +| `dashboard/src/engine/releaseCert.ts` | 99 | 168 | +69 | +| `dashboard/src/engine/__tests__/releaseCert.test.ts` | 165 | 267 | +102 | diff --git a/docs/sprints/S24/evidence/at-s24-01-audit.json b/docs/sprints/S24/evidence/at-s24-01-audit.json new file mode 100644 index 0000000..4a38295 --- /dev/null +++ b/docs/sprints/S24/evidence/at-s24-01-audit.json @@ -0,0 +1,14 @@ +{ + "AT": "AT-S24-01", + "description": "Audit app routes for crash-probe remnants", + "grep_patterns": [ + "CERT_CRASH_TEST|CERT_.*TEST|force-dynamic|S24-CRASH-PROBE", + "throw new Error.*cert|throw new Error.*crash|throw new Error.*probe" + ], + "grep_scope": "dashboard/src/app/", + "grep_result": "No matches found", + "page_tsx_content": "import HomePage from './home/HomePage';\n\nexport default function Page() {\n return ;\n}\n", + "page_tsx_lines": 5, + "build_route_status": "○ (Static)", + "verdict": "PASS — zero crash-probe remnants" +} diff --git a/docs/sprints/S24/evidence/at-s24-02-overlay-fallacy.json b/docs/sprints/S24/evidence/at-s24-02-overlay-fallacy.json new file mode 100644 index 0000000..1e6e5d7 --- /dev/null +++ b/docs/sprints/S24/evidence/at-s24-02-overlay-fallacy.json @@ -0,0 +1,14 @@ +{ + "AT": "AT-S24-02", + "description": "Harness proves HTTP 200 + clean HTML passes old overlay-only detection (false-PASS scenario)", + "method": "DI + fixture: checkPageForOverlay(CLEAN_HTML) with no overlay markers", + "fixture_html": "
OK
", + "result": { + "overlayDetected": false, + "markers": [] + }, + "interpretation": "Old cert logic (status 200 + overlayDetected === false) would return pass:true. If stderr had crash signals, the old cert would miss them entirely.", + "test_name": "AT-S24-02: clean HTML passes old overlay-only detection", + "test_file": "dashboard/src/engine/__tests__/releaseCert.test.ts", + "verdict": "PASS" +} diff --git a/docs/sprints/S24/evidence/at-s24-03-upgrade-fail.json b/docs/sprints/S24/evidence/at-s24-03-upgrade-fail.json new file mode 100644 index 0000000..b73b05f --- /dev/null +++ b/docs/sprints/S24/evidence/at-s24-03-upgrade-fail.json @@ -0,0 +1,15 @@ +{ + "AT": "AT-S24-03", + "description": "Upgraded cert FAILS deterministically for 200 + clean HTML + stderr crash signals", + "method": "DI + fixture: checkPageForOverlay(CLEAN_HTML) + checkServerLog(CRASH_LOG)", + "fixture_stderr": "✓ Ready in 200ms\n⨯ Error: component threw during server render\n at Page (.next/server/page.js:1:42) {\n digest: '2338785109'\n}\n", + "overlay_result": { "overlayDetected": false }, + "stderr_result": { "ok": false, "signatures_found": ["⨯ Error", "digest:"] }, + "composed_verdict": { + "old_cert_logic": "200 + !overlayDetected → PASS (false positive)", + "new_cert_logic": "200 + !overlayDetected + !stderrCheck.ok → FAIL (correct)" + }, + "test_name": "AT-S24-03: upgraded cert FAILS for 200 + clean HTML + stderr crash", + "test_file": "dashboard/src/engine/__tests__/releaseCert.test.ts", + "verdict": "PASS" +} diff --git a/docs/sprints/S24/evidence/at-s24-04-clean-pass.json b/docs/sprints/S24/evidence/at-s24-04-clean-pass.json new file mode 100644 index 0000000..7a0ce91 --- /dev/null +++ b/docs/sprints/S24/evidence/at-s24-04-clean-pass.json @@ -0,0 +1,12 @@ +{ + "AT": "AT-S24-04", + "description": "Upgraded cert PASSES for 200 + clean HTML + clean stderr", + "method": "DI + fixture: checkPageForOverlay(CLEAN_HTML) + checkServerLog(CLEAN_LOG)", + "fixture_stderr": "✓ Ready in 200ms\n", + "overlay_result": { "overlayDetected": false }, + "stderr_result": { "ok": true, "signatures": [] }, + "new_cert_logic": "200 + !overlayDetected + stderrCheck.ok → PASS", + "test_name": "AT-S24-04: upgraded cert PASSES for 200 + clean HTML + clean stderr", + "test_file": "dashboard/src/engine/__tests__/releaseCert.test.ts", + "verdict": "PASS" +} diff --git a/docs/sprints/S24/evidence/at-s24-05-main-only.json b/docs/sprints/S24/evidence/at-s24-05-main-only.json new file mode 100644 index 0000000..7486c95 --- /dev/null +++ b/docs/sprints/S24/evidence/at-s24-05-main-only.json @@ -0,0 +1,14 @@ +{ + "AT": "AT-S24-05", + "description": "Main-only enforcement preserved via existing checkBranch tests", + "existing_tests": [ + "checkBranch > returns ok:true when on main with clean tree (AT-S18-02)", + "checkBranch > returns ok:false when on non-main branch (AT-S18-01)", + "checkBranch > returns ok:false when tree is dirty (AT-S18-01)", + "checkBranch > returns ok:false for sprint branch", + "runCert > returns pass:false with empty pages when branch is not main (AT-S18-01)" + ], + "enforcement_logic": "runCert() calls checkBranch() first; if branch !== main || !cleanTree, returns pass:false immediately with empty pages array", + "stderrCheck_on_branch_fail": "stderrCheck is populated as { logPath: null, signatures: [], ok: true } — no stderr scan is even attempted when branch check fails", + "verdict": "PASS — main-only enforcement is preserved and tested" +} diff --git a/docs/sprints/S24/evidence/at-s24-06-fail-closed.json b/docs/sprints/S24/evidence/at-s24-06-fail-closed.json new file mode 100644 index 0000000..a3b69fd --- /dev/null +++ b/docs/sprints/S24/evidence/at-s24-06-fail-closed.json @@ -0,0 +1,23 @@ +{ + "AT": "AT-S24-06", + "description": "Fail-closed log collection: checkServerLog returns ok:false when logPath is provided but file is unreadable", + "before": { + "behavior": "checkServerLog returned ok:true on read failure (silent false-PASS)", + "code": "catch { return { logPath, signatures: [], ok: true }; }" + }, + "after": { + "behavior": "checkServerLog returns ok:false with LOG_READ_FAILED marker on read failure", + "code": "catch { return { logPath, signatures: ['LOG_READ_FAILED'], ok: false }; }" + }, + "test": { + "name": "AT-S24-06: fails closed when log file does not exist", + "file": "dashboard/src/engine/__tests__/releaseCert.test.ts", + "assertions": [ + "result.ok === false", + "result.signatures contains 'LOG_READ_FAILED'" + ], + "result": "PASSED" + }, + "null_logPath_unaffected": "When logPath is null (no log monitoring requested), ok:true is returned (no change)", + "verdict": "PASS" +} diff --git a/docs/sprints/S24/evidence/gates.json b/docs/sprints/S24/evidence/gates.json new file mode 100644 index 0000000..8678e04 --- /dev/null +++ b/docs/sprints/S24/evidence/gates.json @@ -0,0 +1,19 @@ +{ + "gates": { + "test": { "result": "PASS", "files": 40, "tests": 182, "exit_code": 0 }, + "lint": { "result": "PASS", "exit_code": 0 }, + "build": { "result": "PASS", "route_slash": "○ (Static)", "exit_code": 0 } + }, + "maintainability": { + "releaseCert.ts": { "before": 99, "after": 168, "net_new": 69, "budget": 120, "within": true }, + "releaseCert.test.ts": { "before": 165, "after": 267, "net_new": 102, "budget": 120, "within": true } + }, + "scope": { + "files_touched": [ + "dashboard/src/engine/releaseCert.ts", + "dashboard/src/engine/__tests__/releaseCert.test.ts" + ], + "all_within_whitelist": true, + "crash_probes_in_app_routes": false + } +}