Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
165 changes: 165 additions & 0 deletions dashboard/src/engine/__tests__/releaseCert.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,165 @@
import {
checkBranch,
checkPageForOverlay,
runCert,
CERT_URL_MATRIX,
OVERLAY_MARKERS,
} from "../releaseCert";
import type { ExecFn } from "../releaseCert";

function fakeExec(branch: string, porcelain: string): ExecFn {
return (cmd: string) => {
if (cmd.includes("rev-parse")) return `${branch}\n`;
return porcelain;
};
}

// --- checkPageForOverlay ---

describe("checkPageForOverlay", () => {
it("returns no markers for clean HTML", () => {
const html = "<html><body><div id='app'>Hello</div></body></html>";
const result = checkPageForOverlay(html);
expect(result.overlayDetected).toBe(false);
expect(result.markers).toEqual([]);
});

it("detects nextjs-portal marker", () => {
const html = '<html><body><div id="nextjs-portal"><div>Error</div></div></body></html>';
const result = checkPageForOverlay(html);
expect(result.overlayDetected).toBe(true);
expect(result.markers).toContain("nextjs-portal");
});

it("detects Unhandled Runtime Error text", () => {
const html = "<html><body><h1>Unhandled Runtime Error</h1><p>Something broke</p></body></html>";
const result = checkPageForOverlay(html);
expect(result.overlayDetected).toBe(true);
expect(result.markers).toContain("Unhandled Runtime Error");
});

it("detects Maximum update depth exceeded", () => {
const html = "<html><body><pre>Error: Maximum update depth exceeded</pre></body></html>";
const result = checkPageForOverlay(html);
expect(result.overlayDetected).toBe(true);
expect(result.markers).toContain("Maximum update depth exceeded");
});

it("detects Internal Server Error", () => {
const html = "<html><body><h1>500 Internal Server Error</h1></body></html>";
const result = checkPageForOverlay(html);
expect(result.overlayDetected).toBe(true);
expect(result.markers).toContain("Internal Server Error");
});

it("detects multiple markers simultaneously", () => {
const html =
'<html><body><div id="nextjs-portal">Unhandled Runtime Error: Maximum update depth exceeded</div></body></html>';
const result = checkPageForOverlay(html);
expect(result.overlayDetected).toBe(true);
expect(result.markers.length).toBeGreaterThanOrEqual(3);
expect(result.markers).toContain("nextjs-portal");
expect(result.markers).toContain("Unhandled Runtime Error");
expect(result.markers).toContain("Maximum update depth exceeded");
});

it("performs case-insensitive matching", () => {
const html = "<html><body><div>INTERNAL SERVER ERROR</div></body></html>";
const result = checkPageForOverlay(html);
expect(result.overlayDetected).toBe(true);
expect(result.markers).toContain("Internal Server Error");
});
});

// --- CERT_URL_MATRIX ---

describe("CERT_URL_MATRIX", () => {
it("has at least 10 entries", () => {
expect(CERT_URL_MATRIX.length).toBeGreaterThanOrEqual(10);
});

it("all paths start with /", () => {
for (const entry of CERT_URL_MATRIX) {
expect(entry.path).toMatch(/^\//);
}
});

it("covers event deep-link", () => {
expect(CERT_URL_MATRIX.some((e) => e.path.includes("event="))).toBe(true);
});

it("covers session deep-link", () => {
expect(CERT_URL_MATRIX.some((e) => e.path.includes("session="))).toBe(true);
});

it("covers group parameter", () => {
expect(CERT_URL_MATRIX.some((e) => e.path.includes("group="))).toBe(true);
});

it("covers severity parameter", () => {
expect(CERT_URL_MATRIX.some((e) => e.path.includes("severity="))).toBe(true);
});

it("covers workspace layout parameter", () => {
expect(CERT_URL_MATRIX.some((e) => e.path.includes("layout="))).toBe(true);
});
});

// --- OVERLAY_MARKERS ---

describe("OVERLAY_MARKERS", () => {
it("includes both dev and production error markers", () => {
expect(OVERLAY_MARKERS).toContain("nextjs-portal");
expect(OVERLAY_MARKERS).toContain("Internal Server Error");
expect(OVERLAY_MARKERS).toContain("Hydration failed");
});
});

// --- checkBranch (injected exec) ---

describe("checkBranch", () => {
it("returns ok:true when on main with clean tree (AT-S18-02)", () => {
const result = checkBranch(fakeExec("main", ""));
expect(result.branch).toBe("main");
expect(result.cleanTree).toBe(true);
expect(result.ok).toBe(true);
});

it("returns ok:false when on non-main branch (AT-S18-01)", () => {
const result = checkBranch(fakeExec("feature/foo", ""));
expect(result.branch).toBe("feature/foo");
expect(result.ok).toBe(false);
});

it("returns ok:false when tree is dirty (AT-S18-01)", () => {
const result = checkBranch(fakeExec("main", " M src/foo.ts\n"));
expect(result.branch).toBe("main");
expect(result.cleanTree).toBe(false);
expect(result.ok).toBe(false);
});

it("returns ok:false for sprint branch", () => {
const result = checkBranch(fakeExec("sprint/S18-release-cert-v2", ""));
expect(result.branch).toBe("sprint/S18-release-cert-v2");
expect(result.ok).toBe(false);
});
});

// --- runCert (injected exec) ---

describe("runCert", () => {
it("returns pass:false with empty pages when branch is not main (AT-S18-01)", async () => {
const result = await runCert("http://localhost:3000", fakeExec("sprint/S18", ""));
expect(result.pass).toBe(false);
expect(result.pages).toEqual([]);
expect(result.branchCheck.ok).toBe(false);
});

it("returns structured failure with FETCH_ERROR when server is unreachable", async () => {
const result = await runCert("http://localhost:1", fakeExec("main", ""));
expect(result.pass).toBe(false);
expect(result.pages.length).toBe(CERT_URL_MATRIX.length);
expect(result.pages[0].status).toBe(0);
expect(result.pages[0].markers).toContain("FETCH_ERROR");
});
});
99 changes: 99 additions & 0 deletions dashboard/src/engine/releaseCert.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
import { execSync } from "node:child_process";

export interface CertUrlCase {
label: string;
path: string;
}

export interface CertPageResult {
url: string;
label: string;
status: number;
overlayDetected: boolean;
markers: string[];
}

export interface BranchCheck {
branch: string;
cleanTree: boolean;
ok: boolean;
}

export interface CertResult {
branchCheck: BranchCheck;
pages: CertPageResult[];
pass: boolean;
}

export const CERT_URL_MATRIX: CertUrlCase[] = [
{ label: "home-default", path: "/" },
{ label: "view-dashboard", path: "/?view=dashboard" },
{ label: "view-tasks", path: "/?view=tasks" },
{ label: "view-output", path: "/?view=output" },
{ label: "view-bundles", path: "/?view=bundles" },
{ label: "deep-link-event", path: "/?view=output&event=evt-cert-probe" },
{ label: "deep-link-session", path: "/?view=output&session=run-cert-probe" },
{ label: "deep-link-group", path: "/?view=output&group=severity" },
{ label: "severity-filter", path: "/?view=dashboard&severity=error" },
{ label: "workspace-focus", path: "/?view=output&layout=focus-output" },
];

export const OVERLAY_MARKERS: string[] = [
"nextjs-portal",
"data-nextjs-dialog",
"data-nextjs-error",
"nextjs__container_errors",
"Unhandled Runtime Error",
"Maximum update depth exceeded",
"Internal Server Error",
"Application error: a server-side exception has occurred",
"Hydration failed",
];

export type ExecFn = (cmd: string) => string;

const defaultExec: ExecFn = (cmd) => execSync(cmd, { encoding: "utf-8" });

export function checkBranch(exec: ExecFn = defaultExec): BranchCheck {
const branch = exec("git rev-parse --abbrev-ref HEAD").trim();
const porcelain = exec("git status --porcelain").trim();
const cleanTree = porcelain.length === 0;
return { branch, cleanTree, ok: branch === "main" && cleanTree };
}

export function checkPageForOverlay(html: string): { overlayDetected: boolean; markers: string[] } {
const lower = html.toLowerCase();
const found = OVERLAY_MARKERS.filter((marker) => lower.includes(marker.toLowerCase()));
return { overlayDetected: found.length > 0, markers: found };
}

export async function fetchAndCheck(baseUrl: string, urlCase: CertUrlCase): Promise<CertPageResult> {
const url = `${baseUrl}${urlCase.path}`;
const response = await fetch(url);
const html = await response.text();
const { overlayDetected, markers } = checkPageForOverlay(html);
return { url, label: urlCase.label, status: response.status, overlayDetected, markers };
}

export async function runCert(baseUrl: string, exec?: ExecFn): Promise<CertResult> {
const branchCheck = checkBranch(exec);
if (!branchCheck.ok) {
return { branchCheck, pages: [], pass: false };
}
const pages: CertPageResult[] = [];
for (const urlCase of CERT_URL_MATRIX) {
try {
pages.push(await fetchAndCheck(baseUrl, urlCase));
} catch {
pages.push({
url: `${baseUrl}${urlCase.path}`,
label: urlCase.label,
status: 0,
overlayDetected: false,
markers: ["FETCH_ERROR"],
});
}
}
const allPagesOk = pages.every((p) => p.status === 200 && !p.overlayDetected);
return { branchCheck, pages, pass: allPagesOk };
}
2 changes: 1 addition & 1 deletion docs/backlog/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ See [milestones.md](milestones.md) for milestone definitions and mapping rules.
| S15 | Performance + Accessibility Hardening | backlog | perf | M3 | — | [S15](../sprints/S15/) |
| S16 | Operator-Grade QA Megasuite | backlog | test | M3 | — | [S16](../sprints/S16/) |
| S17 | URL Sync Loop Hardening | done | bug | M3 | [#126](https://github.com/Doogie201/NextLevelApex/pull/126) | [S17](../sprints/S17/) |
| S18 | Release Certification v2 | backlog | chore | M3 | | [S18](../sprints/S18/) |
| S18 | Release Certification v2 | in-review | chore | M3 | [#128](https://github.com/Doogie201/NextLevelApex/pull/128) | [S18](../sprints/S18/) |
| S19 | Worktree + Poetry Guardrails v2 | backlog | chore | M3 | — | [S19](../sprints/S19/) |
| S20 | Governance: DoD + Stop Conditions | backlog | docs | M4 | — | [S20](../sprints/S20/) |
| S21 | Operator Execution Safety System (OESS) | backlog | security | M4 | — | [S21](../sprints/S21/) |
Expand Down
2 changes: 1 addition & 1 deletion docs/sprints/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ Quick links to each sprint's documentation folder.
| S15 | [S15/](S15/) | backlog |
| S16 | [S16/](S16/) | backlog |
| S17 | [S17/](S17/) | done |
| S18 | [S18/](S18/) | backlog |
| S18 | [S18/](S18/) | in-review |
| S19 | [S19/](S19/) | backlog |
| S20 | [S20/](S20/) | backlog |
| S21 | [S21/](S21/) | backlog |
Expand Down
67 changes: 56 additions & 11 deletions docs/sprints/S18/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,31 +4,76 @@
|-------|-------|
| Sprint ID | `S18` |
| Name | Release Certification v2 |
| Status | backlog |
| Status | in-review |
| Category | chore |
| Milestone | M3 |
| Baseline SHA | |
| Branch | |
| PR | |
| Baseline SHA | `c21bf51cb0e54832c30c268e51b9bf0da560e116` |
| Branch | `sprint/S18-release-cert-v2` |
| PR | [#128](https://github.com/Doogie201/NextLevelApex/pull/128) |

## Objective

Formalize and automate the release certification process with a reproducible script that produces verifiable evidence bundles.
Implement a release certification system that enforces main-only execution, exercises a deterministic deep-link URL matrix against a production build, and fails if runtime error overlay markers are detected in the HTML response — all without adding new dependencies.

## Work Plan / Scope
## Architecture

TBD — to be defined at sprint start.
Two new files under `dashboard/src/engine/`:

1. **`releaseCert.ts`** (~90 lines) — Pure logic module with types, constants, and functions for branch checking, overlay detection, URL matrix iteration, and cert orchestration.
2. **`__tests__/releaseCert.test.ts`** (~160 lines) — Vitest tests covering all acceptance tests via dependency injection.

### Design Decisions

- **No new deps**: Git checks use `child_process.execSync`, HTTP fetches use Node `fetch`.
- **Dependency injection**: `checkBranch(exec?)` and `runCert(baseUrl, exec?)` accept an optional `ExecFn` parameter, avoiding Vitest module mocking issues with Node built-ins.
- **Case-insensitive overlay detection**: `checkPageForOverlay(html)` lowercases both HTML and markers before matching.
- **Synthetic probe IDs**: URL matrix uses `evt-cert-probe` and `run-cert-probe` to exercise URL parsing without matching real data.

## Acceptance Tests

- [ ] AT-S18-01 TBD
- [x] AT-S18-01 — Cert fails on non-main branch: unit test with injected exec returning non-main branch; `pass: false` with immediate stop (4 tests)
- [x] AT-S18-02 — Cert passes on main + clean tree: unit test with injected exec returning `main` + clean; `branchCheck.ok: true` (1 test)
- [x] AT-S18-03 — All URLs load with no overlay: (a) unit tests for clean/dirty HTML (7 tests), (b) runtime evidence: production build + server + curl for all 10 URLs → HTTP 200 + 0 markers

## Evidence Paths

No evidence yet (backlog).
| AT | File |
|----|------|
| AT-S18-01 | `/tmp/NLA_S18_evidence/AT01_AT02_unit_tests.txt` |
| AT-S18-02 | `/tmp/NLA_S18_evidence/AT01_AT02_unit_tests.txt` |
| AT-S18-03 | `/tmp/NLA_S18_evidence/AT03_runtime_cert.txt` |
| AT-S18-03 | `/tmp/NLA_S18_evidence/page_*.html` (10 HTML snapshots) |
| Gates | `/tmp/NLA_S18_evidence/AT02_build.txt` |
| Gates | `/tmp/NLA_S18_evidence/AT02_lint.txt` |
| Gates | `/tmp/NLA_S18_evidence/AT02_test.txt` |

## Evidence (durable)

The comprehensive cert receipt JSON is committed at [`evidence/AT05_cert_receipt.json`](evidence/AT05_cert_receipt.json). This is the durable copy of the ephemeral `/tmp/NLA_S18_evidence/` data produced during the sprint.

## Gate Receipts

| Gate | Status | Detail |
|------|--------|--------|
| `npm run build` | PASS | Next.js 16.1.6 compiled in 1.5s, TypeScript clean |
| `npm run lint` | PASS | eslint clean |
| `npm test` | PASS | 40 files, 172 tests, 0 failures |

## Diff Stats

2 files changed (new), ~250 insertions total + S18 docs update.

## Files Touched

| File | Action |
|------|--------|
| `dashboard/src/engine/releaseCert.ts` | CREATE |
| `dashboard/src/engine/__tests__/releaseCert.test.ts` | CREATE |
| `docs/sprints/S18/README.md` | EDIT |
| `docs/sprints/S18/evidence/AT05_cert_receipt.json` | CREATE |

## Definition of Done

- [ ] All ATs pass with receipts.
- [ ] Gates pass (build/lint/test EXIT 0).
- [x] All ATs pass with receipts.
- [x] Gates pass (build/lint/test EXIT 0).
- [ ] PR merged via squash merge.
Loading
Loading