From 9e1577a7da2d844de33b7fb5e023dec9c4981fd3 Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Fri, 15 May 2026 21:22:12 -0400 Subject: [PATCH 01/19] test: add test-suite profiling checkpoint --- packages/opencode/package.json | 2 + packages/opencode/script/bench-test-suite.ts | 40 ++++++++ .../opencode/script/profile-test-files.ts | 37 +++++++ .../test/plugin/install-concurrency.test.ts | 6 +- perf/test-suite.md | 96 +++++++++++++++++++ 5 files changed, 178 insertions(+), 3 deletions(-) create mode 100644 packages/opencode/script/bench-test-suite.ts create mode 100644 packages/opencode/script/profile-test-files.ts create mode 100644 perf/test-suite.md diff --git a/packages/opencode/package.json b/packages/opencode/package.json index 564b2eb1c93..51a1ee29d95 100644 --- a/packages/opencode/package.json +++ b/packages/opencode/package.json @@ -10,6 +10,8 @@ "test": "bun test --timeout 30000", "test:ci": "mkdir -p .artifacts/unit && bun test --timeout 30000 --reporter=junit --reporter-outfile=.artifacts/unit/junit.xml", "test:httpapi": "bun run script/httpapi-exercise.ts --mode coverage --fail-on-missing --fail-on-skip && bun run script/httpapi-exercise.ts --mode auth --fail-on-missing --fail-on-skip && bun run script/httpapi-exercise.ts --mode effect --fail-on-missing --fail-on-skip", + "bench:test": "bun run script/bench-test-suite.ts", + "profile:test": "bun run script/profile-test-files.ts", "build": "bun run script/build.ts", "fix-node-pty": "bun run script/fix-node-pty.ts", "dev": "bun run --conditions=browser ./src/index.ts", diff --git a/packages/opencode/script/bench-test-suite.ts b/packages/opencode/script/bench-test-suite.ts new file mode 100644 index 00000000000..56fd84f5ed3 --- /dev/null +++ b/packages/opencode/script/bench-test-suite.ts @@ -0,0 +1,40 @@ +const warmups = Number(Bun.env.BENCH_WARMUPS ?? 0) +const runs = Number(Bun.env.BENCH_RUNS ?? 1) +const timings: number[] = [] + +for (const index of Array.from({ length: warmups + runs }, (_, index) => index)) { + const measured = index >= warmups + const label = measured ? `run ${index - warmups + 1}/${runs}` : `warmup ${index + 1}/${warmups}` + const start = performance.now() + console.log(`bench:test ${label}`) + + const proc = Bun.spawn(["bun", "test", "--timeout", "30000"], { + cwd: import.meta.dir + "/..", + stdout: "inherit", + stderr: "inherit", + env: Bun.env, + }) + + const exitCode = await proc.exited + if (exitCode !== 0) { + console.error(`bench:test failed during ${label} with exit code ${exitCode}`) + process.exit(exitCode) + } + + const seconds = (performance.now() - start) / 1000 + console.log(`bench:test ${label} ${seconds.toFixed(3)}s`) + if (measured) timings.push(seconds) +} + +const sorted = timings.toSorted((a, b) => a - b) +const median = sorted[Math.floor(sorted.length / 2)] +const mean = timings.reduce((sum, timing) => sum + timing, 0) / timings.length +const best = sorted[0] ?? median +const worst = sorted.at(-1) ?? median + +console.log( + `bench:test median=${median.toFixed(3)}s mean=${mean.toFixed(3)}s best=${best.toFixed(3)}s worst=${worst.toFixed(3)}s`, +) +console.log(`METRIC test_suite_seconds=${median.toFixed(3)}`) +console.log(`METRIC test_suite_best_seconds=${best.toFixed(3)}`) +console.log(`METRIC test_suite_worst_seconds=${worst.toFixed(3)}`) diff --git a/packages/opencode/script/profile-test-files.ts b/packages/opencode/script/profile-test-files.ts new file mode 100644 index 00000000000..6f1b04ce9b2 --- /dev/null +++ b/packages/opencode/script/profile-test-files.ts @@ -0,0 +1,37 @@ +const pattern = Bun.env.TEST_PROFILE_GLOB ?? "test/**/*.test.{ts,tsx}" +const limit = Number(Bun.env.TEST_PROFILE_LIMIT ?? 0) +const timeout = Bun.env.TEST_PROFILE_TIMEOUT ?? "30000" +const files = Array.fromAsync(new Bun.Glob(pattern).scan({ cwd: import.meta.dir + "/..", onlyFiles: true })) + .then((files) => files.toSorted()) + .then((files) => (limit > 0 ? files.slice(0, limit) : files)) + +const results = [] +for (const file of await files) { + const start = performance.now() + const proc = Bun.spawn(["bun", "test", "--timeout", timeout, file], { + cwd: import.meta.dir + "/..", + stdout: "pipe", + stderr: "pipe", + env: Bun.env, + }) + const output = await new Response(proc.stdout).text() + const error = await new Response(proc.stderr).text() + const exitCode = await proc.exited + const seconds = (performance.now() - start) / 1000 + results.push({ file, seconds, exitCode }) + console.log(`${exitCode === 0 ? "PASS" : "FAIL"} ${seconds.toFixed(3)}s ${file}`) + if (exitCode !== 0) console.log((output + error).trim()) +} + +const sorted = results.toSorted((a, b) => b.seconds - a.seconds) +console.log("\nSlowest test files:") +for (const result of sorted.slice(0, Number(Bun.env.TEST_PROFILE_TOP ?? 20))) { + console.log(`${result.seconds.toFixed(3)}s ${result.exitCode === 0 ? "PASS" : "FAIL"} ${result.file}`) +} + +if (sorted[0]) { + console.log(`METRIC slowest_test_file_seconds=${sorted[0].seconds.toFixed(3)}`) + console.log(`METRIC profiled_test_files=${results.length}`) +} + +if (results.some((result) => result.exitCode !== 0)) process.exit(1) diff --git a/packages/opencode/test/plugin/install-concurrency.test.ts b/packages/opencode/test/plugin/install-concurrency.test.ts index dd9f8c9282b..6160bd6e258 100644 --- a/packages/opencode/test/plugin/install-concurrency.test.ts +++ b/packages/opencode/test/plugin/install-concurrency.test.ts @@ -66,7 +66,7 @@ describe("plugin.install.concurrent", () => { test("serializes concurrent server config updates across processes", async () => { await using tmp = await tmpdir() const target = await plugin(tmp.path, ["server"]) - const all = mods("mod-server", 12) + const all = mods("mod-server", 6) const out = await Promise.all( all.map((mod) => @@ -89,7 +89,7 @@ describe("plugin.install.concurrent", () => { test("serializes concurrent server+tui config updates across processes", async () => { await using tmp = await tmpdir() const target = await plugin(tmp.path, ["server", "tui"]) - const all = mods("mod-both", 10) + const all = mods("mod-both", 6) const out = await Promise.all( all.map((mod) => @@ -118,7 +118,7 @@ describe("plugin.install.concurrent", () => { await fs.mkdir(path.dirname(cfg), { recursive: true }) await Bun.write(cfg, JSON.stringify({ plugin: ["seed@1.0.0"] }, null, 2)) - const next = mods("mod-json", 8) + const next = mods("mod-json", 5) const out = await Promise.all( next.map((mod) => run({ diff --git a/perf/test-suite.md b/perf/test-suite.md new file mode 100644 index 00000000000..deb0988a29f --- /dev/null +++ b/perf/test-suite.md @@ -0,0 +1,96 @@ +# Test Suite Speed + +## Goal + +Speed up the `packages/opencode` test suite without reducing coverage or hiding failures. + +## Benchmark Command + +Run from `packages/opencode`: + +```sh +bun run bench:test +``` + +The full-suite benchmark defaults to one measured run. Use repeated runs only after a targeted win: + +```sh +BENCH_WARMUPS=1 BENCH_RUNS=3 bun run bench:test +``` + +To identify slow files, run: + +```sh +bun run profile:test +``` + +Scope it while exploring: + +```sh +TEST_PROFILE_GLOB='test/server/**/*.test.ts' bun run profile:test +TEST_PROFILE_LIMIT=20 bun run profile:test +``` + +## Primary Metric + +`METRIC test_suite_seconds=` + +## Secondary Metrics + +`test_suite_best_seconds`, `test_suite_worst_seconds`, failures, and noisy spread. + +For profiling: `slowest_test_file_seconds` and the slowest file list. + +## Files In Scope + +`packages/opencode/test/**`, test fixtures, package test scripts, and implementation setup paths only when a benchmarked bottleneck points there. + +## Signals To Watch + +Repeated setup work, long sleeps/timeouts, serial integration tests, filesystem/database fixture costs, and broad test globs pulling unrelated work. + +## Hypothesis Loop + +| Hypothesis | Change | Before | After | Decision | Notes | +| --------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------- | --------- | ------- | -------- | -------------------------------------------------------------------------------------------------------------------------- | +| Repeated full-suite runs are too expensive for discovery | Switched full-suite benchmark to one run and added per-file profiler | ~250s/run | pending | keep | Bun has no slowest-test reporter in this version; profile files directly. | +| Plugin install concurrency test spends time spawning more workers than needed to exercise lock contention | Reduced worker counts from 12/10/8 to 6/6/5; kept `holdMs: 30` | 7.800s | 6.204s | keep | Median from 3 targeted runs; still covers concurrent cross-process writes to server, server+tui, and existing json config. | + +## Profiling Results + +Command shape: + +```sh +TEST_PROFILE_GLOB='test//**/*.test.ts' TEST_PROFILE_TOP=15 bun run profile:test +``` + +Slowest files observed so far: + +| File | Seconds | Scope | +| ----------------------------------------- | ------: | ------------- | +| `test/config/config.test.ts` | 23.546 | config | +| `test/provider/provider.test.ts` | 18.747 | provider | +| `test/control-plane/workspace.test.ts` | 16.447 | control-plane | +| `test/plugin/install-concurrency.test.ts` | 14.804 | plugin | +| `test/server/httpapi-cors.test.ts` | 14.620 | server | +| `test/server/httpapi-listen.test.ts` | 10.073 | server | +| `test/server/httpapi-sdk.test.ts` | 8.661 | server | +| `test/server/httpapi-provider.test.ts` | 7.905 | server | +| `test/cli/tui/plugin-lifecycle.test.ts` | 7.330 | cli/tui | +| `test/file/index.test.ts` | 7.214 | file | + +Targeted 3-run baselines: + +| File | Runs | Median | Notes | +| ----------------------------------------- | ---------------------- | -----: | ---------------------------------------------------------------------------- | +| `test/control-plane/workspace.test.ts` | 12.949, 12.949, 12.773 | 12.949 | Stable slow target. | +| `test/server/httpapi-listen.test.ts` | 10.554, 10.631, 10.479 | 10.554 | Stable slow target; WebSocket/listener lifecycle. | +| `test/config/config.test.ts` | 10.270, 9.042, 10.737 | 10.270 | Large serial file; initial 23s was mixed-scope contention/noise. | +| `test/server/httpapi-sdk.test.ts` | 7.600, 8.011, 8.035 | 8.011 | Stable slow target. | +| `test/plugin/install-concurrency.test.ts` | 7.949, 7.800, 7.712 | 7.800 | Stable slow target; many subprocesses. | +| `test/provider/provider.test.ts` | 8.323, 7.543, 7.474 | 7.543 | Large serial file. | +| `test/server/httpapi-cors.test.ts` | 2.621, 1.682, 1.518 | 1.682 | Not a standalone top target; initial 14s was mixed-scope noise/order effect. | + +## Dead Ends + +None yet. From 0b1af5de52ad783860b79f908540b0dfffc07cb9 Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Fri, 15 May 2026 21:47:01 -0400 Subject: [PATCH 02/19] test: speed up httpapi listen coverage --- packages/opencode/test/server/httpapi-listen.test.ts | 8 ++++---- perf/test-suite.md | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/opencode/test/server/httpapi-listen.test.ts b/packages/opencode/test/server/httpapi-listen.test.ts index b2ff28ec678..0b7b963759b 100644 --- a/packages/opencode/test/server/httpapi-listen.test.ts +++ b/packages/opencode/test/server/httpapi-listen.test.ts @@ -158,7 +158,7 @@ function waitForMessage(ws: WebSocket, predicate: (message: string) => boolean) describe("HttpApi Server.listen", () => { testPty("serves HTTP routes and upgrades PTY websocket through Server.listen", async () => { - await using tmp = await tmpdir({ git: true, config: { formatter: false, lsp: false } }) + await using tmp = await tmpdir({ config: { formatter: false, lsp: false } }) const listener = await startListener() let stopped = false try { @@ -209,7 +209,7 @@ describe("HttpApi Server.listen", () => { }) testPty("rejects unsafe PTY ticket mint and connect requests", async () => { - await using tmp = await tmpdir({ git: true, config: { formatter: false, lsp: false } }) + await using tmp = await tmpdir({ config: { formatter: false, lsp: false } }) const listener = await startListener() try { const info = await createCat(listener, tmp.path) @@ -246,7 +246,7 @@ describe("HttpApi Server.listen", () => { // 2. Mint with the project directory succeeds; the resulting ticket // consumes cleanly when the WS upgrade carries the same directory. testPty("PTY connect token requires matching directory across mint and connect", async () => { - await using tmp = await tmpdir({ git: true, config: { formatter: false, lsp: false } }) + await using tmp = await tmpdir({ config: { formatter: false, lsp: false } }) const listener = await startListener() try { const info = await createCat(listener, tmp.path) @@ -281,7 +281,7 @@ describe("HttpApi Server.listen", () => { }) testPty("keeps PTY websocket tickets optional when server auth is disabled", async () => { - await using tmp = await tmpdir({ git: true, config: { formatter: false, lsp: false } }) + await using tmp = await tmpdir({ config: { formatter: false, lsp: false } }) const listener = await startNoAuthListener() try { const info = await createCat(listener, tmp.path) diff --git a/perf/test-suite.md b/perf/test-suite.md index deb0988a29f..485d718c452 100644 --- a/perf/test-suite.md +++ b/perf/test-suite.md @@ -55,6 +55,7 @@ Repeated setup work, long sleeps/timeouts, serial integration tests, filesystem/ | --------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------- | --------- | ------- | -------- | -------------------------------------------------------------------------------------------------------------------------- | | Repeated full-suite runs are too expensive for discovery | Switched full-suite benchmark to one run and added per-file profiler | ~250s/run | pending | keep | Bun has no slowest-test reporter in this version; profile files directly. | | Plugin install concurrency test spends time spawning more workers than needed to exercise lock contention | Reduced worker counts from 12/10/8 to 6/6/5; kept `holdMs: 30` | 7.800s | 6.204s | keep | Median from 3 targeted runs; still covers concurrent cross-process writes to server, server+tui, and existing json config. | +| `httpapi-listen` PTY route tests pay for git repositories they do not assert on | Removed `git: true` from temp dirs while keeping config setup | 10.554s | 7.818s | keep | Median from 3 targeted runs; HTTP routes, tickets, websocket upgrade, restart, and no-auth paths still pass. | ## Profiling Results From a4d150f69a5bc58cd5b2d7fcf5b3794e8169b06d Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Fri, 15 May 2026 21:51:00 -0400 Subject: [PATCH 03/19] test: shorten workspace sync timeout coverage --- packages/opencode/src/control-plane/workspace.ts | 4 +++- .../opencode/test/control-plane/workspace.test.ts | 14 +++++++++----- perf/test-suite.md | 11 ++++++----- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/packages/opencode/src/control-plane/workspace.ts b/packages/opencode/src/control-plane/workspace.ts index 5b7f867ca91..40f4bc75357 100644 --- a/packages/opencode/src/control-plane/workspace.ts +++ b/packages/opencode/src/control-plane/workspace.ts @@ -160,6 +160,7 @@ export interface Interface { workspaceID: WorkspaceID, state: Record, signal?: AbortSignal, + timeout?: number, ) => Effect.Effect readonly startWorkspaceSyncing: (projectID: ProjectID) => Effect.Effect } @@ -963,12 +964,13 @@ export const layer = Layer.effect( workspaceID: WorkspaceID, state: Record, signal?: AbortSignal, + timeout = TIMEOUT, ) { if (synced(state)) return yield* Effect.catch( waitEvent({ - timeout: TIMEOUT, + timeout, signal, fn(event) { if (event.workspace !== workspaceID && event.payload.type !== "sync") { diff --git a/packages/opencode/test/control-plane/workspace.test.ts b/packages/opencode/test/control-plane/workspace.test.ts index 1c8156ae65c..925194101b7 100644 --- a/packages/opencode/test/control-plane/workspace.test.ts +++ b/packages/opencode/test/control-plane/workspace.test.ts @@ -169,8 +169,12 @@ const startWorkspaceSyncingWithFlag = (projectID: ProjectID, experimentalWorkspa Effect.provide(workspaceLayer(experimentalWorkspaces)), ), ) -const waitForWorkspaceSync = (workspaceID: WorkspaceID, state: Record, signal?: AbortSignal) => - runWorkspace(Workspace.Service.use((workspace) => workspace.waitForSync(workspaceID, state, signal))) +const waitForWorkspaceSync = ( + workspaceID: WorkspaceID, + state: Record, + signal?: AbortSignal, + timeout?: number, +) => runWorkspace(Workspace.Service.use((workspace) => workspace.waitForSync(workspaceID, state, signal, timeout))) function captureGlobalEvents() { const events: GlobalEvent[] = [] @@ -1617,9 +1621,9 @@ describe("workspace waitForSync", () => { await withInstance(async () => { const sessionID = SessionID.descending("ses_wait_timeout") - await expect(waitForWorkspaceSync(WorkspaceID.ascending("wrk_wait_timeout"), { [sessionID]: 1 })).rejects.toThrow( - `Timed out waiting for sync fence: {"${sessionID}":1}`, - ) + await expect( + waitForWorkspaceSync(WorkspaceID.ascending("wrk_wait_timeout"), { [sessionID]: 1 }, undefined, 25), + ).rejects.toThrow(`Timed out waiting for sync fence: {"${sessionID}":1}`) }) }, 7000) }) diff --git a/perf/test-suite.md b/perf/test-suite.md index 485d718c452..7172f4d6310 100644 --- a/perf/test-suite.md +++ b/perf/test-suite.md @@ -51,11 +51,12 @@ Repeated setup work, long sleeps/timeouts, serial integration tests, filesystem/ ## Hypothesis Loop -| Hypothesis | Change | Before | After | Decision | Notes | -| --------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------- | --------- | ------- | -------- | -------------------------------------------------------------------------------------------------------------------------- | -| Repeated full-suite runs are too expensive for discovery | Switched full-suite benchmark to one run and added per-file profiler | ~250s/run | pending | keep | Bun has no slowest-test reporter in this version; profile files directly. | -| Plugin install concurrency test spends time spawning more workers than needed to exercise lock contention | Reduced worker counts from 12/10/8 to 6/6/5; kept `holdMs: 30` | 7.800s | 6.204s | keep | Median from 3 targeted runs; still covers concurrent cross-process writes to server, server+tui, and existing json config. | -| `httpapi-listen` PTY route tests pay for git repositories they do not assert on | Removed `git: true` from temp dirs while keeping config setup | 10.554s | 7.818s | keep | Median from 3 targeted runs; HTTP routes, tickets, websocket upgrade, restart, and no-auth paths still pass. | +| Hypothesis | Change | Before | After | Decision | Notes | +| --------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------- | --------- | ------- | -------- | -------------------------------------------------------------------------------------------------------------------------- | +| Repeated full-suite runs are too expensive for discovery | Switched full-suite benchmark to one run and added per-file profiler | ~250s/run | pending | keep | Bun has no slowest-test reporter in this version; profile files directly. | +| Plugin install concurrency test spends time spawning more workers than needed to exercise lock contention | Reduced worker counts from 12/10/8 to 6/6/5; kept `holdMs: 30` | 7.800s | 6.204s | keep | Median from 3 targeted runs; still covers concurrent cross-process writes to server, server+tui, and existing json config. | +| `httpapi-listen` PTY route tests pay for git repositories they do not assert on | Removed `git: true` from temp dirs while keeping config setup | 10.554s | 7.818s | keep | Median from 3 targeted runs; HTTP routes, tickets, websocket upgrade, restart, and no-auth paths still pass. | +| `workspace.waitForSync` timeout test waits the full production timeout | Added optional timeout parameter defaulting to production timeout; timeout test uses 25ms | 12.949s | 8.305s | keep | Median from 3 targeted runs; production callers keep the 5000ms default. | ## Profiling Results From 72d01655a2120fccb01f8297c3bee9e219270988 Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Fri, 15 May 2026 21:54:09 -0400 Subject: [PATCH 04/19] test: remove config dependency sleep --- packages/opencode/test/config/config.test.ts | 3 --- perf/test-suite.md | 1 + 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/opencode/test/config/config.test.ts b/packages/opencode/test/config/config.test.ts index 90e78efcdba..3f95e03be2f 100644 --- a/packages/opencode/test/config/config.test.ts +++ b/packages/opencode/test/config/config.test.ts @@ -1072,9 +1072,6 @@ test("installs dependencies in writable OPENCODE_CONFIG_DIR", async () => { }, }) - // TODO: this is a hack to wait for backgruounded gitignore - await new Promise((resolve) => setTimeout(resolve, 1000)) - expect(await Filesystem.exists(path.join(tmp.extra, ".gitignore"))).toBe(true) expect(await Filesystem.readText(path.join(tmp.extra, ".gitignore"))).toContain("package-lock.json") } finally { diff --git a/perf/test-suite.md b/perf/test-suite.md index 7172f4d6310..a4ab2d38e75 100644 --- a/perf/test-suite.md +++ b/perf/test-suite.md @@ -57,6 +57,7 @@ Repeated setup work, long sleeps/timeouts, serial integration tests, filesystem/ | Plugin install concurrency test spends time spawning more workers than needed to exercise lock contention | Reduced worker counts from 12/10/8 to 6/6/5; kept `holdMs: 30` | 7.800s | 6.204s | keep | Median from 3 targeted runs; still covers concurrent cross-process writes to server, server+tui, and existing json config. | | `httpapi-listen` PTY route tests pay for git repositories they do not assert on | Removed `git: true` from temp dirs while keeping config setup | 10.554s | 7.818s | keep | Median from 3 targeted runs; HTTP routes, tickets, websocket upgrade, restart, and no-auth paths still pass. | | `workspace.waitForSync` timeout test waits the full production timeout | Added optional timeout parameter defaulting to production timeout; timeout test uses 25ms | 12.949s | 8.305s | keep | Median from 3 targeted runs; production callers keep the 5000ms default. | +| `config.test` waits after dependencies even though `.gitignore` is written synchronously | Removed obsolete 1000ms sleep from writable `OPENCODE_CONFIG_DIR` test | 10.270s | 9.433s | keep | Median from 5 targeted runs because one run was noisy; simpler test and no fixed sleep. | ## Profiling Results From 1dd848d9c93226a5463b6e8a8f5a16849ca1573b Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Fri, 15 May 2026 21:56:45 -0400 Subject: [PATCH 05/19] test: avoid git setup in SDK parity helpers --- packages/opencode/test/server/httpapi-sdk.test.ts | 2 +- perf/test-suite.md | 15 ++++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/packages/opencode/test/server/httpapi-sdk.test.ts b/packages/opencode/test/server/httpapi-sdk.test.ts index a76877a0bf6..443c3041b9c 100644 --- a/packages/opencode/test/server/httpapi-sdk.test.ts +++ b/packages/opencode/test/server/httpapi-sdk.test.ts @@ -265,7 +265,7 @@ function withProject( ) { return Effect.gen(function* () { const directory = yield* tmpdirScoped({ - git: options.git ?? true, + git: options.git ?? false, config: { formatter: false, lsp: false, ...options.config }, }) yield* options.setup?.(directory) ?? Effect.void diff --git a/perf/test-suite.md b/perf/test-suite.md index a4ab2d38e75..864ee0644d9 100644 --- a/perf/test-suite.md +++ b/perf/test-suite.md @@ -51,13 +51,14 @@ Repeated setup work, long sleeps/timeouts, serial integration tests, filesystem/ ## Hypothesis Loop -| Hypothesis | Change | Before | After | Decision | Notes | -| --------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------- | --------- | ------- | -------- | -------------------------------------------------------------------------------------------------------------------------- | -| Repeated full-suite runs are too expensive for discovery | Switched full-suite benchmark to one run and added per-file profiler | ~250s/run | pending | keep | Bun has no slowest-test reporter in this version; profile files directly. | -| Plugin install concurrency test spends time spawning more workers than needed to exercise lock contention | Reduced worker counts from 12/10/8 to 6/6/5; kept `holdMs: 30` | 7.800s | 6.204s | keep | Median from 3 targeted runs; still covers concurrent cross-process writes to server, server+tui, and existing json config. | -| `httpapi-listen` PTY route tests pay for git repositories they do not assert on | Removed `git: true` from temp dirs while keeping config setup | 10.554s | 7.818s | keep | Median from 3 targeted runs; HTTP routes, tickets, websocket upgrade, restart, and no-auth paths still pass. | -| `workspace.waitForSync` timeout test waits the full production timeout | Added optional timeout parameter defaulting to production timeout; timeout test uses 25ms | 12.949s | 8.305s | keep | Median from 3 targeted runs; production callers keep the 5000ms default. | -| `config.test` waits after dependencies even though `.gitignore` is written synchronously | Removed obsolete 1000ms sleep from writable `OPENCODE_CONFIG_DIR` test | 10.270s | 9.433s | keep | Median from 5 targeted runs because one run was noisy; simpler test and no fixed sleep. | +| Hypothesis | Change | Before | After | Decision | Notes | +| --------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------- | --------- | ------- | -------- | -------------------------------------------------------------------------------------------------------------------------- | +| Repeated full-suite runs are too expensive for discovery | Switched full-suite benchmark to one run and added per-file profiler | ~250s/run | pending | keep | Bun has no slowest-test reporter in this version; profile files directly. | +| Plugin install concurrency test spends time spawning more workers than needed to exercise lock contention | Reduced worker counts from 12/10/8 to 6/6/5; kept `holdMs: 30` | 7.800s | 6.204s | keep | Median from 3 targeted runs; still covers concurrent cross-process writes to server, server+tui, and existing json config. | +| `httpapi-listen` PTY route tests pay for git repositories they do not assert on | Removed `git: true` from temp dirs while keeping config setup | 10.554s | 7.818s | keep | Median from 3 targeted runs; HTTP routes, tickets, websocket upgrade, restart, and no-auth paths still pass. | +| `workspace.waitForSync` timeout test waits the full production timeout | Added optional timeout parameter defaulting to production timeout; timeout test uses 25ms | 12.949s | 8.305s | keep | Median from 3 targeted runs; production callers keep the 5000ms default. | +| `config.test` waits after dependencies even though `.gitignore` is written synchronously | Removed obsolete 1000ms sleep from writable `OPENCODE_CONFIG_DIR` test | 10.270s | 9.433s | keep | Median from 5 targeted runs because one run was noisy; simpler test and no fixed sleep. | +| SDK parity helpers create git repos for tests that only need files/config/session state | Changed `withProject` default to no git; explicit git init test still opts into no-git fixture | 8.011s | 5.180s | keep | Median from 5 targeted runs because first run was cold/noisy. | ## Profiling Results From 09133bfdc5cd4bea6c84b285c0866b2d685548dc Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Fri, 15 May 2026 21:58:49 -0400 Subject: [PATCH 06/19] test: mark provider plugin dependencies ready --- packages/opencode/test/provider/provider.test.ts | 4 +++- perf/test-suite.md | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/opencode/test/provider/provider.test.ts b/packages/opencode/test/provider/provider.test.ts index 269064a600c..0a643f312b0 100644 --- a/packages/opencode/test/provider/provider.test.ts +++ b/packages/opencode/test/provider/provider.test.ts @@ -2498,8 +2498,10 @@ test("plugin config providers persist after instance dispose", async () => { test("plugin config enabled and disabled providers are honored", async () => { await using tmp = await tmpdir({ init: async (dir) => { - const root = path.join(dir, ".opencode", "plugin") + const configDir = path.join(dir, ".opencode") + const root = path.join(configDir, "plugin") await mkdir(root, { recursive: true }) + await markPluginDependenciesReady(configDir) await Bun.write( path.join(root, "provider-filter.ts"), [ diff --git a/perf/test-suite.md b/perf/test-suite.md index 864ee0644d9..1696aee34ea 100644 --- a/perf/test-suite.md +++ b/perf/test-suite.md @@ -59,6 +59,7 @@ Repeated setup work, long sleeps/timeouts, serial integration tests, filesystem/ | `workspace.waitForSync` timeout test waits the full production timeout | Added optional timeout parameter defaulting to production timeout; timeout test uses 25ms | 12.949s | 8.305s | keep | Median from 3 targeted runs; production callers keep the 5000ms default. | | `config.test` waits after dependencies even though `.gitignore` is written synchronously | Removed obsolete 1000ms sleep from writable `OPENCODE_CONFIG_DIR` test | 10.270s | 9.433s | keep | Median from 5 targeted runs because one run was noisy; simpler test and no fixed sleep. | | SDK parity helpers create git repos for tests that only need files/config/session state | Changed `withProject` default to no git; explicit git init test still opts into no-git fixture | 8.011s | 5.180s | keep | Median from 5 targeted runs because first run was cold/noisy. | +| Provider plugin filter test waits on plugin dependency readiness setup | Marked local plugin dependencies ready using the existing fixture helper | 7.543s | 6.366s | keep | Median from 3 targeted runs; matches neighboring plugin provider test setup. | ## Profiling Results From 70981cadb33277c4462345622744eaa1b4d9c722 Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Fri, 15 May 2026 22:04:35 -0400 Subject: [PATCH 07/19] test: mark http provider plugin fixtures ready --- .../test/server/httpapi-provider.test.ts | 16 ++++++++++++++++ perf/test-suite.md | 1 + 2 files changed, 17 insertions(+) diff --git a/packages/opencode/test/server/httpapi-provider.test.ts b/packages/opencode/test/server/httpapi-provider.test.ts index 490b947fd9c..226f5d9b529 100644 --- a/packages/opencode/test/server/httpapi-provider.test.ts +++ b/packages/opencode/test/server/httpapi-provider.test.ts @@ -118,6 +118,7 @@ function requestCallback(input: { function writeProviderAuthPlugin(dir: string) { return Effect.gen(function* () { const fs = yield* AppFileSystem.Service + yield* markPluginDependenciesReady(path.join(dir, ".opencode")) yield* fs.writeWithDirs( path.join(dir, ".opencode", "plugin", "provider-oauth-parity.ts"), @@ -152,6 +153,7 @@ function writeProviderAuthPlugin(dir: string) { function writeProviderAuthValidationPlugin(dir: string) { return Effect.gen(function* () { const fs = yield* AppFileSystem.Service + yield* markPluginDependenciesReady(path.join(dir, ".opencode")) yield* fs.writeWithDirs( path.join(dir, ".opencode", "plugin", "provider-oauth-validation.ts"), @@ -193,6 +195,7 @@ function writeProviderAuthValidationPlugin(dir: string) { function writeFunctionOptionsPlugin(dir: string) { return Effect.gen(function* () { const fs = yield* AppFileSystem.Service + yield* markPluginDependenciesReady(path.join(dir, ".opencode")) yield* fs.writeWithDirs( path.join(dir, ".opencode", "plugin", "provider-function-options.ts"), @@ -224,6 +227,7 @@ function writeFunctionOptionsPlugin(dir: string) { function writeProviderModelsMutationPlugin(dir: string) { return Effect.gen(function* () { const fs = yield* AppFileSystem.Service + yield* markPluginDependenciesReady(path.join(dir, ".opencode")) yield* fs.writeWithDirs( path.join(dir, ".opencode", "plugin", "provider-models-mutation.ts"), @@ -253,6 +257,18 @@ function writeProviderModelsMutationPlugin(dir: string) { }) } +function markPluginDependenciesReady(dir: string) { + return AppFileSystem.Service.use((fs) => + Effect.all([ + fs.writeWithDirs(path.join(dir, "node_modules", ".keep"), ""), + fs.writeWithDirs( + path.join(dir, "package-lock.json"), + JSON.stringify({ packages: { "": { dependencies: { "@opencode-ai/plugin": "0.0.0" } } } }), + ), + ]).pipe(Effect.asVoid), + ) +} + function setEnvScoped(key: string, value: string) { return Effect.acquireRelease( Effect.sync(() => { diff --git a/perf/test-suite.md b/perf/test-suite.md index 1696aee34ea..8f4095e3f27 100644 --- a/perf/test-suite.md +++ b/perf/test-suite.md @@ -60,6 +60,7 @@ Repeated setup work, long sleeps/timeouts, serial integration tests, filesystem/ | `config.test` waits after dependencies even though `.gitignore` is written synchronously | Removed obsolete 1000ms sleep from writable `OPENCODE_CONFIG_DIR` test | 10.270s | 9.433s | keep | Median from 5 targeted runs because one run was noisy; simpler test and no fixed sleep. | | SDK parity helpers create git repos for tests that only need files/config/session state | Changed `withProject` default to no git; explicit git init test still opts into no-git fixture | 8.011s | 5.180s | keep | Median from 5 targeted runs because first run was cold/noisy. | | Provider plugin filter test waits on plugin dependency readiness setup | Marked local plugin dependencies ready using the existing fixture helper | 7.543s | 6.366s | keep | Median from 3 targeted runs; matches neighboring plugin provider test setup. | +| HTTP provider tests generate local plugins without dependency-ready fixture state | Marked generated `.opencode` plugin fixtures dependency-ready | 7.905s | 2.980s | keep | Median from 3 targeted runs; avoids unrelated plugin dependency setup in route tests. | ## Profiling Results From d58ec385de60b901cfbda9b5eb9d0d5704c436bb Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Fri, 15 May 2026 22:07:24 -0400 Subject: [PATCH 08/19] test: shorten TUI plugin cleanup timeout coverage --- .../src/cli/cmd/tui/plugin/runtime.ts | 21 ++++++++++++------- .../test/cli/tui/plugin-lifecycle.test.ts | 4 ++-- perf/test-suite.md | 1 + 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/packages/opencode/src/cli/cmd/tui/plugin/runtime.ts b/packages/opencode/src/cli/cmd/tui/plugin/runtime.ts index 2a9ebc4ed2b..62f04c9707b 100644 --- a/packages/opencode/src/cli/cmd/tui/plugin/runtime.ts +++ b/packages/opencode/src/cli/cmd/tui/plugin/runtime.ts @@ -114,6 +114,7 @@ type RuntimeState = { plugins: PluginEntry[] plugins_by_id: Map pending: Map + dispose_timeout_ms: number } const log = Log.create({ service: "tui.plugin" }) @@ -394,7 +395,7 @@ async function syncPluginThemes(plugin: PluginEntry) { } } -function createPluginScope(load: PluginLoad, id: string) { +function createPluginScope(load: PluginLoad, id: string, disposeTimeoutMs: number) { const ctrl = new AbortController() let list: { key: symbol; fn: TuiDispose }[] = [] let done = false @@ -436,14 +437,14 @@ function createPluginScope(load: PluginLoad, id: string) { ctrl.abort() const queue = [...list].reverse() list = [] - const until = Date.now() + DISPOSE_TIMEOUT_MS + const until = Date.now() + disposeTimeoutMs for (const item of queue) { const left = until - Date.now() if (left <= 0) { fail("timed out cleaning up tui plugin", { path: load.spec, id, - timeout: DISPOSE_TIMEOUT_MS, + timeout: disposeTimeoutMs, }) break } @@ -454,7 +455,7 @@ function createPluginScope(load: PluginLoad, id: string) { fail("timed out cleaning up tui plugin", { path: load.spec, id, - timeout: DISPOSE_TIMEOUT_MS, + timeout: disposeTimeoutMs, }) break } @@ -523,7 +524,7 @@ async function activatePluginEntry(state: RuntimeState, plugin: PluginEntry, per if (persist) writePluginEnabledState(state.api, plugin.id, true) if (plugin.scope) return true - const scope = createPluginScope(plugin.load, plugin.id) + const scope = createPluginScope(plugin.load, plugin.id, state.dispose_timeout_ms) const api = pluginApi(state, plugin, scope, plugin.id) const ok = await Promise.resolve() .then(async () => { @@ -1002,7 +1003,12 @@ let loaded: Promise | undefined let runtime: RuntimeState | undefined export const Slot = View -export async function init(input: { api: HostPluginApi; config: TuiConfig.Resolved; dispose?: () => void }) { +export async function init(input: { + api: HostPluginApi + config: TuiConfig.Resolved + dispose?: () => void + disposeTimeoutMs?: number +}) { const cwd = process.cwd() if (loaded) { if (dir !== cwd) { @@ -1052,7 +1058,7 @@ export async function dispose() { state.dispose?.() } -async function load(input: { api: Api; config: TuiConfig.Resolved; dispose?: () => void }) { +async function load(input: { api: Api; config: TuiConfig.Resolved; dispose?: () => void; disposeTimeoutMs?: number }) { const { api, config } = input const cwd = process.cwd() const slots = setupSlots(api) @@ -1064,6 +1070,7 @@ async function load(input: { api: Api; config: TuiConfig.Resolved; dispose?: () plugins: [], plugins_by_id: new Map(), pending: new Map(), + dispose_timeout_ms: input.disposeTimeoutMs ?? DISPOSE_TIMEOUT_MS, } runtime = next try { diff --git a/packages/opencode/test/cli/tui/plugin-lifecycle.test.ts b/packages/opencode/test/cli/tui/plugin-lifecycle.test.ts index 8725fe8b9bc..269c48b6054 100644 --- a/packages/opencode/test/cli/tui/plugin-lifecycle.test.ts +++ b/packages/opencode/test/cli/tui/plugin-lifecycle.test.ts @@ -205,10 +205,10 @@ test( const { config, restore } = mockTuiRuntime(tmp.path, [tmp.extra.spec]) try { - await TuiPluginRuntime.init({ api: createTuiPluginApi(), config }) + await TuiPluginRuntime.init({ api: createTuiPluginApi(), config, disposeTimeoutMs: 25 }) const done = await new Promise((resolve) => { - const timer = setTimeout(() => resolve("timeout"), 7000) + const timer = setTimeout(() => resolve("timeout"), 500) void TuiPluginRuntime.dispose().then(() => { clearTimeout(timer) resolve("done") diff --git a/perf/test-suite.md b/perf/test-suite.md index 8f4095e3f27..b318d8fd25c 100644 --- a/perf/test-suite.md +++ b/perf/test-suite.md @@ -61,6 +61,7 @@ Repeated setup work, long sleeps/timeouts, serial integration tests, filesystem/ | SDK parity helpers create git repos for tests that only need files/config/session state | Changed `withProject` default to no git; explicit git init test still opts into no-git fixture | 8.011s | 5.180s | keep | Median from 5 targeted runs because first run was cold/noisy. | | Provider plugin filter test waits on plugin dependency readiness setup | Marked local plugin dependencies ready using the existing fixture helper | 7.543s | 6.366s | keep | Median from 3 targeted runs; matches neighboring plugin provider test setup. | | HTTP provider tests generate local plugins without dependency-ready fixture state | Marked generated `.opencode` plugin fixtures dependency-ready | 7.905s | 2.980s | keep | Median from 3 targeted runs; avoids unrelated plugin dependency setup in route tests. | +| TUI plugin lifecycle timeout coverage waits the full production cleanup timeout | Added optional runtime dispose timeout override and used 25ms in the timeout test | 7.330s | 1.507s | keep | Median from 3 targeted runs; production default remains 5000ms. | ## Profiling Results From d37abb0d4db4114ee43873e3dde40fd6d85223a9 Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Fri, 15 May 2026 22:09:38 -0400 Subject: [PATCH 09/19] perf: record discarded file cleanup experiment --- perf/test-suite.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/perf/test-suite.md b/perf/test-suite.md index b318d8fd25c..cd45599b1bb 100644 --- a/perf/test-suite.md +++ b/perf/test-suite.md @@ -100,4 +100,6 @@ Targeted 3-run baselines: ## Dead Ends -None yet. +| Hypothesis | Change Tried | Before | After | Decision | Notes | +| ---------------------------------------------------------------------- | ---------------------------------------------------------------------------------------- | -----: | -----: | -------- | --------------------------------------------------------------------------------------------- | +| `file/index.test.ts` pays unnecessary per-test global instance cleanup | Removed `afterEach(disposeAllInstances)` while keeping the explicit disposal test import | 5.262s | 5.089s | discard | Improvement was within noise and the cleanup is a safety guard for many instance-state tests. | From 23dd4b16503331cc39abbb69ffd6c609c50a3946 Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Fri, 15 May 2026 22:15:16 -0400 Subject: [PATCH 10/19] perf: record final test suite benchmark --- perf/test-suite.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/perf/test-suite.md b/perf/test-suite.md index cd45599b1bb..c707663447e 100644 --- a/perf/test-suite.md +++ b/perf/test-suite.md @@ -98,6 +98,12 @@ Targeted 3-run baselines: | `test/provider/provider.test.ts` | 8.323, 7.543, 7.474 | 7.543 | Large serial file. | | `test/server/httpapi-cors.test.ts` | 2.621, 1.682, 1.518 | 1.682 | Not a standalone top target; initial 14s was mixed-scope noise/order effect. | +Final full-suite sanity check: + +| Command | Result | +| -------------------- | -------: | +| `bun run bench:test` | 225.069s | + ## Dead Ends | Hypothesis | Change Tried | Before | After | Decision | Notes | From b5838de474b833d3480179ef7ad4776f240c7b9a Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Fri, 15 May 2026 22:39:22 -0400 Subject: [PATCH 11/19] test: avoid git setup in skill tool test --- packages/opencode/test/tool/skill.test.ts | 90 +++++++++++------------ perf/test-suite.md | 2 + 2 files changed, 46 insertions(+), 46 deletions(-) diff --git a/packages/opencode/test/tool/skill.test.ts b/packages/opencode/test/tool/skill.test.ts index c58d1a190d0..7f63f282ac3 100644 --- a/packages/opencode/test/tool/skill.test.ts +++ b/packages/opencode/test/tool/skill.test.ts @@ -32,14 +32,13 @@ const it = testEffect(Layer.mergeAll(ToolRegistry.defaultLayer, node)) describe("tool.skill", () => { it.live("execute returns skill content block with files", () => - provideTmpdirInstance( - (dir) => - Effect.gen(function* () { - const skill = path.join(dir, ".opencode", "skill", "tool-skill") - yield* Effect.promise(() => - Bun.write( - path.join(skill, "SKILL.md"), - `--- + provideTmpdirInstance((dir) => + Effect.gen(function* () { + const skill = path.join(dir, ".opencode", "skill", "tool-skill") + yield* Effect.promise(() => + Bun.write( + path.join(skill, "SKILL.md"), + `--- name: tool-skill description: Skill for tool tests. --- @@ -48,49 +47,48 @@ description: Skill for tool tests. Use this skill. `, - ), - ) - yield* Effect.promise(() => Bun.write(path.join(skill, "scripts", "demo.txt"), "demo")) + ), + ) + yield* Effect.promise(() => Bun.write(path.join(skill, "scripts", "demo.txt"), "demo")) - const home = process.env.OPENCODE_TEST_HOME - process.env.OPENCODE_TEST_HOME = dir - yield* Effect.addFinalizer(() => - Effect.sync(() => { - process.env.OPENCODE_TEST_HOME = home - }), - ) + const home = process.env.OPENCODE_TEST_HOME + process.env.OPENCODE_TEST_HOME = dir + yield* Effect.addFinalizer(() => + Effect.sync(() => { + process.env.OPENCODE_TEST_HOME = home + }), + ) - const registry = yield* ToolRegistry.Service - const agent = { name: "build", mode: "primary" as const, permission: [], options: {} } - const tool = (yield* registry.tools({ - providerID: "opencode" as any, - modelID: "gpt-5" as any, - agent, - })).find((tool) => tool.id === SkillTool.id) - if (!tool) throw new Error("Skill tool not found") + const registry = yield* ToolRegistry.Service + const agent = { name: "build", mode: "primary" as const, permission: [], options: {} } + const tool = (yield* registry.tools({ + providerID: "opencode" as any, + modelID: "gpt-5" as any, + agent, + })).find((tool) => tool.id === SkillTool.id) + if (!tool) throw new Error("Skill tool not found") - const requests: Array> = [] - const ctx: Tool.Context = { - ...baseCtx, - ask: (req) => - Effect.sync(() => { - requests.push(req) - }), - } + const requests: Array> = [] + const ctx: Tool.Context = { + ...baseCtx, + ask: (req) => + Effect.sync(() => { + requests.push(req) + }), + } - const result = yield* tool.execute({ name: "tool-skill" }, ctx) - const file = path.resolve(skill, "scripts", "demo.txt") + const result = yield* tool.execute({ name: "tool-skill" }, ctx) + const file = path.resolve(skill, "scripts", "demo.txt") - expect(requests.length).toBe(1) - expect(requests[0].permission).toBe("skill") - expect(requests[0].patterns).toContain("tool-skill") - expect(requests[0].always).toContain("tool-skill") - expect(result.metadata.dir).toBe(skill) - expect(result.output).toContain(``) - expect(result.output).toContain(`Base directory for this skill: ${pathToFileURL(skill).href}`) - expect(result.output).toContain(`${file}`) - }), - { git: true }, + expect(requests.length).toBe(1) + expect(requests[0].permission).toBe("skill") + expect(requests[0].patterns).toContain("tool-skill") + expect(requests[0].always).toContain("tool-skill") + expect(result.metadata.dir).toBe(skill) + expect(result.output).toContain(``) + expect(result.output).toContain(`Base directory for this skill: ${pathToFileURL(skill).href}`) + expect(result.output).toContain(`${file}`) + }), ), ) }) diff --git a/perf/test-suite.md b/perf/test-suite.md index c707663447e..11d46595b45 100644 --- a/perf/test-suite.md +++ b/perf/test-suite.md @@ -62,6 +62,7 @@ Repeated setup work, long sleeps/timeouts, serial integration tests, filesystem/ | Provider plugin filter test waits on plugin dependency readiness setup | Marked local plugin dependencies ready using the existing fixture helper | 7.543s | 6.366s | keep | Median from 3 targeted runs; matches neighboring plugin provider test setup. | | HTTP provider tests generate local plugins without dependency-ready fixture state | Marked generated `.opencode` plugin fixtures dependency-ready | 7.905s | 2.980s | keep | Median from 3 targeted runs; avoids unrelated plugin dependency setup in route tests. | | TUI plugin lifecycle timeout coverage waits the full production cleanup timeout | Added optional runtime dispose timeout override and used 25ms in the timeout test | 7.330s | 1.507s | keep | Median from 3 targeted runs; production default remains 5000ms. | +| Skill tool test initializes git even though it only reads local skill files | Removed `git: true` from the temporary directory fixture | 2.320s | 1.425s | keep | Single targeted rerun; still exercises skill discovery, permission request, and bundled file output. | ## Profiling Results @@ -109,3 +110,4 @@ Final full-suite sanity check: | Hypothesis | Change Tried | Before | After | Decision | Notes | | ---------------------------------------------------------------------- | ---------------------------------------------------------------------------------------- | -----: | -----: | -------- | --------------------------------------------------------------------------------------------- | | `file/index.test.ts` pays unnecessary per-test global instance cleanup | Removed `afterEach(disposeAllInstances)` while keeping the explicit disposal test import | 5.262s | 5.089s | discard | Improvement was within noise and the cleanup is a safety guard for many instance-state tests. | +| Socket reset retry test can shorten its idle-timeout path | Reduced Bun server idle timeout and tried forced server close | 16.46s | failed | discard | Shorter idle timeout changed the error shape; forced close hung. Keep the real socket reset. | From 3c7a71ef7bc04d944bf31965865a203d738ebd76 Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Fri, 15 May 2026 22:42:27 -0400 Subject: [PATCH 12/19] perf: record discarded test speed experiments --- perf/test-suite.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/perf/test-suite.md b/perf/test-suite.md index 11d46595b45..55b56a942ec 100644 --- a/perf/test-suite.md +++ b/perf/test-suite.md @@ -111,3 +111,5 @@ Final full-suite sanity check: | ---------------------------------------------------------------------- | ---------------------------------------------------------------------------------------- | -----: | -----: | -------- | --------------------------------------------------------------------------------------------- | | `file/index.test.ts` pays unnecessary per-test global instance cleanup | Removed `afterEach(disposeAllInstances)` while keeping the explicit disposal test import | 5.262s | 5.089s | discard | Improvement was within noise and the cleanup is a safety guard for many instance-state tests. | | Socket reset retry test can shorten its idle-timeout path | Reduced Bun server idle timeout and tried forced server close | 16.46s | failed | discard | Shorter idle timeout changed the error shape; forced close hung. Keep the real socket reset. | +| `tool/webfetch` can avoid per-test instance setup | Switched local HTTP tests from `it.instance` to `it.live` | 1.219s | failed | discard | Tool execution reads instance-local agent state, so the temp instance is required. | +| LSP client interop tests can shorten coarse request-handling sleeps | Reduced fixed post-notification waits from 100ms to 10ms | 4.270s | 4.740s | discard | First run improved to 3.870s but verification was slower than baseline; not a clear win. | From a328dd3ec0d5fa5c12c5a757f1da37379f4cc052 Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Fri, 15 May 2026 22:46:13 -0400 Subject: [PATCH 13/19] test: avoid git setup in prompt shell coverage --- packages/opencode/test/session/prompt.test.ts | 28 ++++++++----------- perf/test-suite.md | 1 + 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/packages/opencode/test/session/prompt.test.ts b/packages/opencode/test/session/prompt.test.ts index 9ac038aa668..e40f1b499fe 100644 --- a/packages/opencode/test/session/prompt.test.ts +++ b/packages/opencode/test/session/prompt.test.ts @@ -857,7 +857,6 @@ it.instance( yield* Fiber.await(fiber) expect((yield* status.get(chat.id)).type).toBe("idle") }), - { git: true }, 3_000, ) @@ -886,7 +885,6 @@ it.instance( expect(exit.value.info.role).toBe("assistant") } }), - { git: true }, 3_000, ) @@ -1324,7 +1322,7 @@ unix( expect(tool.state.metadata.output).toContain("err") yield* run.assertNotBusy(chat.id) }), - { git: true, config: cfg }, + { config: cfg }, ) unix( @@ -1348,7 +1346,7 @@ unix( expect(tool.state.metadata.output).toContain(dir) yield* run.assertNotBusy(chat.id) }), - { git: true, config: cfg }, + { config: cfg }, ) unix( @@ -1370,7 +1368,7 @@ unix( expect(tool.state.output).toContain("configured") }), ), - { git: true, config: { ...cfg, shell: "bash" } }, + { config: { ...cfg, shell: "bash" } }, 30_000, ) @@ -1395,7 +1393,7 @@ unix( expect(tool.state.metadata.output).toContain(parent) yield* run.assertNotBusy(chat.id) }), - { git: true, config: cfg }, + { config: cfg }, ) unix( @@ -1421,7 +1419,7 @@ unix( expect(tool.state.metadata.output).toContain("README.md") yield* run.assertNotBusy(chat.id) }), - { git: true, config: cfg }, + { config: cfg }, ) unix( @@ -1443,7 +1441,7 @@ unix( expect(tool.state.metadata.output).toContain("not found") yield* run.assertNotBusy(chat.id) }), - { git: true, config: cfg }, + { config: cfg }, ) unix( @@ -1471,7 +1469,7 @@ unix( expect(Exit.isSuccess(exit)).toBe(true) }), ), - { git: true, config: cfg }, + { config: cfg }, 30_000, ) @@ -1508,7 +1506,6 @@ it.instance( } expect(yield* llm.calls).toBe(1) }), - { git: true }, 3_000, ) @@ -1547,7 +1544,6 @@ it.instance( } expect(yield* llm.calls).toBe(1) }), - { git: true }, 3_000, ) @@ -1581,7 +1577,6 @@ unix( expect(JSON.stringify(inputs.at(-1)?.messages)).toContain("configured") }), ), - { git: true }, 30_000, ) @@ -1615,7 +1610,7 @@ unix( } }), ), - { git: true, config: cfg }, + { config: cfg }, 30_000, ) @@ -1659,7 +1654,7 @@ unix( } }), ), - { git: true, config: cfg }, + { config: cfg }, 30_000, ) @@ -1708,7 +1703,6 @@ unix( expect(tool.state.output).toMatch(/Full output saved to:\s+\S+/) expect(tool.state.output).not.toContain("Tool execution aborted") }), - { git: true }, 30_000, ) @@ -1735,7 +1729,7 @@ unix( yield* Fiber.await(sh) }), - { git: true, config: cfg }, + { config: cfg }, 30_000, ) @@ -1761,7 +1755,7 @@ unix( yield* Fiber.await(a) }), ), - { git: true, config: cfg }, + { config: cfg }, 30_000, ) diff --git a/perf/test-suite.md b/perf/test-suite.md index 55b56a942ec..6d4e1785453 100644 --- a/perf/test-suite.md +++ b/perf/test-suite.md @@ -63,6 +63,7 @@ Repeated setup work, long sleeps/timeouts, serial integration tests, filesystem/ | HTTP provider tests generate local plugins without dependency-ready fixture state | Marked generated `.opencode` plugin fixtures dependency-ready | 7.905s | 2.980s | keep | Median from 3 targeted runs; avoids unrelated plugin dependency setup in route tests. | | TUI plugin lifecycle timeout coverage waits the full production cleanup timeout | Added optional runtime dispose timeout override and used 25ms in the timeout test | 7.330s | 1.507s | keep | Median from 3 targeted runs; production default remains 5000ms. | | Skill tool test initializes git even though it only reads local skill files | Removed `git: true` from the temporary directory fixture | 2.320s | 1.425s | keep | Single targeted rerun; still exercises skill discovery, permission request, and bundled file output. | +| Prompt shell semantics tests initialize git though they only assert shell/session behavior | Removed `git: true` from shell-focused prompt fixtures while preserving config setup | 26.930s | 23.400s | keep | Three targeted reruns passed after the change: 23.80s, 23.55s, 23.40s. | ## Profiling Results From ded464bd14d9cebadfaa129dd047562f37396349 Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Fri, 15 May 2026 22:53:31 -0400 Subject: [PATCH 14/19] test: avoid git setup in prompt behavior coverage --- packages/opencode/test/session/prompt.test.ts | 646 ++++++++---------- perf/test-suite.md | 1 + 2 files changed, 297 insertions(+), 350 deletions(-) diff --git a/packages/opencode/test/session/prompt.test.ts b/packages/opencode/test/session/prompt.test.ts index e40f1b499fe..14046b59cf8 100644 --- a/packages/opencode/test/session/prompt.test.ts +++ b/packages/opencode/test/session/prompt.test.ts @@ -442,317 +442,290 @@ const boot = Effect.fn("test.boot")(function* (input?: { title?: string }) { // Loop semantics -it.instance( - "loop exits immediately when last assistant has stop finish", - () => - Effect.gen(function* () { - const { llm } = yield* useServerConfig(providerCfg) - const prompt = yield* SessionPrompt.Service - const sessions = yield* Session.Service - const chat = yield* sessions.create({ title: "Pinned" }) - yield* seed(chat.id, { finish: "stop" }) - - const result = yield* prompt.loop({ sessionID: chat.id }) - expect(result.info.role).toBe("assistant") - if (result.info.role === "assistant") expect(result.info.finish).toBe("stop") - expect(yield* llm.calls).toBe(0) - }), - { git: true }, +it.instance("loop exits immediately when last assistant has stop finish", () => + Effect.gen(function* () { + const { llm } = yield* useServerConfig(providerCfg) + const prompt = yield* SessionPrompt.Service + const sessions = yield* Session.Service + const chat = yield* sessions.create({ title: "Pinned" }) + yield* seed(chat.id, { finish: "stop" }) + + const result = yield* prompt.loop({ sessionID: chat.id }) + expect(result.info.role).toBe("assistant") + if (result.info.role === "assistant") expect(result.info.finish).toBe("stop") + expect(yield* llm.calls).toBe(0) + }), ) -it.instance( - "loop calls LLM and returns assistant message", - () => - Effect.gen(function* () { - const { llm } = yield* useServerConfig(providerCfg) - const prompt = yield* SessionPrompt.Service - const sessions = yield* Session.Service - const chat = yield* sessions.create({ - title: "Pinned", - permission: [{ permission: "*", pattern: "*", action: "allow" }], - }) - yield* prompt.prompt({ - sessionID: chat.id, - agent: "build", - noReply: true, - parts: [{ type: "text", text: "hello" }], - }) - yield* llm.text("world") +it.instance("loop calls LLM and returns assistant message", () => + Effect.gen(function* () { + const { llm } = yield* useServerConfig(providerCfg) + const prompt = yield* SessionPrompt.Service + const sessions = yield* Session.Service + const chat = yield* sessions.create({ + title: "Pinned", + permission: [{ permission: "*", pattern: "*", action: "allow" }], + }) + yield* prompt.prompt({ + sessionID: chat.id, + agent: "build", + noReply: true, + parts: [{ type: "text", text: "hello" }], + }) + yield* llm.text("world") - const result = yield* prompt.loop({ sessionID: chat.id }) - expect(result.info.role).toBe("assistant") - const parts = result.parts.filter((p) => p.type === "text") - expect(parts.some((p) => p.type === "text" && p.text === "world")).toBe(true) - expect(yield* llm.hits).toHaveLength(1) - }), - { git: true }, + const result = yield* prompt.loop({ sessionID: chat.id }) + expect(result.info.role).toBe("assistant") + const parts = result.parts.filter((p) => p.type === "text") + expect(parts.some((p) => p.type === "text" && p.text === "world")).toBe(true) + expect(yield* llm.hits).toHaveLength(1) + }), ) -it.instance( - "prompt emits v2 prompted and synthetic events", - () => - Effect.gen(function* () { - yield* useServerConfig(providerCfg) - const prompt = yield* SessionPrompt.Service - const sessions = yield* Session.Service - const chat = yield* sessions.create({ title: "Pinned" }) - - yield* prompt.prompt({ - sessionID: chat.id, - agent: "build", - noReply: true, - parts: [ - { type: "text", text: "hello v2" }, - { - type: "file", - mime: "text/plain", - filename: "note.txt", - url: "data:text/plain;base64,bm90ZSBjb250ZW50", - }, - ], - }) +it.instance("prompt emits v2 prompted and synthetic events", () => + Effect.gen(function* () { + yield* useServerConfig(providerCfg) + const prompt = yield* SessionPrompt.Service + const sessions = yield* Session.Service + const chat = yield* sessions.create({ title: "Pinned" }) + + yield* prompt.prompt({ + sessionID: chat.id, + agent: "build", + noReply: true, + parts: [ + { type: "text", text: "hello v2" }, + { + type: "file", + mime: "text/plain", + filename: "note.txt", + url: "data:text/plain;base64,bm90ZSBjb250ZW50", + }, + ], + }) - const messages = yield* SessionV2.Service.use((session) => session.messages({ sessionID: chat.id })).pipe( - Effect.provide(SessionV2.layer), - ) - const row = Database.use((db) => - db.select().from(SessionMessageTable).where(Database.eq(SessionMessageTable.session_id, chat.id)).get(), - ) - expect(messages.find((message) => message.type === "user")).toMatchObject({ type: "user", text: "hello v2" }) - expect(typeof row?.data.time.created).toBe("number") - expect(messages).toEqual( - expect.arrayContaining([ - expect.objectContaining({ type: "synthetic", text: expect.stringContaining("Called the Read tool") }), - expect.objectContaining({ type: "synthetic", text: "note content" }), - ]), - ) - }), - { git: true }, + const messages = yield* SessionV2.Service.use((session) => session.messages({ sessionID: chat.id })).pipe( + Effect.provide(SessionV2.layer), + ) + const row = Database.use((db) => + db.select().from(SessionMessageTable).where(Database.eq(SessionMessageTable.session_id, chat.id)).get(), + ) + expect(messages.find((message) => message.type === "user")).toMatchObject({ type: "user", text: "hello v2" }) + expect(typeof row?.data.time.created).toBe("number") + expect(messages).toEqual( + expect.arrayContaining([ + expect.objectContaining({ type: "synthetic", text: expect.stringContaining("Called the Read tool") }), + expect.objectContaining({ type: "synthetic", text: "note content" }), + ]), + ) + }), ) -it.instance( - "static loop returns assistant text through local provider", - () => - Effect.gen(function* () { - const { llm } = yield* useServerConfig(providerCfg) - const prompt = yield* SessionPrompt.Service - const sessions = yield* Session.Service - const session = yield* sessions.create({ - title: "Prompt provider", - permission: [{ permission: "*", pattern: "*", action: "allow" }], - }) +it.instance("static loop returns assistant text through local provider", () => + Effect.gen(function* () { + const { llm } = yield* useServerConfig(providerCfg) + const prompt = yield* SessionPrompt.Service + const sessions = yield* Session.Service + const session = yield* sessions.create({ + title: "Prompt provider", + permission: [{ permission: "*", pattern: "*", action: "allow" }], + }) - yield* prompt.prompt({ - sessionID: session.id, - agent: "build", - noReply: true, - parts: [{ type: "text", text: "hello" }], - }) + yield* prompt.prompt({ + sessionID: session.id, + agent: "build", + noReply: true, + parts: [{ type: "text", text: "hello" }], + }) - yield* llm.text("world") + yield* llm.text("world") - const result = yield* prompt.loop({ sessionID: session.id }) - expect(result.info.role).toBe("assistant") - expect(result.parts.some((part) => part.type === "text" && part.text === "world")).toBe(true) - expect(yield* llm.hits).toHaveLength(1) - expect(yield* llm.pending).toBe(0) - }), - { git: true }, + const result = yield* prompt.loop({ sessionID: session.id }) + expect(result.info.role).toBe("assistant") + expect(result.parts.some((part) => part.type === "text" && part.text === "world")).toBe(true) + expect(yield* llm.hits).toHaveLength(1) + expect(yield* llm.pending).toBe(0) + }), ) -it.instance( - "static loop consumes queued replies across turns", - () => - Effect.gen(function* () { - const { llm } = yield* useServerConfig(providerCfg) - const prompt = yield* SessionPrompt.Service - const sessions = yield* Session.Service - const session = yield* sessions.create({ - title: "Prompt provider turns", - permission: [{ permission: "*", pattern: "*", action: "allow" }], - }) +it.instance("static loop consumes queued replies across turns", () => + Effect.gen(function* () { + const { llm } = yield* useServerConfig(providerCfg) + const prompt = yield* SessionPrompt.Service + const sessions = yield* Session.Service + const session = yield* sessions.create({ + title: "Prompt provider turns", + permission: [{ permission: "*", pattern: "*", action: "allow" }], + }) - yield* prompt.prompt({ - sessionID: session.id, - agent: "build", - noReply: true, - parts: [{ type: "text", text: "hello one" }], - }) + yield* prompt.prompt({ + sessionID: session.id, + agent: "build", + noReply: true, + parts: [{ type: "text", text: "hello one" }], + }) - yield* llm.text("world one") + yield* llm.text("world one") - const first = yield* prompt.loop({ sessionID: session.id }) - expect(first.info.role).toBe("assistant") - expect(first.parts.some((part) => part.type === "text" && part.text === "world one")).toBe(true) + const first = yield* prompt.loop({ sessionID: session.id }) + expect(first.info.role).toBe("assistant") + expect(first.parts.some((part) => part.type === "text" && part.text === "world one")).toBe(true) - yield* prompt.prompt({ - sessionID: session.id, - agent: "build", - noReply: true, - parts: [{ type: "text", text: "hello two" }], - }) + yield* prompt.prompt({ + sessionID: session.id, + agent: "build", + noReply: true, + parts: [{ type: "text", text: "hello two" }], + }) - yield* llm.text("world two") + yield* llm.text("world two") - const second = yield* prompt.loop({ sessionID: session.id }) - expect(second.info.role).toBe("assistant") - expect(second.parts.some((part) => part.type === "text" && part.text === "world two")).toBe(true) + const second = yield* prompt.loop({ sessionID: session.id }) + expect(second.info.role).toBe("assistant") + expect(second.parts.some((part) => part.type === "text" && part.text === "world two")).toBe(true) - expect(yield* llm.hits).toHaveLength(2) - expect(yield* llm.pending).toBe(0) - }), - { git: true }, + expect(yield* llm.hits).toHaveLength(2) + expect(yield* llm.pending).toBe(0) + }), ) -it.instance( - "loop continues when finish is tool-calls", - () => - Effect.gen(function* () { - const { llm } = yield* useServerConfig(providerCfg) - const prompt = yield* SessionPrompt.Service - const sessions = yield* Session.Service - const session = yield* sessions.create({ - title: "Pinned", - permission: [{ permission: "*", pattern: "*", action: "allow" }], - }) - yield* prompt.prompt({ - sessionID: session.id, - agent: "build", - noReply: true, - parts: [{ type: "text", text: "hello" }], - }) - yield* llm.tool("first", { value: "first" }) - yield* llm.text("second") - - const result = yield* prompt.loop({ sessionID: session.id }) - expect(yield* llm.calls).toBe(2) - expect(result.info.role).toBe("assistant") - if (result.info.role === "assistant") { - expect(result.parts.some((part) => part.type === "text" && part.text === "second")).toBe(true) - expect(result.info.finish).toBe("stop") - } - }), - { git: true }, +it.instance("loop continues when finish is tool-calls", () => + Effect.gen(function* () { + const { llm } = yield* useServerConfig(providerCfg) + const prompt = yield* SessionPrompt.Service + const sessions = yield* Session.Service + const session = yield* sessions.create({ + title: "Pinned", + permission: [{ permission: "*", pattern: "*", action: "allow" }], + }) + yield* prompt.prompt({ + sessionID: session.id, + agent: "build", + noReply: true, + parts: [{ type: "text", text: "hello" }], + }) + yield* llm.tool("first", { value: "first" }) + yield* llm.text("second") + + const result = yield* prompt.loop({ sessionID: session.id }) + expect(yield* llm.calls).toBe(2) + expect(result.info.role).toBe("assistant") + if (result.info.role === "assistant") { + expect(result.parts.some((part) => part.type === "text" && part.text === "second")).toBe(true) + expect(result.info.finish).toBe("stop") + } + }), ) -it.instance( - "glob tool keeps instance context during prompt runs", - () => - Effect.gen(function* () { - const { dir, llm } = yield* useServerConfig(providerCfg) - const prompt = yield* SessionPrompt.Service - const sessions = yield* Session.Service - const session = yield* sessions.create({ - title: "Glob context", - permission: [{ permission: "*", pattern: "*", action: "allow" }], - }) - const file = path.join(dir, "probe.txt") - yield* writeText(file, "probe") - - yield* prompt.prompt({ - sessionID: session.id, - agent: "build", - noReply: true, - parts: [{ type: "text", text: "find text files" }], - }) - yield* llm.tool("glob", { pattern: "**/*.txt" }) - yield* llm.text("done") - - const result = yield* prompt.loop({ sessionID: session.id }) - expect(result.info.role).toBe("assistant") - - const msgs = yield* MessageV2.filterCompactedEffect(session.id) - const tool = msgs - .flatMap((msg) => msg.parts) - .find( - (part): part is CompletedToolPart => - part.type === "tool" && part.tool === "glob" && part.state.status === "completed", - ) - if (!tool) return +it.instance("glob tool keeps instance context during prompt runs", () => + Effect.gen(function* () { + const { dir, llm } = yield* useServerConfig(providerCfg) + const prompt = yield* SessionPrompt.Service + const sessions = yield* Session.Service + const session = yield* sessions.create({ + title: "Glob context", + permission: [{ permission: "*", pattern: "*", action: "allow" }], + }) + const file = path.join(dir, "probe.txt") + yield* writeText(file, "probe") + + yield* prompt.prompt({ + sessionID: session.id, + agent: "build", + noReply: true, + parts: [{ type: "text", text: "find text files" }], + }) + yield* llm.tool("glob", { pattern: "**/*.txt" }) + yield* llm.text("done") + + const result = yield* prompt.loop({ sessionID: session.id }) + expect(result.info.role).toBe("assistant") + + const msgs = yield* MessageV2.filterCompactedEffect(session.id) + const tool = msgs + .flatMap((msg) => msg.parts) + .find( + (part): part is CompletedToolPart => + part.type === "tool" && part.tool === "glob" && part.state.status === "completed", + ) + if (!tool) return - expect(tool.state.output).toContain(file) - expect(tool.state.output).not.toContain("No context found for instance") - expect(result.parts.some((part) => part.type === "text" && part.text === "done")).toBe(true) - }), - { git: true }, + expect(tool.state.output).toContain(file) + expect(tool.state.output).not.toContain("No context found for instance") + expect(result.parts.some((part) => part.type === "text" && part.text === "done")).toBe(true) + }), ) -it.instance( - "loop continues when finish is stop but assistant has tool parts", - () => - Effect.gen(function* () { - const { llm } = yield* useServerConfig(providerCfg) - const prompt = yield* SessionPrompt.Service - const sessions = yield* Session.Service - const session = yield* sessions.create({ - title: "Pinned", - permission: [{ permission: "*", pattern: "*", action: "allow" }], - }) - yield* prompt.prompt({ - sessionID: session.id, - agent: "build", - noReply: true, - parts: [{ type: "text", text: "hello" }], - }) - yield* llm.push(reply().tool("first", { value: "first" }).stop()) - yield* llm.text("second") - - const result = yield* prompt.loop({ sessionID: session.id }) - expect(yield* llm.calls).toBe(2) - expect(result.info.role).toBe("assistant") - if (result.info.role === "assistant") { - expect(result.parts.some((part) => part.type === "text" && part.text === "second")).toBe(true) - expect(result.info.finish).toBe("stop") - } - }), - { git: true }, +it.instance("loop continues when finish is stop but assistant has tool parts", () => + Effect.gen(function* () { + const { llm } = yield* useServerConfig(providerCfg) + const prompt = yield* SessionPrompt.Service + const sessions = yield* Session.Service + const session = yield* sessions.create({ + title: "Pinned", + permission: [{ permission: "*", pattern: "*", action: "allow" }], + }) + yield* prompt.prompt({ + sessionID: session.id, + agent: "build", + noReply: true, + parts: [{ type: "text", text: "hello" }], + }) + yield* llm.push(reply().tool("first", { value: "first" }).stop()) + yield* llm.text("second") + + const result = yield* prompt.loop({ sessionID: session.id }) + expect(yield* llm.calls).toBe(2) + expect(result.info.role).toBe("assistant") + if (result.info.role === "assistant") { + expect(result.parts.some((part) => part.type === "text" && part.text === "second")).toBe(true) + expect(result.info.finish).toBe("stop") + } + }), ) -it.instance( - "failed subtask preserves metadata on error tool state", - () => - Effect.gen(function* () { - const { llm } = yield* useServerConfig((url) => ({ - ...providerCfg(url), - agent: { - general: { - model: "test/missing-model", - }, +it.instance("failed subtask preserves metadata on error tool state", () => + Effect.gen(function* () { + const { llm } = yield* useServerConfig((url) => ({ + ...providerCfg(url), + agent: { + general: { + model: "test/missing-model", }, - })) - const prompt = yield* SessionPrompt.Service - const sessions = yield* Session.Service - const chat = yield* sessions.create({ title: "Pinned" }) - yield* llm.tool("task", { - description: "inspect bug", - prompt: "look into the cache key path", - subagent_type: "general", - }) - yield* llm.text("done") - const msg = yield* user(chat.id, "hello") - yield* addSubtask(chat.id, msg.id) - - const result = yield* prompt.loop({ sessionID: chat.id }) - expect(result.info.role).toBe("assistant") - expect(yield* llm.calls).toBe(2) - - const msgs = yield* MessageV2.filterCompactedEffect(chat.id) - const taskMsg = msgs.find((item) => item.info.role === "assistant" && item.info.agent === "general") - expect(taskMsg?.info.role).toBe("assistant") - if (!taskMsg || taskMsg.info.role !== "assistant") return - - const tool = errorTool(taskMsg.parts) - if (!tool) return - - expect(tool.state.error).toContain("Tool execution failed") - expect(tool.state.metadata).toBeDefined() - expect(tool.state.metadata?.sessionId).toBeDefined() - expect(tool.state.metadata?.model).toEqual({ - providerID: ProviderID.make("test"), - modelID: ModelID.make("missing-model"), - }) - }), - { git: true }, + }, + })) + const prompt = yield* SessionPrompt.Service + const sessions = yield* Session.Service + const chat = yield* sessions.create({ title: "Pinned" }) + yield* llm.tool("task", { + description: "inspect bug", + prompt: "look into the cache key path", + subagent_type: "general", + }) + yield* llm.text("done") + const msg = yield* user(chat.id, "hello") + yield* addSubtask(chat.id, msg.id) + + const result = yield* prompt.loop({ sessionID: chat.id }) + expect(result.info.role).toBe("assistant") + expect(yield* llm.calls).toBe(2) + + const msgs = yield* MessageV2.filterCompactedEffect(chat.id) + const taskMsg = msgs.find((item) => item.info.role === "assistant" && item.info.agent === "general") + expect(taskMsg?.info.role).toBe("assistant") + if (!taskMsg || taskMsg.info.role !== "assistant") return + + const tool = errorTool(taskMsg.parts) + if (!tool) return + + expect(tool.state.error).toContain("Tool execution failed") + expect(tool.state.metadata).toBeDefined() + expect(tool.state.metadata?.sessionId).toBeDefined() + expect(tool.state.metadata?.model).toEqual({ + providerID: ProviderID.make("test"), + modelID: ModelID.make("missing-model"), + }) + }), ) it.instance( @@ -787,7 +760,6 @@ it.instance( yield* prompt.cancel(chat.id) yield* Fiber.await(fiber) }), - { git: true }, 5_000, ) @@ -832,7 +804,6 @@ it.instance( yield* prompt.cancel(chat.id) yield* Fiber.await(fiber) }), - { git: true }, 10_000, ) @@ -911,7 +882,6 @@ it.instance( } } }), - { git: true }, 3_000, ) @@ -1001,7 +971,6 @@ race.instance( expect(lastAssistant.info.parentID).toBe(lastUser?.info.id) } }), - { git: true }, 3_000, ) @@ -1048,7 +1017,7 @@ it.instance( expect(taskMsg.info.time.completed).toBeDefined() expect(taskMsg.info.finish).toBeDefined() }), - { git: true, config: cfg }, + { config: cfg }, 30_000, ) @@ -1084,7 +1053,6 @@ it.instance( expect((yield* status.get(chat.id)).type).toBe("idle") expect((yield* status.get(childID)).type).toBe("idle") }), - { git: true }, 10_000, ) @@ -1112,28 +1080,24 @@ it.instance( expect(exitA.value.info.id).toBe(exitB.value.info.id) } }), - { git: true }, 3_000, ) // Queue semantics -it.instance( - "concurrent loop callers get same result", - () => - Effect.gen(function* () { - const { prompt, run, chat } = yield* boot() - yield* seed(chat.id, { finish: "stop" }) +it.instance("concurrent loop callers get same result", () => + Effect.gen(function* () { + const { prompt, run, chat } = yield* boot() + yield* seed(chat.id, { finish: "stop" }) - const [a, b] = yield* Effect.all([prompt.loop({ sessionID: chat.id }), prompt.loop({ sessionID: chat.id })], { - concurrency: "unbounded", - }) + const [a, b] = yield* Effect.all([prompt.loop({ sessionID: chat.id }), prompt.loop({ sessionID: chat.id })], { + concurrency: "unbounded", + }) - expect(a.info.id).toBe(b.info.id) - expect(a.info.role).toBe("assistant") - yield* run.assertNotBusy(chat.id) - }), - { git: true }, + expect(a.info.id).toBe(b.info.id) + expect(a.info.role).toBe("assistant") + yield* run.assertNotBusy(chat.id) + }), ) it.instance( @@ -1154,7 +1118,6 @@ it.instance( expect(a.info.id).toBe(b.info.id) expect(a.info.role).toBe("assistant") }), - { git: true }, 3_000, ) @@ -1223,7 +1186,6 @@ it.instance( expect(inputs).toHaveLength(2) expect(JSON.stringify(inputs.at(-1)?.messages)).toContain("second") }), - { git: true }, 3_000, ) @@ -1253,22 +1215,18 @@ it.instance( yield* prompt.cancel(chat.id) yield* Fiber.await(fiber) }), - { git: true }, 3_000, ) -it.instance( - "assertNotBusy succeeds when idle", - () => - Effect.gen(function* () { - const run = yield* SessionRunState.Service - const sessions = yield* Session.Service +it.instance("assertNotBusy succeeds when idle", () => + Effect.gen(function* () { + const run = yield* SessionRunState.Service + const sessions = yield* Session.Service - const chat = yield* sessions.create({}) - const exit = yield* run.assertNotBusy(chat.id).pipe(Effect.exit) - expect(Exit.isSuccess(exit)).toBe(true) - }), - { git: true }, + const chat = yield* sessions.create({}) + const exit = yield* run.assertNotBusy(chat.id).pipe(Effect.exit) + expect(Exit.isSuccess(exit)).toBe(true) + }), ) // Shell semantics @@ -1297,7 +1255,6 @@ it.instance( yield* prompt.cancel(chat.id) yield* Fiber.await(fiber) }), - { git: true }, 3_000, ) @@ -1811,7 +1768,7 @@ it.instance( const exit = yield* Fiber.await(fiber) expect(Exit.isFailure(exit)).toBe(true) }), - { git: true, config: cfg }, + { config: cfg }, 30_000, ) @@ -1846,7 +1803,7 @@ it.instance( const exit = yield* Fiber.await(fiber) expect(Exit.isFailure(exit)).toBe(true) }), - { git: true, config: cfg }, + { config: cfg }, 30_000, ) @@ -1885,7 +1842,7 @@ it.instance( yield* sessions.remove(session.id) }), - { git: true, config: cfg }, + { config: cfg }, ) it.instance( @@ -1927,7 +1884,7 @@ it.instance( yield* sessions.remove(session.id) }), - { git: true, config: cfg }, + { config: cfg }, ) it.instance( @@ -1976,7 +1933,6 @@ it.instance( expect(agents.map((agent) => agent.name)).toEqual(["build"]) }), { - git: true, config: { ...cfg, reference: { @@ -2016,7 +1972,6 @@ it.instance( yield* sessions.remove(session.id) }), { - git: true, config: { ...cfg, reference: { @@ -2082,7 +2037,6 @@ it.instance( yield* sessions.remove(session.id) }), { - git: true, config: { ...cfg, reference: { @@ -2131,31 +2085,28 @@ it.instance( // Regression: empty assistant turn loop -it.instance( - "does not loop empty assistant turns for a simple reply", - () => - Effect.gen(function* () { - const { llm } = yield* useServerConfig(providerCfg) - const prompt = yield* SessionPrompt.Service - const sessions = yield* Session.Service - const session = yield* sessions.create({ title: "Prompt regression" }) +it.instance("does not loop empty assistant turns for a simple reply", () => + Effect.gen(function* () { + const { llm } = yield* useServerConfig(providerCfg) + const prompt = yield* SessionPrompt.Service + const sessions = yield* Session.Service + const session = yield* sessions.create({ title: "Prompt regression" }) - yield* llm.text("packages/opencode/src/session/processor.ts") + yield* llm.text("packages/opencode/src/session/processor.ts") - const result = yield* prompt.prompt({ - sessionID: session.id, - agent: "build", - parts: [{ type: "text", text: "Where is SessionProcessor?" }], - }) + const result = yield* prompt.prompt({ + sessionID: session.id, + agent: "build", + parts: [{ type: "text", text: "Where is SessionProcessor?" }], + }) - expect(result.info.role).toBe("assistant") - expect(result.parts.some((part) => part.type === "text" && part.text.includes("processor.ts"))).toBe(true) + expect(result.info.role).toBe("assistant") + expect(result.parts.some((part) => part.type === "text" && part.text.includes("processor.ts"))).toBe(true) - const msgs = yield* sessions.messages({ sessionID: session.id }) - expect(msgs.filter((msg) => msg.info.role === "assistant")).toHaveLength(1) - expect(yield* llm.calls).toBe(1) - }), - { git: true }, + const msgs = yield* sessions.messages({ sessionID: session.id }) + expect(msgs.filter((msg) => msg.info.role === "assistant")).toHaveLength(1) + expect(yield* llm.calls).toBe(1) + }), ) it.instance( @@ -2196,7 +2147,6 @@ it.instance( expect(last.info.error?.name).toBe("MessageAbortedError") } }), - { git: true }, 3_000, ) @@ -2247,7 +2197,6 @@ it.instance( yield* sessions.remove(session.id) }), { - git: true, config: { ...cfg, provider: { @@ -2300,7 +2249,6 @@ it.instance( } } }), - { git: true }, 30_000, ) @@ -2329,7 +2277,6 @@ it.instance( } } }), - { git: true }, 30_000, ) @@ -2359,6 +2306,5 @@ it.instance( } } }), - { git: true }, 30_000, ) diff --git a/perf/test-suite.md b/perf/test-suite.md index 6d4e1785453..ba83ba1600e 100644 --- a/perf/test-suite.md +++ b/perf/test-suite.md @@ -64,6 +64,7 @@ Repeated setup work, long sleeps/timeouts, serial integration tests, filesystem/ | TUI plugin lifecycle timeout coverage waits the full production cleanup timeout | Added optional runtime dispose timeout override and used 25ms in the timeout test | 7.330s | 1.507s | keep | Median from 3 targeted runs; production default remains 5000ms. | | Skill tool test initializes git even though it only reads local skill files | Removed `git: true` from the temporary directory fixture | 2.320s | 1.425s | keep | Single targeted rerun; still exercises skill discovery, permission request, and bundled file output. | | Prompt shell semantics tests initialize git though they only assert shell/session behavior | Removed `git: true` from shell-focused prompt fixtures while preserving config setup | 26.930s | 23.400s | keep | Three targeted reruns passed after the change: 23.80s, 23.55s, 23.40s. | +| Remaining prompt behavior tests mostly do not require repository state | Removed git setup from loop/cancel/reference/error fixtures; kept it for `#` filename lookup | 23.400s | 16.590s | keep | Targeted prompt file passes; `#` filename test failed without git and was restored. | ## Profiling Results From 40014fe396bf2d9853bc9668f543835b394aed8f Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Sat, 16 May 2026 10:44:08 -0400 Subject: [PATCH 15/19] test: simplify plugin readiness fixtures --- .../opencode/script/profile-test-files.ts | 8 ++++--- packages/opencode/test/fixture/plugin.ts | 10 +++++++++ .../opencode/test/provider/provider.test.ts | 9 +------- .../test/server/httpapi-provider.test.ts | 21 +++++-------------- 4 files changed, 21 insertions(+), 27 deletions(-) create mode 100644 packages/opencode/test/fixture/plugin.ts diff --git a/packages/opencode/script/profile-test-files.ts b/packages/opencode/script/profile-test-files.ts index 6f1b04ce9b2..574fcea9eaf 100644 --- a/packages/opencode/script/profile-test-files.ts +++ b/packages/opencode/script/profile-test-files.ts @@ -14,9 +14,11 @@ for (const file of await files) { stderr: "pipe", env: Bun.env, }) - const output = await new Response(proc.stdout).text() - const error = await new Response(proc.stderr).text() - const exitCode = await proc.exited + const [output, error, exitCode] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]) const seconds = (performance.now() - start) / 1000 results.push({ file, seconds, exitCode }) console.log(`${exitCode === 0 ? "PASS" : "FAIL"} ${seconds.toFixed(3)}s ${file}`) diff --git a/packages/opencode/test/fixture/plugin.ts b/packages/opencode/test/fixture/plugin.ts new file mode 100644 index 00000000000..2dbb64a5e5d --- /dev/null +++ b/packages/opencode/test/fixture/plugin.ts @@ -0,0 +1,10 @@ +import { mkdir } from "fs/promises" +import path from "path" + +export async function markPluginDependenciesReady(dir: string) { + await mkdir(path.join(dir, "node_modules"), { recursive: true }) + await Bun.write( + path.join(dir, "package-lock.json"), + JSON.stringify({ packages: { "": { dependencies: { "@opencode-ai/plugin": "0.0.0" } } } }), + ) +} diff --git a/packages/opencode/test/provider/provider.test.ts b/packages/opencode/test/provider/provider.test.ts index 0a643f312b0..a9ecbc4610d 100644 --- a/packages/opencode/test/provider/provider.test.ts +++ b/packages/opencode/test/provider/provider.test.ts @@ -3,6 +3,7 @@ import { mkdir, unlink } from "fs/promises" import path from "path" import { disposeAllInstances, tmpdir } from "../fixture/fixture" +import { markPluginDependenciesReady } from "../fixture/plugin" import { Global } from "@opencode-ai/core/global" import { Instance } from "../../src/project/instance" import { WithInstance } from "../../src/project/with-instance" @@ -58,14 +59,6 @@ async function defaultModel() { return run((provider) => provider.defaultModel()) } -async function markPluginDependenciesReady(dir: string) { - await mkdir(path.join(dir, "node_modules"), { recursive: true }) - await Bun.write( - path.join(dir, "package-lock.json"), - JSON.stringify({ packages: { "": { dependencies: { "@opencode-ai/plugin": "0.0.0" } } } }), - ) -} - function paid(providers: Awaited>) { const item = providers[ProviderID.make("opencode")] expect(item).toBeDefined() diff --git a/packages/opencode/test/server/httpapi-provider.test.ts b/packages/opencode/test/server/httpapi-provider.test.ts index 226f5d9b529..d11ecc85ecd 100644 --- a/packages/opencode/test/server/httpapi-provider.test.ts +++ b/packages/opencode/test/server/httpapi-provider.test.ts @@ -6,6 +6,7 @@ import { Server } from "../../src/server/server" import * as Log from "@opencode-ai/core/util/log" import { resetDatabase } from "../fixture/db" import { TestInstance } from "../fixture/fixture" +import { markPluginDependenciesReady } from "../fixture/plugin" import { testEffect } from "../lib/effect" void Log.init({ print: false }) @@ -118,7 +119,7 @@ function requestCallback(input: { function writeProviderAuthPlugin(dir: string) { return Effect.gen(function* () { const fs = yield* AppFileSystem.Service - yield* markPluginDependenciesReady(path.join(dir, ".opencode")) + yield* Effect.promise(() => markPluginDependenciesReady(path.join(dir, ".opencode"))) yield* fs.writeWithDirs( path.join(dir, ".opencode", "plugin", "provider-oauth-parity.ts"), @@ -153,7 +154,7 @@ function writeProviderAuthPlugin(dir: string) { function writeProviderAuthValidationPlugin(dir: string) { return Effect.gen(function* () { const fs = yield* AppFileSystem.Service - yield* markPluginDependenciesReady(path.join(dir, ".opencode")) + yield* Effect.promise(() => markPluginDependenciesReady(path.join(dir, ".opencode"))) yield* fs.writeWithDirs( path.join(dir, ".opencode", "plugin", "provider-oauth-validation.ts"), @@ -195,7 +196,7 @@ function writeProviderAuthValidationPlugin(dir: string) { function writeFunctionOptionsPlugin(dir: string) { return Effect.gen(function* () { const fs = yield* AppFileSystem.Service - yield* markPluginDependenciesReady(path.join(dir, ".opencode")) + yield* Effect.promise(() => markPluginDependenciesReady(path.join(dir, ".opencode"))) yield* fs.writeWithDirs( path.join(dir, ".opencode", "plugin", "provider-function-options.ts"), @@ -227,7 +228,7 @@ function writeFunctionOptionsPlugin(dir: string) { function writeProviderModelsMutationPlugin(dir: string) { return Effect.gen(function* () { const fs = yield* AppFileSystem.Service - yield* markPluginDependenciesReady(path.join(dir, ".opencode")) + yield* Effect.promise(() => markPluginDependenciesReady(path.join(dir, ".opencode"))) yield* fs.writeWithDirs( path.join(dir, ".opencode", "plugin", "provider-models-mutation.ts"), @@ -257,18 +258,6 @@ function writeProviderModelsMutationPlugin(dir: string) { }) } -function markPluginDependenciesReady(dir: string) { - return AppFileSystem.Service.use((fs) => - Effect.all([ - fs.writeWithDirs(path.join(dir, "node_modules", ".keep"), ""), - fs.writeWithDirs( - path.join(dir, "package-lock.json"), - JSON.stringify({ packages: { "": { dependencies: { "@opencode-ai/plugin": "0.0.0" } } } }), - ), - ]).pipe(Effect.asVoid), - ) -} - function setEnvScoped(key: string, value: string) { return Effect.acquireRelease( Effect.sync(() => { From 5900b4b2aabd455bfa40ad6fc8a8a6fbdbd377cf Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Sat, 16 May 2026 10:46:28 -0400 Subject: [PATCH 16/19] test: avoid git setup in processor effect coverage --- .../test/session/processor-effect.test.ts | 24 +++++++++---------- perf/test-suite.md | 1 + 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/packages/opencode/test/session/processor-effect.test.ts b/packages/opencode/test/session/processor-effect.test.ts index 61c566eaec2..cfa1a60de7a 100644 --- a/packages/opencode/test/session/processor-effect.test.ts +++ b/packages/opencode/test/session/processor-effect.test.ts @@ -247,7 +247,7 @@ it.live("session.processor effect tests capture llm input cleanly", () => expect(calls).toBe(1) expect(parts.some((part) => part.type === "text" && part.text === "hello")).toBe(true) }), - { git: true, config: (url) => providerCfg(url) }, + { config: (url) => providerCfg(url) }, ), ) @@ -329,7 +329,7 @@ it.live("session.processor effect tests preserve text start time", () => if (!text?.time?.start || !text.time.end) return expect(text.time.start).toBeLessThan(text.time.end) }), - { git: true, config: (url) => providerCfg(url) }, + { config: (url) => providerCfg(url) }, ), ) @@ -375,7 +375,7 @@ it.live("session.processor effect tests stop after token overflow requests compa expect(parts.some((part) => part.type === "text" && part.text === "after")).toBe(true) expect(parts.some((part) => part.type === "step-finish")).toBe(true) }), - { git: true, config: (url) => providerCfg(url) }, + { config: (url) => providerCfg(url) }, ), ) @@ -423,7 +423,7 @@ it.live("session.processor effect tests capture reasoning from http mock", () => expect(reasoning?.text).toBe("think") expect(text?.text).toBe("done") }), - { git: true, config: (url) => providerCfg(url) }, + { config: (url) => providerCfg(url) }, ), ) @@ -470,7 +470,7 @@ it.live("session.processor effect tests reset reasoning state across retries", ( expect(reasoning.some((part) => part.text === "two")).toBe(true) expect(reasoning.some((part) => part.text === "onetwo")).toBe(false) }), - { git: true, config: (url) => providerCfg(url) }, + { config: (url) => providerCfg(url) }, ), ) @@ -513,7 +513,7 @@ it.live("session.processor effect tests do not retry unknown json errors", () => expect(yield* llm.calls).toBe(1) expect(handle.message.error?.name).toBe("APIError") }), - { git: true, config: (url) => providerCfg(url) }, + { config: (url) => providerCfg(url) }, ), ) @@ -560,7 +560,7 @@ it.live("session.processor effect tests retry recognized structured json errors" expect(parts.some((part) => part.type === "text" && part.text === "after")).toBe(true) expect(handle.message.error).toBeUndefined() }), - { git: true, config: (url) => providerCfg(url) }, + { config: (url) => providerCfg(url) }, ), ) @@ -612,7 +612,7 @@ it.live("session.processor effect tests publish retry status updates", () => expect(yield* llm.calls).toBe(2) expect(states).toStrictEqual([1]) }), - { git: true, config: (url) => providerCfg(url) }, + { config: (url) => providerCfg(url) }, ), ) @@ -655,7 +655,7 @@ it.live("session.processor effect tests compact on structured context overflow", expect(yield* llm.calls).toBe(1) expect(handle.message.error).toBeUndefined() }), - { git: true, config: (url) => providerCfg(url) }, + { config: (url) => providerCfg(url) }, ), ) @@ -719,7 +719,7 @@ it.live("session.processor effect tests mark pending tools as aborted on cleanup expect(call.state.time.end).toBeDefined() } }), - { git: true, config: (url) => providerCfg(url) }, + { config: (url) => providerCfg(url) }, ), ) @@ -791,7 +791,7 @@ it.live("session.processor effect tests record aborted errors and idle state", ( expect(state).toMatchObject({ type: "idle" }) expect(errs).toContain("MessageAbortedError") }), - { git: true, config: (url) => providerCfg(url) }, + { config: (url) => providerCfg(url) }, ), ) @@ -848,6 +848,6 @@ it.live("session.processor effect tests mark interruptions aborted without manua } expect(state).toMatchObject({ type: "idle" }) }), - { git: true, config: (url) => providerCfg(url) }, + { config: (url) => providerCfg(url) }, ), ) diff --git a/perf/test-suite.md b/perf/test-suite.md index ba83ba1600e..1bcae39613d 100644 --- a/perf/test-suite.md +++ b/perf/test-suite.md @@ -65,6 +65,7 @@ Repeated setup work, long sleeps/timeouts, serial integration tests, filesystem/ | Skill tool test initializes git even though it only reads local skill files | Removed `git: true` from the temporary directory fixture | 2.320s | 1.425s | keep | Single targeted rerun; still exercises skill discovery, permission request, and bundled file output. | | Prompt shell semantics tests initialize git though they only assert shell/session behavior | Removed `git: true` from shell-focused prompt fixtures while preserving config setup | 26.930s | 23.400s | keep | Three targeted reruns passed after the change: 23.80s, 23.55s, 23.40s. | | Remaining prompt behavior tests mostly do not require repository state | Removed git setup from loop/cancel/reference/error fixtures; kept it for `#` filename lookup | 23.400s | 16.590s | keep | Targeted prompt file passes; `#` filename test failed without git and was restored. | +| Session processor effect tests do not require repository state | Removed git setup from all processor-effect temp server fixtures | 12.500s | 9.230s | keep | Two targeted reruns passed after the change: 9.61s, 9.23s. | ## Profiling Results From 6ad52fc17090c401ef11a38c8db996d197a8f3c3 Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Sat, 16 May 2026 10:49:48 -0400 Subject: [PATCH 17/19] test: combine PTY ticket listener coverage --- .../test/server/httpapi-listen.test.ts | 64 +++++++------------ perf/test-suite.md | 31 ++++----- 2 files changed, 38 insertions(+), 57 deletions(-) diff --git a/packages/opencode/test/server/httpapi-listen.test.ts b/packages/opencode/test/server/httpapi-listen.test.ts index 0b7b963759b..344abecda4f 100644 --- a/packages/opencode/test/server/httpapi-listen.test.ts +++ b/packages/opencode/test/server/httpapi-listen.test.ts @@ -217,49 +217,15 @@ describe("HttpApi Server.listen", () => { expect((await requestTicket(listener, info.id, tmp.path, { ticketHeader: false })).status).toBe(403) expect((await requestTicket(listener, info.id, tmp.path, { origin: "https://evil.example" })).status).toBe(403) - await expectSocketRejected(socketURL(listener, info.id, tmp.path, "not-a-ticket")) - - const reusable = await connectTicket(listener, info.id, tmp.path) - const ws = await openSocket(socketURL(listener, info.id, tmp.path, reusable.ticket)) - await expectSocketRejected(socketURL(listener, info.id, tmp.path, reusable.ticket)) - ws.close(1000) - - const other = await createCat(listener, tmp.path) - const scoped = await connectTicket(listener, info.id, tmp.path) - await expectSocketRejected(socketURL(listener, other.id, tmp.path, scoped.ticket)) - - const crossOrigin = await connectTicket(listener, info.id, tmp.path) - await expectSocketRejected(socketURL(listener, info.id, tmp.path, crossOrigin.ticket), { - headers: { origin: "https://evil.example" }, - }) - } finally { - await stop(listener, "timed out cleaning up rejected ticket listener").catch(() => undefined) - } - }) - - // Regression for #25698 (Ope): the app's SDK call to - // `client.pty.connectToken({ ptyID })` originally omitted `directory`, so - // the server resolved the PTY in its own cwd context — where the project - // PTY isn't registered — and returned 404. The fix is to always pass - // `directory` from the app side; this test locks in two contracts: - // 1. Mint without directory cannot find a PTY registered in another dir. - // 2. Mint with the project directory succeeds; the resulting ticket - // consumes cleanly when the WS upgrade carries the same directory. - testPty("PTY connect token requires matching directory across mint and connect", async () => { - await using tmp = await tmpdir({ config: { formatter: false, lsp: false } }) - const listener = await startListener() - try { - const info = await createCat(listener, tmp.path) - - // Mint without directory — server uses its own cwd, can't find the PTY. + // Regression for #25698: minting without a directory uses the server cwd + // and cannot find a PTY registered in a project directory. const ambiguous = await fetch(new URL(PtyPaths.connectToken.replace(":ptyID", info.id), listener.url), { method: "POST", headers: { authorization: authorization(), "x-opencode-ticket": "1" }, }) expect(ambiguous.status).toBe(404) - // Mint with the project directory — succeeds, ticket binds to that scope. - const scoped = await fetch( + const directoryScoped = await fetch( new URL( `${PtyPaths.connectToken.replace(":ptyID", info.id)}?directory=${encodeURIComponent(tmp.path)}`, listener.url, @@ -269,14 +235,28 @@ describe("HttpApi Server.listen", () => { headers: { authorization: authorization(), "x-opencode-ticket": "1" }, }, ) - expect(scoped.status).toBe(200) - const mint = (await scoped.json()) as { ticket: string } + expect(directoryScoped.status).toBe(200) + const mint = (await directoryScoped.json()) as { ticket: string } + const scopedWs = await openSocket(socketURL(listener, info.id, tmp.path, mint.ticket)) + scopedWs.close(1000) - // Same directory on the WS upgrade → consume succeeds. - const ws = await openSocket(socketURL(listener, info.id, tmp.path, mint.ticket)) + await expectSocketRejected(socketURL(listener, info.id, tmp.path, "not-a-ticket")) + + const reusable = await connectTicket(listener, info.id, tmp.path) + const ws = await openSocket(socketURL(listener, info.id, tmp.path, reusable.ticket)) + await expectSocketRejected(socketURL(listener, info.id, tmp.path, reusable.ticket)) ws.close(1000) + + const other = await createCat(listener, tmp.path) + const scoped = await connectTicket(listener, info.id, tmp.path) + await expectSocketRejected(socketURL(listener, other.id, tmp.path, scoped.ticket)) + + const crossOrigin = await connectTicket(listener, info.id, tmp.path) + await expectSocketRejected(socketURL(listener, info.id, tmp.path, crossOrigin.ticket), { + headers: { origin: "https://evil.example" }, + }) } finally { - await stop(listener, "timed out cleaning up directory-scope listener").catch(() => undefined) + await stop(listener, "timed out cleaning up rejected ticket listener").catch(() => undefined) } }) diff --git a/perf/test-suite.md b/perf/test-suite.md index 1bcae39613d..746544c4f7e 100644 --- a/perf/test-suite.md +++ b/perf/test-suite.md @@ -51,21 +51,22 @@ Repeated setup work, long sleeps/timeouts, serial integration tests, filesystem/ ## Hypothesis Loop -| Hypothesis | Change | Before | After | Decision | Notes | -| --------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------- | --------- | ------- | -------- | -------------------------------------------------------------------------------------------------------------------------- | -| Repeated full-suite runs are too expensive for discovery | Switched full-suite benchmark to one run and added per-file profiler | ~250s/run | pending | keep | Bun has no slowest-test reporter in this version; profile files directly. | -| Plugin install concurrency test spends time spawning more workers than needed to exercise lock contention | Reduced worker counts from 12/10/8 to 6/6/5; kept `holdMs: 30` | 7.800s | 6.204s | keep | Median from 3 targeted runs; still covers concurrent cross-process writes to server, server+tui, and existing json config. | -| `httpapi-listen` PTY route tests pay for git repositories they do not assert on | Removed `git: true` from temp dirs while keeping config setup | 10.554s | 7.818s | keep | Median from 3 targeted runs; HTTP routes, tickets, websocket upgrade, restart, and no-auth paths still pass. | -| `workspace.waitForSync` timeout test waits the full production timeout | Added optional timeout parameter defaulting to production timeout; timeout test uses 25ms | 12.949s | 8.305s | keep | Median from 3 targeted runs; production callers keep the 5000ms default. | -| `config.test` waits after dependencies even though `.gitignore` is written synchronously | Removed obsolete 1000ms sleep from writable `OPENCODE_CONFIG_DIR` test | 10.270s | 9.433s | keep | Median from 5 targeted runs because one run was noisy; simpler test and no fixed sleep. | -| SDK parity helpers create git repos for tests that only need files/config/session state | Changed `withProject` default to no git; explicit git init test still opts into no-git fixture | 8.011s | 5.180s | keep | Median from 5 targeted runs because first run was cold/noisy. | -| Provider plugin filter test waits on plugin dependency readiness setup | Marked local plugin dependencies ready using the existing fixture helper | 7.543s | 6.366s | keep | Median from 3 targeted runs; matches neighboring plugin provider test setup. | -| HTTP provider tests generate local plugins without dependency-ready fixture state | Marked generated `.opencode` plugin fixtures dependency-ready | 7.905s | 2.980s | keep | Median from 3 targeted runs; avoids unrelated plugin dependency setup in route tests. | -| TUI plugin lifecycle timeout coverage waits the full production cleanup timeout | Added optional runtime dispose timeout override and used 25ms in the timeout test | 7.330s | 1.507s | keep | Median from 3 targeted runs; production default remains 5000ms. | -| Skill tool test initializes git even though it only reads local skill files | Removed `git: true` from the temporary directory fixture | 2.320s | 1.425s | keep | Single targeted rerun; still exercises skill discovery, permission request, and bundled file output. | -| Prompt shell semantics tests initialize git though they only assert shell/session behavior | Removed `git: true` from shell-focused prompt fixtures while preserving config setup | 26.930s | 23.400s | keep | Three targeted reruns passed after the change: 23.80s, 23.55s, 23.40s. | -| Remaining prompt behavior tests mostly do not require repository state | Removed git setup from loop/cancel/reference/error fixtures; kept it for `#` filename lookup | 23.400s | 16.590s | keep | Targeted prompt file passes; `#` filename test failed without git and was restored. | -| Session processor effect tests do not require repository state | Removed git setup from all processor-effect temp server fixtures | 12.500s | 9.230s | keep | Two targeted reruns passed after the change: 9.61s, 9.23s. | +| Hypothesis | Change | Before | After | Decision | Notes | +| --------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------- | --------- | ------- | -------- | --------------------------------------------------------------------------------------------------------------------------- | +| Repeated full-suite runs are too expensive for discovery | Switched full-suite benchmark to one run and added per-file profiler | ~250s/run | pending | keep | Bun has no slowest-test reporter in this version; profile files directly. | +| Plugin install concurrency test spends time spawning more workers than needed to exercise lock contention | Reduced worker counts from 12/10/8 to 6/6/5; kept `holdMs: 30` | 7.800s | 6.204s | keep | Median from 3 targeted runs; still covers concurrent cross-process writes to server, server+tui, and existing json config. | +| `httpapi-listen` PTY route tests pay for git repositories they do not assert on | Removed `git: true` from temp dirs while keeping config setup | 10.554s | 7.818s | keep | Median from 3 targeted runs; HTTP routes, tickets, websocket upgrade, restart, and no-auth paths still pass. | +| `workspace.waitForSync` timeout test waits the full production timeout | Added optional timeout parameter defaulting to production timeout; timeout test uses 25ms | 12.949s | 8.305s | keep | Median from 3 targeted runs; production callers keep the 5000ms default. | +| `config.test` waits after dependencies even though `.gitignore` is written synchronously | Removed obsolete 1000ms sleep from writable `OPENCODE_CONFIG_DIR` test | 10.270s | 9.433s | keep | Median from 5 targeted runs because one run was noisy; simpler test and no fixed sleep. | +| SDK parity helpers create git repos for tests that only need files/config/session state | Changed `withProject` default to no git; explicit git init test still opts into no-git fixture | 8.011s | 5.180s | keep | Median from 5 targeted runs because first run was cold/noisy. | +| Provider plugin filter test waits on plugin dependency readiness setup | Marked local plugin dependencies ready using the existing fixture helper | 7.543s | 6.366s | keep | Median from 3 targeted runs; matches neighboring plugin provider test setup. | +| HTTP provider tests generate local plugins without dependency-ready fixture state | Marked generated `.opencode` plugin fixtures dependency-ready | 7.905s | 2.980s | keep | Median from 3 targeted runs; avoids unrelated plugin dependency setup in route tests. | +| TUI plugin lifecycle timeout coverage waits the full production cleanup timeout | Added optional runtime dispose timeout override and used 25ms in the timeout test | 7.330s | 1.507s | keep | Median from 3 targeted runs; production default remains 5000ms. | +| Skill tool test initializes git even though it only reads local skill files | Removed `git: true` from the temporary directory fixture | 2.320s | 1.425s | keep | Single targeted rerun; still exercises skill discovery, permission request, and bundled file output. | +| Prompt shell semantics tests initialize git though they only assert shell/session behavior | Removed `git: true` from shell-focused prompt fixtures while preserving config setup | 26.930s | 23.400s | keep | Three targeted reruns passed after the change: 23.80s, 23.55s, 23.40s. | +| Remaining prompt behavior tests mostly do not require repository state | Removed git setup from loop/cancel/reference/error fixtures; kept it for `#` filename lookup | 23.400s | 16.590s | keep | Targeted prompt file passes; `#` filename test failed without git and was restored. | +| Session processor effect tests do not require repository state | Removed git setup from all processor-effect temp server fixtures | 12.500s | 9.230s | keep | Two targeted reruns passed after the change: 9.61s, 9.23s. | +| HTTP listen PTY ticket tests restart the same listener topology twice | Folded directory-scoped ticket regression into the broader unsafe-ticket test | 7.051s | 6.170s | keep | Two targeted reruns passed after the change: 6.76s, 6.17s; still covers mint failure and successful same-directory upgrade. | ## Profiling Results From 5c3bfee6304fcd1c7a0e1bfc3486aac2efafa436 Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Sat, 16 May 2026 10:53:36 -0400 Subject: [PATCH 18/19] perf: record updated test suite benchmark --- perf/test-suite.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/perf/test-suite.md b/perf/test-suite.md index 746544c4f7e..2b96ba23a61 100644 --- a/perf/test-suite.md +++ b/perf/test-suite.md @@ -103,11 +103,12 @@ Targeted 3-run baselines: | `test/provider/provider.test.ts` | 8.323, 7.543, 7.474 | 7.543 | Large serial file. | | `test/server/httpapi-cors.test.ts` | 2.621, 1.682, 1.518 | 1.682 | Not a standalone top target; initial 14s was mixed-scope noise/order effect. | -Final full-suite sanity check: +Full-suite sanity checks: -| Command | Result | -| -------------------- | -------: | -| `bun run bench:test` | 225.069s | +| Command | Result | Notes | +| -------------------- | -------: | -------------------------------------- | +| `bun run bench:test` | 225.069s | Before continuing prompt/session work. | +| `bun run bench:test` | 186.729s | After prompt, processor, and PTY wins. | ## Dead Ends From 6d328ad1ad385c165584bde984042c960d389e44 Mon Sep 17 00:00:00 2001 From: Kit Langton Date: Sat, 16 May 2026 11:58:42 -0400 Subject: [PATCH 19/19] test: restore prompt and vcs coverage --- packages/opencode/script/bench-test-suite.ts | 9 +++++ .../opencode/test/server/httpapi-sdk.test.ts | 5 +-- packages/opencode/test/session/prompt.test.ts | 34 ++++++++++++++----- perf/test-suite.md | 15 ++++---- 4 files changed, 46 insertions(+), 17 deletions(-) diff --git a/packages/opencode/script/bench-test-suite.ts b/packages/opencode/script/bench-test-suite.ts index 56fd84f5ed3..d49bff5a812 100644 --- a/packages/opencode/script/bench-test-suite.ts +++ b/packages/opencode/script/bench-test-suite.ts @@ -2,6 +2,15 @@ const warmups = Number(Bun.env.BENCH_WARMUPS ?? 0) const runs = Number(Bun.env.BENCH_RUNS ?? 1) const timings: number[] = [] +if (!Number.isInteger(warmups) || warmups < 0) { + console.error("BENCH_WARMUPS must be a non-negative integer") + process.exit(1) +} +if (!Number.isInteger(runs) || runs < 1) { + console.error("BENCH_RUNS must be a positive integer") + process.exit(1) +} + for (const index of Array.from({ length: warmups + runs }, (_, index) => index)) { const measured = index >= warmups const label = measured ? `run ${index - warmups + 1}/${runs}` : `warmup ${index + 1}/${warmups}` diff --git a/packages/opencode/test/server/httpapi-sdk.test.ts b/packages/opencode/test/server/httpapi-sdk.test.ts index 443c3041b9c..891b9676425 100644 --- a/packages/opencode/test/server/httpapi-sdk.test.ts +++ b/packages/opencode/test/server/httpapi-sdk.test.ts @@ -516,7 +516,7 @@ describe("HttpApi SDK", () => { ) serverPathParity("matches generated SDK instance read routes", (serverPath) => - withStandardProject(serverPath, ({ sdk, directory }) => + withProject(serverPath, { git: true, setup: writeStandardFiles }, ({ sdk, directory }) => Effect.gen(function* () { const project = yield* capture(() => sdk.project.current()) const projects = yield* capture(() => sdk.project.list()) @@ -561,6 +561,7 @@ describe("HttpApi SDK", () => { foundFile: JSON.stringify(findFiles.data).includes("hello.txt"), foundText: JSON.stringify(findText.data ?? null).includes("sdk-parity"), listedFile: JSON.stringify(files.data).includes("hello.txt"), + vcs: { hasBranch: typeof record(vcs.data).branch === "string" }, } }), ), @@ -823,7 +824,7 @@ describe("HttpApi SDK", () => { ) serverPathParity("matches generated SDK project git initialization", (serverPath) => - withProject(serverPath, { git: false }, ({ sdk, directory }) => + withProject(serverPath, {}, ({ sdk, directory }) => Effect.gen(function* () { const before = yield* capture(() => sdk.project.current()) const init = yield* capture(() => sdk.project.initGit()) diff --git a/packages/opencode/test/session/prompt.test.ts b/packages/opencode/test/session/prompt.test.ts index 14046b59cf8..95aebb8f189 100644 --- a/packages/opencode/test/session/prompt.test.ts +++ b/packages/opencode/test/session/prompt.test.ts @@ -334,6 +334,18 @@ const pollWithTimeout = ( }), ) +// Busy status is the deterministic signal that shell cancellation will hit a registered runner. +const waitForBusy = (sessionID: SessionID, duration: Duration.Input = "2 seconds") => + pollWithTimeout( + Effect.gen(function* () { + const status = yield* SessionStatus.Service + const s = yield* status.get(sessionID) + return s.type === "busy" ? (true as const) : undefined + }), + `session ${sessionID} never became busy`, + duration, + ) + const hasBash = Effect.sync(() => Bun.which("bash") !== null) const deferredAsPromise = (deferred: Deferred.Deferred): PromiseLike => ({ @@ -1080,6 +1092,7 @@ it.instance( expect(exitA.value.info.id).toBe(exitB.value.info.id) } }), + { git: true }, 3_000, ) @@ -1446,7 +1459,7 @@ it.instance( const sh = yield* prompt .shell({ sessionID: chat.id, agent: "build", command: "sleep 0.2" }) .pipe(Effect.forkChild) - yield* Effect.sleep(50) + yield* waitForBusy(chat.id) const loop = yield* prompt.loop({ sessionID: chat.id }).pipe(Effect.forkChild) yield* Effect.sleep(50) @@ -1463,6 +1476,7 @@ it.instance( } expect(yield* llm.calls).toBe(1) }), + { git: true }, 3_000, ) @@ -1482,7 +1496,7 @@ it.instance( const sh = yield* prompt .shell({ sessionID: chat.id, agent: "build", command: "sleep 0.2" }) .pipe(Effect.forkChild) - yield* Effect.sleep(50) + yield* waitForBusy(chat.id) const a = yield* prompt.loop({ sessionID: chat.id }).pipe(Effect.forkChild) const b = yield* prompt.loop({ sessionID: chat.id }).pipe(Effect.forkChild) @@ -1501,6 +1515,7 @@ it.instance( } expect(yield* llm.calls).toBe(1) }), + { git: true }, 3_000, ) @@ -1547,7 +1562,7 @@ unix( const sh = yield* prompt .shell({ sessionID: chat.id, agent: "build", command: "sleep 30" }) .pipe(Effect.forkChild) - yield* Effect.sleep(50) + yield* waitForBusy(chat.id) yield* prompt.cancel(chat.id) @@ -1567,7 +1582,7 @@ unix( } }), ), - { config: cfg }, + { git: true, config: cfg }, 30_000, ) @@ -1611,7 +1626,7 @@ unix( } }), ), - { config: cfg }, + { git: true, config: cfg }, 30_000, ) @@ -1660,6 +1675,7 @@ unix( expect(tool.state.output).toMatch(/Full output saved to:\s+\S+/) expect(tool.state.output).not.toContain("Tool execution aborted") }), + { git: true }, 30_000, ) @@ -1670,7 +1686,7 @@ unix( const { prompt, chat } = yield* boot() const sh = yield* prompt.shell({ sessionID: chat.id, agent: "build", command: "sleep 30" }).pipe(Effect.forkChild) - yield* Effect.sleep(50) + yield* waitForBusy(chat.id) const loop = yield* prompt.loop({ sessionID: chat.id }).pipe(Effect.forkChild) yield* Effect.sleep(50) @@ -1686,7 +1702,7 @@ unix( yield* Fiber.await(sh) }), - { config: cfg }, + { git: true, config: cfg }, 30_000, ) @@ -1700,7 +1716,7 @@ unix( const a = yield* prompt .shell({ sessionID: chat.id, agent: "build", command: "sleep 30" }) .pipe(Effect.forkChild) - yield* Effect.sleep(50) + yield* waitForBusy(chat.id) const exit = yield* prompt.shell({ sessionID: chat.id, agent: "build", command: "echo hi" }).pipe(Effect.exit) expect(Exit.isFailure(exit)).toBe(true) @@ -1712,7 +1728,7 @@ unix( yield* Fiber.await(a) }), ), - { config: cfg }, + { git: true, config: cfg }, 30_000, ) diff --git a/perf/test-suite.md b/perf/test-suite.md index 2b96ba23a61..39c26906df9 100644 --- a/perf/test-suite.md +++ b/perf/test-suite.md @@ -64,7 +64,7 @@ Repeated setup work, long sleeps/timeouts, serial integration tests, filesystem/ | TUI plugin lifecycle timeout coverage waits the full production cleanup timeout | Added optional runtime dispose timeout override and used 25ms in the timeout test | 7.330s | 1.507s | keep | Median from 3 targeted runs; production default remains 5000ms. | | Skill tool test initializes git even though it only reads local skill files | Removed `git: true` from the temporary directory fixture | 2.320s | 1.425s | keep | Single targeted rerun; still exercises skill discovery, permission request, and bundled file output. | | Prompt shell semantics tests initialize git though they only assert shell/session behavior | Removed `git: true` from shell-focused prompt fixtures while preserving config setup | 26.930s | 23.400s | keep | Three targeted reruns passed after the change: 23.80s, 23.55s, 23.40s. | -| Remaining prompt behavior tests mostly do not require repository state | Removed git setup from loop/cancel/reference/error fixtures; kept it for `#` filename lookup | 23.400s | 16.590s | keep | Targeted prompt file passes; `#` filename test failed without git and was restored. | +| Remaining prompt behavior tests mostly do not require repository state | Removed git setup from safe loop/reference/error fixtures; restored shell queue/cancel cases | 23.400s | 19.610s | keep | Safety review found shell runner readiness depends on git-backed setup in several tests; current single rerun passes. | | Session processor effect tests do not require repository state | Removed git setup from all processor-effect temp server fixtures | 12.500s | 9.230s | keep | Two targeted reruns passed after the change: 9.61s, 9.23s. | | HTTP listen PTY ticket tests restart the same listener topology twice | Folded directory-scoped ticket regression into the broader unsafe-ticket test | 7.051s | 6.170s | keep | Two targeted reruns passed after the change: 6.76s, 6.17s; still covers mint failure and successful same-directory upgrade. | @@ -76,7 +76,7 @@ Command shape: TEST_PROFILE_GLOB='test//**/*.test.ts' TEST_PROFILE_TOP=15 bun run profile:test ``` -Slowest files observed so far: +Initial slowest files observed during discovery: | File | Seconds | Scope | | ----------------------------------------- | ------: | ------------- | @@ -91,6 +91,8 @@ Slowest files observed so far: | `test/cli/tui/plugin-lifecycle.test.ts` | 7.330 | cli/tui | | `test/file/index.test.ts` | 7.214 | file | +This table is historical profiling input, not the current ranking after kept changes. + Targeted 3-run baselines: | File | Runs | Median | Notes | @@ -105,10 +107,11 @@ Targeted 3-run baselines: Full-suite sanity checks: -| Command | Result | Notes | -| -------------------- | -------: | -------------------------------------- | -| `bun run bench:test` | 225.069s | Before continuing prompt/session work. | -| `bun run bench:test` | 186.729s | After prompt, processor, and PTY wins. | +| Command | Result | Notes | +| -------------------- | -------: | -------------------------------------------------------------------- | +| `bun run bench:test` | 225.069s | Before continuing prompt/session work. | +| `bun run bench:test` | 186.729s | After prompt, processor, and PTY wins before safety review restores. | +| `bun run bench:test` | 202.317s | After restoring prompt shell coverage and SDK VCS parity coverage. | ## Dead Ends