diff --git a/.github/workflows/ts-ci.yml b/.github/workflows/ts-ci.yml index 2faef4b..4cd762d 100644 --- a/.github/workflows/ts-ci.yml +++ b/.github/workflows/ts-ci.yml @@ -2,10 +2,10 @@ name: TypeScript CI on: push: - branches: [main, "feat/ts-migration**"] + branches: [main, "feat/ts-migration**", "feat/ts-integration-tests"] paths: ["typescript/**"] pull_request: - branches: [main, "feat/ts-migration**"] + branches: [main, "feat/ts-migration**", "feat/ts-integration-tests"] paths: ["typescript/**"] jobs: @@ -25,3 +25,5 @@ jobs: - run: npm run typecheck - run: npm run lint - run: npm run test + - run: npm run test:integration + - run: npm run test:performance diff --git a/Makefile b/Makefile index 15dbc63..926fc1e 100644 --- a/Makefile +++ b/Makefile @@ -70,6 +70,22 @@ test-performance: docker-test-build @echo "Running performance tests (benchmarks, memory, stability)..." @docker run --rm --init $(DOCKER_TEST_IMAGE) uv run pytest tests/performance/ +# TypeScript test targets (no Docker required - use bash/cat/echo from host) +.PHONY: test-ts-integration +test-ts-integration: + @echo "Running TypeScript integration tests..." + @cd typescript && npm run test:integration + +.PHONY: test-ts-performance +test-ts-performance: + @echo "Running TypeScript performance tests..." + @cd typescript && npm run test:performance + +.PHONY: test-ts-all +test-ts-all: + @echo "Running all TypeScript tests (unit + integration + performance)..." + @cd typescript && npm run test:all + .PHONY: test-coverage test-coverage: docker-test-build @echo "Running tests with coverage analysis..." diff --git a/typescript/__tests__/integration/pty_integration.test.ts b/typescript/__tests__/integration/pty_integration.test.ts new file mode 100644 index 0000000..9f5d67c --- /dev/null +++ b/typescript/__tests__/integration/pty_integration.test.ts @@ -0,0 +1,207 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import * as os from "node:os"; +import * as fs from "node:fs"; +import * as path from "node:path"; +import { PtyHandler } from "../../src/interactive/pty_handler.js"; +import { PTYError } from "../../src/interactive/models.js"; +import { Settings } from "../../src/config/settings.js"; + +const SETTINGS = new Settings({ ENABLE_COMMAND_VALIDATION: false }); + +function makeCollector(): { chunks: string[]; output: () => string } { + const chunks: string[] = []; + return { chunks, output: () => chunks.join("") }; +} + +async function waitUntil(check: () => boolean, deadlineMs: number, intervalMs = 50): Promise { + const deadline = Date.now() + deadlineMs; + while (Date.now() < deadline) { + if (check()) return true; + await new Promise((r) => setTimeout(r, intervalMs)); + } + return check(); +} + +describe("PTY Integration", () => { + let handler: PtyHandler; + + beforeEach(() => { + handler = new PtyHandler(SETTINGS); + }); + + afterEach(async () => { + await handler.cleanup(); + }); + + it("basic echo command", async () => { + const { chunks, output } = makeCollector(); + await handler.createSession(["echo", "hello world"], undefined, undefined, (d) => chunks.push(d)); + const code = await handler.waitForExit(5000); + expect(code).toBe(0); + expect(output()).toContain("hello world"); + }); + + it("interactive input/output via cat", async () => { + const { chunks, output } = makeCollector(); + await handler.createSession(["cat"], undefined, undefined, (d) => chunks.push(d)); + expect(handler.isProcessAlive()).toBe(true); + + handler.writeInput("test line\n"); + + const found = await waitUntil(() => output().includes("test line"), 5000); + expect(found).toBe(true); + expect(output()).toContain("test line"); + + await handler.terminateProcess(false); + expect(handler.isProcessAlive()).toBe(false); + }); + + it("multi-line output from bash", async () => { + const { chunks, output } = makeCollector(); + await handler.createSession( + ["bash", "-c", "echo line1; echo line2; echo line3"], + undefined, + undefined, + (d) => chunks.push(d), + ); + const code = await handler.waitForExit(5000); + expect(code).toBe(0); + expect(output()).toContain("line1"); + expect(output()).toContain("line2"); + expect(output()).toContain("line3"); + }); + + it("process lifecycle: spawn, verify alive, terminate, cleanup idempotent", async () => { + await handler.createSession(["sleep", "10"]); + expect(handler.isProcessAlive()).toBe(true); + + await handler.terminateProcess(false); + expect(handler.isProcessAlive()).toBe(false); + + await expect(handler.cleanup()).resolves.not.toThrow(); + await expect(handler.cleanup()).resolves.not.toThrow(); + }); + + it("error handling: invalid command throws PTYError or exits non-zero", async () => { + // node-pty behavior is platform-specific: on Linux spawn() throws synchronously + // for a missing executable; on macOS the process entry is created and onExit + // fires immediately with a non-zero code. + try { + await handler.createSession(["/nonexistent/command_xyz"]); + const code = await handler.waitForExit(2000); + expect(code).not.toBeNull(); + expect(code).not.toBe(0); + } catch (e) { + expect(e).toBeInstanceOf(PTYError); + } + }); + + it("concurrent read/write via cat", async () => { + const { chunks, output } = makeCollector(); + await handler.createSession(["cat"], undefined, undefined, (d) => chunks.push(d)); + + const lines = ["concurrent line 0", "concurrent line 1", "concurrent line 2"]; + + const writer = (async () => { + for (const line of lines) { + handler.writeInput(`${line}\n`); + await new Promise((r) => setTimeout(r, 50)); + } + await handler.terminateProcess(false); + })(); + + const reader = (async () => { + const found = await waitUntil( + () => lines.every((l) => output().includes(l)), + 8000, + ); + return found; + })(); + + const [, allFound] = await Promise.all([writer, reader]); + expect(allFound).toBe(true); + for (const line of lines) { + expect(output()).toContain(line); + } + }); + + it("environment variable and working directory", async () => { + const tmpDir = os.tmpdir(); + const tmpFile = path.join(tmpDir, `pty_test_${Date.now()}.txt`); + fs.writeFileSync(tmpFile, ""); + const filename = path.basename(tmpFile); + + try { + const { chunks, output } = makeCollector(); + await handler.createSession( + ["bash", "-c", `echo $TEST_VAR; ls ${filename}`], + { TEST_VAR: "env_test_value" }, + tmpDir, + (d) => chunks.push(d), + ); + const code = await handler.waitForExit(5000); + expect(code).toBe(0); + expect(output()).toContain("env_test_value"); + expect(output()).toContain(filename); + } finally { + fs.unlinkSync(tmpFile); + } + }); + + it("large output handling: 100-line bash loop", async () => { + const { chunks, output } = makeCollector(); + await handler.createSession( + ["bash", "-c", 'for i in $(seq 1 100); do echo "This is line $i with some extra text"; done'], + undefined, + undefined, + (d) => chunks.push(d), + ); + const code = await handler.waitForExit(10000); + expect(code).toBe(0); + expect(output().length).toBeGreaterThan(1000); + expect(output()).toContain("line 1"); + expect(output()).toContain("line 100"); + }); + + it("timeout behavior: waitForExit returns null before process exits", async () => { + await handler.createSession(["sleep", "10"]); + + const result = await handler.waitForExit(100); + expect(result).toBeNull(); + expect(handler.isProcessAlive()).toBe(true); + + await handler.terminateProcess(true); + await waitUntil(() => !handler.isProcessAlive(), 3000); + expect(handler.isProcessAlive()).toBe(false); + }); + + it("sequential sessions: two independent sessions", async () => { + const handler1 = new PtyHandler(SETTINGS); + const collector1 = makeCollector(); + await handler1.createSession( + ["echo", "first_session_output"], + undefined, + undefined, + (d) => collector1.chunks.push(d), + ); + const code1 = await handler1.waitForExit(5000); + await handler1.cleanup(); + + expect(code1).toBe(0); + expect(collector1.output()).toContain("first_session_output"); + + const collector2 = makeCollector(); + await handler.createSession( + ["echo", "second_session_output"], + undefined, + undefined, + (d) => collector2.chunks.push(d), + ); + const code2 = await handler.waitForExit(5000); + + expect(code2).toBe(0); + expect(collector2.output()).toContain("second_session_output"); + expect(collector1.output()).not.toContain("second_session_output"); + expect(collector2.output()).not.toContain("first_session_output"); + }); +}); diff --git a/typescript/__tests__/interactive/pty_handler.test.ts b/typescript/__tests__/interactive/pty_handler.test.ts index 99b60b3..a2be8f4 100644 --- a/typescript/__tests__/interactive/pty_handler.test.ts +++ b/typescript/__tests__/interactive/pty_handler.test.ts @@ -114,7 +114,7 @@ describe("PtyHandler", () => { await expect(handler.createSession(["openroad"])).rejects.toThrow( "could not be started", ); - await expect(handler.createSession(["openroad"])).rejects.toThrow("PATH="); + await expect(handler.createSession(["openroad"])).rejects.toThrow("PATH is"); await expect(handler.createSession(["openroad"])).rejects.toThrow("spawn-helper"); }); diff --git a/typescript/__tests__/performance/benchmarks.test.ts b/typescript/__tests__/performance/benchmarks.test.ts new file mode 100644 index 0000000..ece730b --- /dev/null +++ b/typescript/__tests__/performance/benchmarks.test.ts @@ -0,0 +1,296 @@ +import { describe, it, expect, beforeEach, vi } from "vitest"; +import type { Mock } from "vitest"; +import { OpenROADManager } from "../../src/core/manager.js"; +import { InteractiveSession } from "../../src/interactive/session.js"; +import { CircularBuffer } from "../../src/interactive/buffer.js"; +import { SessionState } from "../../src/core/models.js"; +import type { SessionDetailedMetrics } from "../../src/core/models.js"; + +vi.mock("../../src/interactive/session.js", () => ({ InteractiveSession: vi.fn() })); + +interface MockSession { + sessionId: string; + lastActivity: Date; + checkAlive: Mock; + start: Mock; + sendCommand: Mock; + readOutput: Mock; + getInfo: Mock; + getDetailedMetrics: Mock; + getCommandHistory: Mock; + isIdleTimeout: Mock; + setSessionTimeout: Mock; + terminate: Mock; + cleanup: Mock; +} + +function makeMockSession(sessionId: string): MockSession { + const metrics: SessionDetailedMetrics = { + sessionId, + state: SessionState.ACTIVE, + isAlive: true, + createdAt: new Date().toISOString(), + lastActivity: new Date().toISOString(), + uptimeSeconds: 1, + idleSeconds: 0, + commands: { totalExecuted: 0, currentCount: 0, historyLength: 0 }, + performance: { totalCpuTime: 0, peakMemoryMb: 1, currentMemoryMb: 1 }, + buffer: { currentSize: 0, maxSize: 1024, utilizationPercent: 0 }, + timeout: { configuredSeconds: null, isTimedOut: false }, + }; + return { + sessionId, + lastActivity: new Date(), + checkAlive: vi.fn().mockReturnValue(true), + start: vi.fn().mockResolvedValue(undefined), + sendCommand: vi.fn().mockResolvedValue(undefined), + readOutput: vi.fn().mockResolvedValue({ + output: "ok", + sessionId, + timestamp: new Date().toISOString(), + executionTime: 0.001, + commandCount: 1, + bufferSize: 0, + error: null, + }), + getInfo: vi.fn().mockResolvedValue({ + sessionId, + createdAt: new Date().toISOString(), + isAlive: true, + commandCount: 0, + bufferSize: 0, + uptimeSeconds: 1, + state: SessionState.ACTIVE, + }), + getDetailedMetrics: vi.fn().mockResolvedValue(metrics), + getCommandHistory: vi.fn().mockReturnValue([]), + isIdleTimeout: vi.fn().mockReturnValue(false), + setSessionTimeout: vi.fn(), + terminate: vi.fn().mockResolvedValue(undefined), + cleanup: vi.fn().mockResolvedValue(undefined), + }; +} + +function percentile(sorted: number[], p: number): number { + const idx = Math.max(0, Math.ceil(p * sorted.length) - 1); + return sorted[idx]!; +} + +const MockedSession = vi.mocked(InteractiveSession); + +describe("Performance Benchmarks", () => { + let manager: OpenROADManager; + let created: MockSession[]; + + beforeEach(() => { + vi.clearAllMocks(); + created = []; + MockedSession.mockImplementation(function (this: unknown, sessionId: string) { + const mock = makeMockSession(sessionId); + created.push(mock); + return mock as unknown as InteractiveSession; + } as unknown as (sessionId: string) => InteractiveSession); + manager = new OpenROADManager(100); + }); + + it("session creation latency: avg < 25ms, max < 50ms", async () => { + const N = 10; + const latencies: number[] = []; + for (let i = 0; i < N; i++) { + const t0 = performance.now(); + await manager.createSession({ sessionId: `lat-${i}` }); + latencies.push(performance.now() - t0); + } + const avg = latencies.reduce((a, b) => a + b, 0) / N; + const max = Math.max(...latencies); + console.log(` session creation: avg=${avg.toFixed(2)}ms max=${max.toFixed(2)}ms`); + expect(avg).toBeLessThan(25); + expect(max).toBeLessThan(50); + }); + + it("output streaming throughput: > 10 MB/s, < 5s for 1MB", async () => { + const BUF_SIZE = 10 * 1024 * 1024; + const CHUNK = "x".repeat(1024); + const CHUNKS = 1000; + const buf = new CircularBuffer(BUF_SIZE); + + const t0 = performance.now(); + for (let i = 0; i < CHUNKS; i++) { + await buf.append(CHUNK); + } + await buf.drainAll(); + const durationS = (performance.now() - t0) / 1000; + const throughputMBs = (CHUNKS * 1024) / (1024 * 1024) / durationS; + console.log(` streaming: ${throughputMBs.toFixed(1)} MB/s in ${durationS.toFixed(3)}s`); + expect(throughputMBs).toBeGreaterThan(10); + expect(durationS).toBeLessThan(5); + }); + + it("concurrent session scalability: 50 sessions, p95 < 100ms, p99 < 200ms", async () => { + const N = 50; + const ids = await Promise.all( + Array.from({ length: N }, (_, i) => manager.createSession({ sessionId: `con-${i}` })), + ); + expect(new Set(ids).size).toBe(N); + + const latencies: number[] = []; + await Promise.all( + ids.map(async (id) => { + const t0 = performance.now(); + await manager.executeCommand(id, "version"); + latencies.push(performance.now() - t0); + }), + ); + latencies.sort((a, b) => a - b); + const p95 = percentile(latencies, 0.95); + const p99 = percentile(latencies, 0.99); + console.log(` concurrent: p95=${p95.toFixed(2)}ms p99=${p99.toFixed(2)}ms`); + expect(p95).toBeLessThan(100); + expect(p99).toBeLessThan(200); + }); + + it("memory usage profiling: heap increase < 5x expected for 10 sessions x 1MB", async () => { + const N = 10; + const BUF_SIZE = 1024 * 1024; + const FILL = BUF_SIZE * 0.1; + const CHUNK = "m".repeat(1024); + + if (typeof global.gc === "function") global.gc(); + const before = process.memoryUsage().heapUsed / (1024 * 1024); + + const bufs: CircularBuffer[] = []; + for (let i = 0; i < N; i++) { + const buf = new CircularBuffer(BUF_SIZE); + const writes = Math.floor(FILL / 1024); + for (let j = 0; j < writes; j++) await buf.append(CHUNK); + bufs.push(buf); + } + + if (typeof global.gc === "function") global.gc(); + const after = process.memoryUsage().heapUsed / (1024 * 1024); + const increase = after - before; + const expectedMb = (N * FILL) / (1024 * 1024); + console.log(` memory: increased ${increase.toFixed(1)}MB, expected ${expectedMb.toFixed(1)}MB`); + expect(increase).toBeLessThan(expectedMb * 5); + + for (const buf of bufs) await buf.drainAll(); + }); + + it("buffer overflow performance: > 1000 ops/sec, final size <= capacity", async () => { + const BUF_SIZE = 1024 * 1024; + const CHUNK = "o".repeat(1024); + const WRITES = 2048; + + const buf = new CircularBuffer(BUF_SIZE); + const t0 = performance.now(); + for (let i = 0; i < WRITES; i++) await buf.append(CHUNK); + const durationS = (performance.now() - t0) / 1000; + const opsPerSec = WRITES / durationS; + + const drained = await buf.drainAll(); + const finalSize = drained.reduce((s, c) => s + c.length, 0); + + console.log(` buffer overflow: ${opsPerSec.toFixed(0)} ops/s, final=${(finalSize / 1024).toFixed(1)}KB`); + expect(opsPerSec).toBeGreaterThan(1000); + expect(durationS).toBeLessThan(5); + expect(finalSize).toBeLessThanOrEqual(BUF_SIZE); + }); + + it("command execution latency: avg < 10ms, p95 < 20ms, max < 50ms", async () => { + const N = 50; + const id = await manager.createSession({ sessionId: "cmd-lat" }); + const latencies: number[] = []; + for (let i = 0; i < N; i++) { + const t0 = performance.now(); + await manager.executeCommand(id, `puts ${i}`); + latencies.push(performance.now() - t0); + } + latencies.sort((a, b) => a - b); + const avg = latencies.reduce((a, b) => a + b, 0) / N; + const p95 = percentile(latencies, 0.95); + const max = latencies[latencies.length - 1]!; + console.log(` cmd latency: avg=${avg.toFixed(2)}ms p95=${p95.toFixed(2)}ms max=${max.toFixed(2)}ms`); + expect(avg).toBeLessThan(10); + expect(p95).toBeLessThan(20); + expect(max).toBeLessThan(50); + }); +}); + +describe("Stress Tests", () => { + let manager: OpenROADManager; + let created: MockSession[]; + + beforeEach(() => { + vi.clearAllMocks(); + created = []; + MockedSession.mockImplementation(function (this: unknown, sessionId: string) { + const mock = makeMockSession(sessionId); + created.push(mock); + return mock as unknown as InteractiveSession; + } as unknown as (sessionId: string) => InteractiveSession); + manager = new OpenROADManager(200); + }); + + it("long running session stability: 1000 commands in batches of 50", async () => { + const id = await manager.createSession({ sessionId: "stability" }); + const TOTAL = 1000; + const BATCH = 50; + + for (let i = 0; i < TOTAL / BATCH; i++) { + await Promise.all( + Array.from({ length: BATCH }, (_, j) => + manager.executeCommand(id, `puts ${i * BATCH + j}`), + ), + ); + } + + const session = created[0]!; + expect(session.readOutput.mock.calls.length).toBe(TOTAL); + expect(session.checkAlive()).toBe(true); + }); + + it("resource exhaustion: session limit is enforced once maxSessions is reached, cleanupAll < 10s", async () => { + const LIMIT = 20; + const exhaustedManager = new OpenROADManager(LIMIT); + const MAX = 100; // attempts well past LIMIT so the cap is actually exercised + let succeeded = 0; + + for (let i = 0; i < MAX; i++) { + try { + await exhaustedManager.createSession({ sessionId: `res-${i}` }); + succeeded++; + } catch { + break; + } + } + expect(succeeded).toBe(LIMIT); + await expect( + exhaustedManager.createSession({ sessionId: "res-overflow" }), + ).rejects.toThrow(/maximum session limit reached/i); + + const t0 = performance.now(); + await exhaustedManager.cleanupAll(); + const cleanupMs = performance.now() - t0; + console.log(` cleanup of ${succeeded} sessions: ${cleanupMs.toFixed(0)}ms`); + expect(cleanupMs).toBeLessThan(10000); + }); + + it("large output through 128KB buffer: final size <= capacity, duration < 2s", async () => { + const BUF_SIZE = 128 * 1024; + const CHUNK_SIZE = 16 * 1024; + const WRITES = 320; + const CHUNK = "L".repeat(CHUNK_SIZE); + + const buf = new CircularBuffer(BUF_SIZE); + const t0 = performance.now(); + for (let i = 0; i < WRITES; i++) await buf.append(CHUNK); + const durationS = (performance.now() - t0) / 1000; + + const drained = await buf.drainAll(); + const finalSize = drained.reduce((s, c) => s + c.length, 0); + + console.log(` large output: final=${(finalSize / 1024).toFixed(1)}KB duration=${durationS.toFixed(3)}s`); + expect(finalSize).toBeLessThanOrEqual(BUF_SIZE); + expect(durationS).toBeLessThan(2); + }); +}); diff --git a/typescript/__tests__/performance/memory_monitoring.test.ts b/typescript/__tests__/performance/memory_monitoring.test.ts new file mode 100644 index 0000000..266a162 --- /dev/null +++ b/typescript/__tests__/performance/memory_monitoring.test.ts @@ -0,0 +1,198 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import * as fs from "node:fs"; +import { OpenROADManager } from "../../src/core/manager.js"; +import { InteractiveSession } from "../../src/interactive/session.js"; +import { SessionState } from "../../src/core/models.js"; + +vi.mock("../../src/interactive/session.js", () => ({ InteractiveSession: vi.fn() })); + +function makeMockSession(sessionId: string) { + return { + sessionId, + lastActivity: new Date(), + checkAlive: vi.fn().mockReturnValue(true), + start: vi.fn().mockResolvedValue(undefined), + sendCommand: vi.fn().mockResolvedValue(undefined), + readOutput: vi.fn().mockResolvedValue({ + output: "ok", + sessionId, + timestamp: new Date().toISOString(), + executionTime: 0.001, + commandCount: 1, + bufferSize: 0, + error: null, + }), + getInfo: vi.fn().mockResolvedValue({ + sessionId, + createdAt: new Date().toISOString(), + isAlive: true, + commandCount: 0, + bufferSize: 0, + uptimeSeconds: 1, + state: SessionState.ACTIVE, + }), + getDetailedMetrics: vi.fn().mockResolvedValue(null), + getCommandHistory: vi.fn().mockReturnValue([]), + isIdleTimeout: vi.fn().mockReturnValue(false), + setSessionTimeout: vi.fn(), + terminate: vi.fn().mockResolvedValue(undefined), + cleanup: vi.fn().mockResolvedValue(undefined), + }; +} + +class MemoryMonitor { + private snapshots: Array<{ label: string; rssMb: number; heapUsedMb: number }> = []; + + takeSnapshot(label: string): void { + // Hint to V8 GC if available (requires --expose-gc flag; safe to omit) + if (typeof global.gc === "function") global.gc(); + const mem = process.memoryUsage(); + this.snapshots.push({ + label, + rssMb: mem.rss / (1024 * 1024), + heapUsedMb: mem.heapUsed / (1024 * 1024), + }); + } + + rss(startLabel: string, endLabel: string): number { + const start = this.snapshots.find((s) => s.label === startLabel); + const end = this.snapshots.find((s) => s.label === endLabel); + if (!start || !end) throw new Error(`Snapshot not found: ${startLabel} or ${endLabel}`); + return end.rssMb - start.rssMb; + } + + heap(startLabel: string, endLabel: string): number { + const start = this.snapshots.find((s) => s.label === startLabel); + const end = this.snapshots.find((s) => s.label === endLabel); + if (!start || !end) throw new Error(`Snapshot not found: ${startLabel} or ${endLabel}`); + return end.heapUsedMb - start.heapUsedMb; + } +} + +function getFdCount(): number { + try { + return fs.readdirSync("/proc/self/fd").length; + } catch { + return -1; // /proc not available (macOS) + } +} + +const MockedSession = vi.mocked(InteractiveSession); + +describe("Memory Monitoring", () => { + let manager: OpenROADManager; + + beforeEach(() => { + vi.clearAllMocks(); + MockedSession.mockImplementation(function (this: unknown, sessionId: string) { + return makeMockSession(sessionId) as unknown as InteractiveSession; + } as unknown as (sessionId: string) => InteractiveSession); + manager = new OpenROADManager(100); + }); + + it("session creation memory leak: 10 cycles x 5 sessions, RSS growth < 12MB", async () => { + const mon = new MemoryMonitor(); + mon.takeSnapshot("start"); + + for (let cycle = 0; cycle < 10; cycle++) { + const ids: string[] = []; + for (let i = 0; i < 5; i++) { + const id = await manager.createSession({ sessionId: `mem-${cycle}-${i}` }); + ids.push(id); + } + for (const id of ids) { + await manager.terminateSession(id); + } + } + + mon.takeSnapshot("end"); + const rssDiff = mon.rss("start", "end"); + console.log(` session creation leak: RSS diff ${rssDiff.toFixed(1)}MB`); + // RSS granularity is one OS page (typically 4MB); allow 12MB headroom + expect(rssDiff).toBeLessThan(12); + }); + + it("long running session memory: 1000 ops, growth < 25MB, leaked <= 5MB after cleanup", async () => { + const mon = new MemoryMonitor(); + mon.takeSnapshot("start"); + + const id = await manager.createSession({ sessionId: "long-run" }); + const BATCH = 50; + for (let i = 0; i < 1000 / BATCH; i++) { + await Promise.all(Array.from({ length: BATCH }, (_, j) => manager.executeCommand(id, `puts ${i * BATCH + j}`))); + } + + mon.takeSnapshot("during"); + const duringDiff = mon.rss("start", "during"); + expect(duringDiff).toBeLessThan(25); + + await manager.terminateSession(id); + mon.takeSnapshot("after"); + // Use heap (not RSS) for the leak check - heap tracks actual JS allocations + const leaked = mon.heap("start", "after"); + console.log(` long running: during +${duringDiff.toFixed(1)}MB, heap leaked ${leaked.toFixed(1)}MB`); + expect(leaked).toBeLessThanOrEqual(5); + }); + + it("concurrent session memory: 20 sessions, < 2MB per session", async () => { + const mon = new MemoryMonitor(); + mon.takeSnapshot("before"); + + const N = 20; + const ids = await Promise.all( + Array.from({ length: N }, (_, i) => manager.createSession({ sessionId: `conc-${i}` })), + ); + await Promise.all(ids.map((id) => manager.executeCommand(id, "version"))); + + mon.takeSnapshot("loaded"); + const totalDiff = mon.rss("before", "loaded"); + const perSession = totalDiff / N; + console.log(` concurrent sessions: total +${totalDiff.toFixed(1)}MB = ${perSession.toFixed(2)}MB/session`); + // Allow generous headroom - RSS reporting is coarse at process level + expect(perSession).toBeLessThan(2); + + await manager.cleanupAll(); + }); + + it("file descriptor leak detection (Linux only)", async () => { + const fdBefore = getFdCount(); + if (fdBefore === -1) { + console.log(" FD tracking skipped (not Linux)"); + return; + } + + const CYCLES = 20; + for (let i = 0; i < CYCLES; i++) { + const id = await manager.createSession({ sessionId: `fd-${i}` }); + await manager.terminateSession(id); + } + + const fdAfter = getFdCount(); + const fdDiff = fdAfter - fdBefore; + console.log(` FD leak: before=${fdBefore} after=${fdAfter} diff=${fdDiff}`); + // Small tolerance: OS may cache a few FDs, node-pty releases them on cleanup + expect(fdDiff).toBeLessThanOrEqual(5); + }); + + it.skip("stability simulation: 24-hour scaled run (enable manually, takes ~24s)", async () => { + // Simulates 24 hours of operation at 1s/hour, 100 ops/hour. + // Enable to validate memory_growth_rate < 0.2 MB/hour. + const mon = new MemoryMonitor(); + mon.takeSnapshot("start"); + + const id = await manager.createSession({ sessionId: "stability" }); + for (let hour = 0; hour < 24; hour++) { + for (let op = 0; op < 100; op++) { + await manager.executeCommand(id, `puts ${op}`); + } + await new Promise((r) => setTimeout(r, 1000)); + } + + mon.takeSnapshot("end"); + const totalGrowth = mon.rss("start", "end"); + const growthPerHour = totalGrowth / 24; + console.log(` stability: ${totalGrowth.toFixed(1)}MB total = ${growthPerHour.toFixed(2)}MB/hour`); + expect(growthPerHour).toBeLessThan(0.2); + await manager.cleanupAll(); + }); +}); diff --git a/typescript/__tests__/performance/response_sizes.test.ts b/typescript/__tests__/performance/response_sizes.test.ts new file mode 100644 index 0000000..efe6c95 --- /dev/null +++ b/typescript/__tests__/performance/response_sizes.test.ts @@ -0,0 +1,146 @@ +import { describe, it, expect, vi, beforeEach } from "vitest"; +import { toSnakeCase } from "../../src/tools/base.js"; +import type { InteractiveExecResult } from "../../src/core/models.js"; +import { InteractiveSessionListResult, SessionState } from "../../src/core/models.js"; +import { ListSessionsTool } from "../../src/tools/interactive.js"; +import { OpenROADManager } from "../../src/core/manager.js"; +import { InteractiveSession } from "../../src/interactive/session.js"; + +vi.mock("../../src/interactive/session.js", () => ({ InteractiveSession: vi.fn() })); + +const EXEC_RESULT_TOKEN_BUDGET = 80; +const SESSION_LIST_EMPTY_BUDGET = 30; +const COMPACT_SAVINGS_MIN_PCT = 10.0; + +/** Rough token estimate: 1 token ~4 chars (OpenAI tokenizer rule of thumb). */ +function tokenEstimate(text: string): number { + return Math.max(1, Math.floor(text.length / 4)); +} + +/** Serialize a domain object to the wire format via toSnakeCase. */ +function wireFormat(obj: Record): string { + return JSON.stringify(toSnakeCase(obj)); +} + +function minimalExecResult(): InteractiveExecResult { + return { + output: "% ok", + sessionId: "s1", + timestamp: new Date().toISOString(), + executionTime: 0.01, + commandCount: 0, + bufferSize: 0, + error: null, + }; +} + +function typicalExecResult(): InteractiveExecResult { + return { + output: "OpenROAD 2.0-1234-g1234abcd\nOpenROAD, (C) 2021 The OpenROAD Project.", + sessionId: "abc12345", + timestamp: new Date().toISOString(), + executionTime: 0.123, + commandCount: 5, + bufferSize: 1024, + error: null, + }; +} + +describe("Token Efficiency", () => { + it("exec result minimal token budget < 80 tokens", () => { + const compact = wireFormat(minimalExecResult() as unknown as Record); + const tokens = tokenEstimate(compact); + console.log(` minimal exec result: ${compact.length} chars = ~${tokens} tokens`); + expect(tokens).toBeLessThan(EXEC_RESULT_TOKEN_BUDGET); + }); + + it("exec result typical token budget < 80 tokens", () => { + const compact = wireFormat(typicalExecResult() as unknown as Record); + const tokens = tokenEstimate(compact); + console.log(` typical exec result: ${compact.length} chars = ~${tokens} tokens`); + expect(tokens).toBeLessThan(EXEC_RESULT_TOKEN_BUDGET); + }); + + it("empty session list token budget < 30 tokens", () => { + const result = InteractiveSessionListResult.parse({ + sessions: [], + totalCount: 0, + activeCount: 0, + error: null, + }); + const compact = wireFormat(result as unknown as Record); + const tokens = tokenEstimate(compact); + console.log(` empty session list: ${compact.length} chars = ~${tokens} tokens`); + expect(tokens).toBeLessThan(SESSION_LIST_EMPTY_BUDGET); + }); + + it("compact JSON saves >= 10% tokens vs pretty-print", () => { + const obj = typicalExecResult() as unknown as Record; + const compact = wireFormat(obj); + const pretty = JSON.stringify(toSnakeCase(obj), null, 2); + const savingsPct = ((pretty.length - compact.length) / pretty.length) * 100; + console.log(` compactness savings: ${savingsPct.toFixed(1)}% (${pretty.length} -> ${compact.length} chars)`); + expect(savingsPct).toBeGreaterThanOrEqual(COMPACT_SAVINGS_MIN_PCT); + }); + + it("compact JSON has no newlines or double spaces", () => { + const compact = wireFormat(typicalExecResult() as unknown as Record); + expect(compact).not.toContain("\n"); + expect(compact).not.toContain(" "); + }); + + it("pinned token counts remain stable across refactors", () => { + // These counts pin the exact wire format. If they change, a field was added, + // renamed, or removed - update after verifying the schema change is intentional. + const minimal = wireFormat(minimalExecResult() as unknown as Record); + const emptyList = wireFormat( + InteractiveSessionListResult.parse({ sessions: [], totalCount: 0, activeCount: 0, error: null }) as unknown as Record, + ); + // Token counts (floor(len/4)) measured on first run - update if schema changes. + expect(tokenEstimate(minimal)).toBeLessThanOrEqual(45); // ~37 chars baseline + expect(tokenEstimate(emptyList)).toBeLessThanOrEqual(20); // ~57 chars baseline + }); +}); + +describe("Live Tool Compactness", () => { + const MockedSession = vi.mocked(InteractiveSession); + let manager: OpenROADManager; + + beforeEach(() => { + vi.clearAllMocks(); + MockedSession.mockImplementation(function (this: unknown, sessionId: string) { + return { + sessionId, + lastActivity: new Date(), + checkAlive: vi.fn().mockReturnValue(true), + start: vi.fn().mockResolvedValue(undefined), + getInfo: vi.fn().mockResolvedValue({ + sessionId, + createdAt: new Date().toISOString(), + isAlive: true, + commandCount: 0, + bufferSize: 0, + uptimeSeconds: 1, + state: SessionState.ACTIVE, + }), + terminate: vi.fn().mockResolvedValue(undefined), + cleanup: vi.fn().mockResolvedValue(undefined), + isIdleTimeout: vi.fn().mockReturnValue(false), + } as unknown as InteractiveSession; + } as unknown as (sessionId: string) => InteractiveSession); + manager = new OpenROADManager(10); + }); + + it("ListSessionsTool output is compact JSON for a populated session list", async () => { + await manager.createSession({ sessionId: "live-1" }); + const tool = new ListSessionsTool(manager); + const raw = await tool.execute(); + + expect(raw).not.toContain("\n"); + expect(raw).not.toContain(" "); + const parsed = JSON.parse(raw) as { sessions: unknown[]; total_count: number; active_count: number }; + expect(parsed.sessions).toHaveLength(1); + expect(parsed.active_count).toBe(1); + console.log(` list sessions (1 active): ${raw.length} chars = ~${tokenEstimate(raw)} tokens`); + }); +}); diff --git a/typescript/package.json b/typescript/package.json index ea612bb..de2c451 100644 --- a/typescript/package.json +++ b/typescript/package.json @@ -19,6 +19,9 @@ "lint": "eslint .", "test": "vitest run", "test:coverage": "vitest run --coverage", + "test:integration": "vitest run --config vitest.config.integration.ts", + "test:performance": "vitest run --config vitest.config.performance.ts", + "test:all": "npm run test && npm run test:integration && npm run test:performance", "dev": "tsx src/main.ts" }, "license": "BSD-3-Clause", diff --git a/typescript/scripts/fix-node-pty.cjs b/typescript/scripts/fix-node-pty.cjs index b510e1f..66ad805 100644 --- a/typescript/scripts/fix-node-pty.cjs +++ b/typescript/scripts/fix-node-pty.cjs @@ -19,7 +19,8 @@ for (const entry of fs.readdirSync(prebuildsDir, { withFileTypes: true })) { try { const mode = fs.statSync(helper).mode & 0o777; if ((mode & 0o111) === 0) { - fs.chmodSync(helper, mode | 0o755); + // Only add exec bits; OR'ing 0o755 would also broaden read/write access. + fs.chmodSync(helper, mode | 0o111); } } catch { // Best effort; PTY spawn will surface a clearer error at runtime. diff --git a/typescript/src/interactive/pty_handler.ts b/typescript/src/interactive/pty_handler.ts index cae3282..ecd93b6 100644 --- a/typescript/src/interactive/pty_handler.ts +++ b/typescript/src/interactive/pty_handler.ts @@ -107,17 +107,19 @@ export class PtyHandler { } catch (e) { if (e instanceof PTYError) throw e; const raw = e instanceof Error ? e.message : String(e); - const pathValue = process.env.PATH ?? ""; + // Don't echo the full PATH into an error that may reach clients; just + // note whether it was set so the operator knows where to look in logs. + const pathHint = process.env.PATH ? "set" : "empty"; if (/posix_spawnp failed|ENOENT|command not found/i.test(raw)) { throw new PTYError( `Failed to create PTY session: executable '${executable}' could not be started. ` + `Most likely '${executable}' is missing from PATH or not executable ` + - `(PATH=${JSON.stringify(pathValue)}). Rarely, node-pty's spawn-helper binary lost its ` + + `(PATH is ${pathHint}). Rarely, node-pty's spawn-helper binary lost its ` + `exec bit — the postinstall script normally restores it; if not (e.g. install ran ` + `with --ignore-scripts), run: chmod +x node_modules/node-pty/prebuilds/*/spawn-helper`, ); } - throw new PTYError(`Failed to create PTY session: ${e}`); + throw new PTYError(`Failed to create PTY session: ${raw}`); } } diff --git a/typescript/src/server.ts b/typescript/src/server.ts index 37f786f..5f4ff1e 100644 --- a/typescript/src/server.ts +++ b/typescript/src/server.ts @@ -262,10 +262,20 @@ export async function shutdownOpenroad(): Promise { } } +// Cap request bodies so a large or malicious POST can't buffer unbounded +// memory. 1 MB is generous for JSON-RPC control messages. +const MAX_BODY_BYTES = 1_000_000; + async function readJsonBody(req: IncomingMessage): Promise { const chunks: Buffer[] = []; + let total = 0; for await (const chunk of req) { - chunks.push(chunk as Buffer); + const buf = chunk as Buffer; + total += buf.length; + if (total > MAX_BODY_BYTES) { + throw new Error(`Request body too large (>${MAX_BODY_BYTES} bytes)`); + } + chunks.push(buf); } const raw = Buffer.concat(chunks).toString("utf8"); if (raw.length === 0) return undefined; diff --git a/typescript/vitest.config.integration.ts b/typescript/vitest.config.integration.ts new file mode 100644 index 0000000..5aaba96 --- /dev/null +++ b/typescript/vitest.config.integration.ts @@ -0,0 +1,12 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + globals: true, + environment: "node", + pool: "forks", + include: ["__tests__/integration/**/*.test.ts"], + testTimeout: 30000, + hookTimeout: 10000, + }, +}); diff --git a/typescript/vitest.config.performance.ts b/typescript/vitest.config.performance.ts new file mode 100644 index 0000000..40a23c1 --- /dev/null +++ b/typescript/vitest.config.performance.ts @@ -0,0 +1,16 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + globals: true, + environment: "node", + pool: "forks", + // --expose-gc lets MemoryMonitor.takeSnapshot() force a GC pass before + // measuring, so RSS/heap diffs in memory_monitoring.test.ts are not noise + // from a GC that just had not run yet. + poolOptions: { forks: { execArgv: ["--expose-gc"] } }, + include: ["__tests__/performance/**/*.test.ts"], + testTimeout: 60000, + hookTimeout: 15000, + }, +}); diff --git a/typescript/vitest.config.ts b/typescript/vitest.config.ts index 68321e1..1f41431 100644 --- a/typescript/vitest.config.ts +++ b/typescript/vitest.config.ts @@ -6,6 +6,7 @@ export default defineConfig({ environment: "node", pool: "forks", include: ["__tests__/**/*.test.ts"], + exclude: ["__tests__/integration/**", "__tests__/performance/**"], coverage: { provider: "v8", thresholds: { lines: 80 },