From 985483cf8a63a4d5477bc764c455b719e6d836f0 Mon Sep 17 00:00:00 2001 From: John Watson Date: Mon, 12 Jan 2026 22:37:26 +0000 Subject: [PATCH] feat: add soak test execution to luminork --- bin/si-luminork-api-tests/deno.json | 5 +- bin/si-luminork-api-tests/src/soak-utils.ts | 415 ++++++++++++++++++ .../tests/{ => core}/change-sets.test.ts | 4 +- .../tests/{ => core}/components.test.ts | 4 +- .../tests/{ => core}/schemas.test.ts | 6 +- .../tests/{ => core}/system-status.test.ts | 2 +- .../tests/soak/component-creation.test.ts | 276 ++++++++++++ 7 files changed, 703 insertions(+), 9 deletions(-) create mode 100644 bin/si-luminork-api-tests/src/soak-utils.ts rename bin/si-luminork-api-tests/tests/{ => core}/change-sets.test.ts (98%) rename bin/si-luminork-api-tests/tests/{ => core}/components.test.ts (98%) rename bin/si-luminork-api-tests/tests/{ => core}/schemas.test.ts (99%) rename bin/si-luminork-api-tests/tests/{ => core}/system-status.test.ts (95%) create mode 100644 bin/si-luminork-api-tests/tests/soak/component-creation.test.ts diff --git a/bin/si-luminork-api-tests/deno.json b/bin/si-luminork-api-tests/deno.json index 6eb0fccc0e..86b6a5591e 100644 --- a/bin/si-luminork-api-tests/deno.json +++ b/bin/si-luminork-api-tests/deno.json @@ -20,11 +20,14 @@ "proseWrap": "preserve" }, "test": { - "include": ["tests/"] + "include": ["tests/core/"] }, "tasks": { "test": "deno test --allow-env --allow-net --allow-read", "test:watch": "deno test --allow-env --allow-net --allow-read --watch", + "test:core": "deno test --allow-env --allow-net --allow-read tests/core/", + "soak": "deno test --allow-env --allow-net --allow-read tests/soak/", + "soak:components": "deno test --allow-env --allow-net --allow-read tests/soak/component-creation.test.ts", "debug": "deno run --allow-env --allow-net --allow-read debug-api.ts", "format": "deno fmt", "lint": "deno lint" diff --git a/bin/si-luminork-api-tests/src/soak-utils.ts b/bin/si-luminork-api-tests/src/soak-utils.ts new file mode 100644 index 0000000000..80b36217ce --- /dev/null +++ b/bin/si-luminork-api-tests/src/soak-utils.ts @@ -0,0 +1,415 @@ +/** + * Soak Test Utilities + * + * Framework for running performance and load tests against the Luminork API. + * Provides progress tracking, metrics collection, and reporting similar to + * standalone soak test scripts. + */ + +export interface SoakTestConfig { + /** Total number of operations to perform (or unlimited if 0) */ + iterations: number; + /** Maximum test duration in milliseconds (or unlimited if 0) */ + duration?: number; + /** Number of parallel test threads (each gets own changeset) */ + parallelThreads?: number; + /** Number of concurrent operations within each thread (default: 1) */ + parallelism?: number; + /** Whether to cleanup resources after test completion (default: true) */ + cleanup?: boolean; + /** Report progress every N operations (default: 100) */ + reportInterval?: number; +} + +export interface SoakMetrics { + /** Total number of operations attempted across all threads */ + totalOperations: number; + /** Number of successful operations across all threads */ + successfulOperations: number; + /** Number of failed operations across all threads */ + failedOperations: number; + /** Average time per operation in milliseconds */ + averageOperationTimeMs: number; + /** Operations per second across all threads */ + operationsPerSecond: number; + /** Total test duration in milliseconds */ + totalDurationMs: number; + /** Test start time */ + startTime: Date; + /** Test end time */ + endTime: Date; + /** Collection of errors encountered */ + errors: Array<{ thread: number; operation: number; error: string; timestamp: Date }>; + /** Per-thread metrics */ + threadMetrics: SoakThreadMetrics[]; +} + +export interface SoakThreadMetrics { + /** Thread identifier */ + threadId: number; + /** Changeset ID used by this thread */ + changeSetId: string; + /** Operations completed by this thread */ + operations: number; + /** Successful operations by this thread */ + successful: number; + /** Failed operations by this thread */ + failed: number; + /** Thread execution time in milliseconds */ + durationMs: number; +} + +export interface SoakOperationResult { + success: boolean; + durationMs: number; + error?: string; + data?: unknown; +} + +export type SoakOperation = (iteration: number, changeSetId: string) => Promise; +export type SoakChangeSetFactory = (threadId: number) => Promise; + +/** + * Soak Test Runner + * + * Executes parallel threads of operations with duration and iteration limits. + * Each thread gets its own changeset and runs operations until either: + * 1. Maximum iterations reached, OR 2. Maximum duration reached + */ +export class SoakTestRunner { + private config: Required; + + constructor(config: SoakTestConfig) { + this.config = { + parallelThreads: 1, + parallelism: 1, + duration: 0, // unlimited by default + cleanup: true, + reportInterval: 100, + ...config, + }; + } + + /** + * Run a soak test with parallel threads, each using its own changeset + */ + async run( + changeSetFactory: SoakChangeSetFactory, + operation: SoakOperation + ): Promise { + const startTime = new Date(); + const allErrors: Array<{ thread: number; operation: number; error: string; timestamp: Date }> = []; + const threadMetrics: SoakThreadMetrics[] = []; + + console.log('============================================================'); + console.log(`๐Ÿš€ Starting soak test with ${this.config.parallelThreads} parallel threads`); + console.log(`โฑ๏ธ Max duration: ${this.config.duration ? (this.config.duration / 1000) + 's' : 'unlimited'}`); + console.log(`๐Ÿ”„ Max iterations per thread: ${this.config.iterations || 'unlimited'}`); + + // Create and start all threads + const threadPromises: Promise[] = []; + + for (let threadId = 1; threadId <= this.config.parallelThreads; threadId++) { + const threadPromise = this.runThread(threadId, changeSetFactory, operation, startTime); + threadPromises.push(threadPromise); + } + + try { + // Wait for all threads to complete + const results = await Promise.all(threadPromises); + threadMetrics.push(...results); + + // Aggregate results from all threads + const endTime = new Date(); + const totalDurationMs = endTime.getTime() - startTime.getTime(); + + const totalOperations = threadMetrics.reduce((sum, tm) => sum + tm.operations, 0); + const successfulOperations = threadMetrics.reduce((sum, tm) => sum + tm.successful, 0); + const failedOperations = threadMetrics.reduce((sum, tm) => sum + tm.failed, 0); + + // Calculate average operation time across all threads + const allThreadDurations = threadMetrics.map(tm => tm.durationMs); + const averageOperationTimeMs = allThreadDurations.length > 0 + ? allThreadDurations.reduce((sum, duration) => sum + duration, 0) / totalOperations + : 0; + + const operationsPerSecond = totalDurationMs > 0 + ? (successfulOperations / totalDurationMs) * 1000 + : 0; + + const metrics: SoakMetrics = { + totalOperations, + successfulOperations, + failedOperations, + averageOperationTimeMs, + operationsPerSecond, + totalDurationMs, + startTime, + endTime, + errors: allErrors, + threadMetrics, + }; + + this.reportFinalResults(metrics); + return metrics; + + } catch (error) { + console.error(`โŒ Soak test failed: ${error instanceof Error ? error.message : String(error)}`); + throw error; + } + } + + /** + * Run a single thread with its own changeset + */ + private async runThread( + threadId: number, + changeSetFactory: SoakChangeSetFactory, + operation: SoakOperation, + globalStartTime: Date + ): Promise { + const threadStartTime = Date.now(); + let operations = 0; + let successful = 0; + let failed = 0; + + // Create changeset for this thread + const changeSetId = await changeSetFactory(threadId); + console.log(`๐Ÿ“ Thread ${threadId}: Using changeset ${changeSetId}`); + + // Create duration timeout if specified + const hasTimeLimit = this.config.duration > 0; + const timeLimit = hasTimeLimit ? globalStartTime.getTime() + this.config.duration : Number.MAX_SAFE_INTEGER; + + // Run operations until iteration limit OR time limit reached + while ( + (this.config.iterations === 0 || operations < this.config.iterations) && + (!hasTimeLimit || Date.now() < timeLimit) + ) { + operations++; + + try { + const operationStart = Date.now(); + await operation(operations, changeSetId); + const operationEnd = Date.now(); + + successful++; + + // Progress reporting (throttled) + if (operations % this.config.reportInterval === 0) { + const elapsed = (Date.now() - globalStartTime.getTime()) / 1000; + const threadOpsPerSec = successful / elapsed; + console.log(`โœ… Thread ${threadId}: [${operations}] ops | ${threadOpsPerSec.toFixed(1)} ops/sec | ${elapsed.toFixed(0)}s elapsed`); + } + + } catch (error) { + failed++; + const errorMessage = error instanceof Error ? error.message : String(error); + console.warn(`โŒ Thread ${threadId}: Operation ${operations} failed: ${errorMessage}`); + + // Don't let errors stop the thread - continue with next operation + } + + // Small delay to prevent overwhelming the API + await sleep(10); + } + + const threadEndTime = Date.now(); + const durationMs = threadEndTime - threadStartTime; + + const threadMetric: SoakThreadMetrics = { + threadId, + changeSetId, + operations, + successful, + failed, + durationMs, + }; + + console.log(`๐Ÿ Thread ${threadId}: Completed ${operations} operations (${successful} successful, ${failed} failed) in ${(durationMs/1000).toFixed(1)}s`); + + return threadMetric; + } + + /** + * Report final test results (matching Python script style) + */ + private reportFinalResults(metrics: SoakMetrics): void { + console.log('\n============================================================'); + console.log('๐Ÿ“Š FINAL RESULTS'); + console.log('============================================================'); + console.log(`Total Threads: ${metrics.threadMetrics.length}`); + console.log(`Operations Completed: ${metrics.successfulOperations}/${metrics.totalOperations}`); + console.log(`Failed Operations: ${metrics.failedOperations}`); + console.log(`Success Rate: ${((metrics.successfulOperations / metrics.totalOperations) * 100).toFixed(1)}%`); + console.log(''); + console.log('โฑ๏ธ TIMING BREAKDOWN:'); + console.log(`Total Test Runtime: ${(metrics.totalDurationMs / 1000).toFixed(1)}s`); + console.log(`Operations per Second (aggregate): ${metrics.operationsPerSecond.toFixed(2)}`); + console.log(''); + + // Per-thread breakdown + console.log('๐Ÿงต PER-THREAD BREAKDOWN:'); + metrics.threadMetrics.forEach(tm => { + const threadOpsPerSec = tm.durationMs > 0 ? (tm.successful / tm.durationMs) * 1000 : 0; + console.log(` Thread ${tm.threadId}: ${tm.operations} ops (${tm.successful}โœ…/${tm.failed}โŒ) | ${threadOpsPerSec.toFixed(2)} ops/sec | ${(tm.durationMs/1000).toFixed(1)}s | CS: ${tm.changeSetId}`); + }); + + if (metrics.successfulOperations === metrics.totalOperations) { + console.log(`\n๐ŸŽ‰ SUCCESS: All ${metrics.totalOperations} operations completed successfully!`); + } else { + console.log(`\nโš ๏ธ PARTIAL SUCCESS: ${metrics.successfulOperations} of ${metrics.totalOperations} operations completed`); + } + + if (metrics.errors.length > 0) { + console.log('\nโŒ ERRORS ENCOUNTERED:'); + metrics.errors.slice(0, 5).forEach((error, index) => { + console.log(`${index + 1}. Thread ${error.thread}, Op ${error.operation}: ${error.error}`); + }); + if (metrics.errors.length > 5) { + console.log(`... and ${metrics.errors.length - 5} more errors`); + } + } + } +} + +/** + * Simple semaphore for controlling parallelism + */ +class Semaphore { + private permits: number; + private waiting: Array<() => void> = []; + + constructor(permits: number) { + this.permits = permits; + } + + async acquire(): Promise<() => void> { + return new Promise((resolve) => { + if (this.permits > 0) { + this.permits--; + resolve(() => this.release()); + } else { + this.waiting.push(() => { + this.permits--; + resolve(() => this.release()); + }); + } + }); + } + + private release(): void { + this.permits++; + if (this.waiting.length > 0) { + const next = this.waiting.shift()!; + next(); + } + } +} + +/** + * Utility function to generate a unique test name with timestamp + */ +export function generateSoakTestName(prefix = 'soak-test'): string { + const timestamp = new Date().toISOString().replace(/[^0-9]/g, '').slice(0, 14); + const random = Math.random().toString(36).substring(2, 8); + return `${prefix}-${timestamp}-${random}`; +} + +/** + * Utility function to sleep for a specified number of milliseconds + */ +export function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** + * Parse command line arguments for soak test configuration + */ +export function parseSoakArgs(): SoakTestConfig & { testName?: string } { + const args = Deno.args; + const config: SoakTestConfig & { testName?: string } = { + iterations: 100, // default iterations + duration: 0, // unlimited by default + parallelThreads: 1, + cleanup: true, + reportInterval: 25, + }; + + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + const nextArg = args[i + 1]; + + switch (arg) { + case '--iterations': + case '-i': + if (nextArg && !isNaN(parseInt(nextArg))) { + config.iterations = parseInt(nextArg); + i++; // skip next arg + } + break; + case '--duration': + case '-d': + if (nextArg) { + const durationStr = nextArg; + let duration = 0; + + // Parse duration with units: 5m, 30s, 300000ms, etc. + if (durationStr.endsWith('m')) { + duration = parseInt(durationStr.slice(0, -1)) * 60 * 1000; + } else if (durationStr.endsWith('s')) { + duration = parseInt(durationStr.slice(0, -1)) * 1000; + } else if (durationStr.endsWith('ms')) { + duration = parseInt(durationStr.slice(0, -2)); + } else { + duration = parseInt(durationStr) * 1000; // assume seconds + } + + config.duration = duration; + i++; // skip next arg + } + break; + case '--threads': + case '-t': + if (nextArg && !isNaN(parseInt(nextArg))) { + config.parallelThreads = parseInt(nextArg); + i++; // skip next arg + } + break; + case '--test': + if (nextArg) { + config.testName = nextArg; + i++; // skip next arg + } + break; + case '--report-interval': + if (nextArg && !isNaN(parseInt(nextArg))) { + config.reportInterval = parseInt(nextArg); + i++; // skip next arg + } + break; + case '--no-cleanup': + config.cleanup = false; + break; + } + } + + return config; +} + +/** + * Print help for soak test command line arguments + */ +export function printSoakHelp(): void { + console.log('Soak Test Arguments:'); + console.log(' --iterations, -i Maximum iterations per thread (default: 100)'); + console.log(' --duration, -d