From 2c6e1c4606bc9051ab3ae89cf228339af04250ce Mon Sep 17 00:00:00 2001 From: hawkeyexl Date: Tue, 21 Oct 2025 22:49:29 -0400 Subject: [PATCH 1/5] Base wrapper for running tests via API --- src/index.js | 39 ++++++++++++++++++++++++++++++--------- src/tests.js | 6 ++++++ 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/src/index.js b/src/index.js index 100d0757..07d349f9 100644 --- a/src/index.js +++ b/src/index.js @@ -1,6 +1,8 @@ -const { detectTests, detectAndResolveTests } = require("doc-detective-resolver"); +const { + detectAndResolveTests, +} = require("doc-detective-resolver"); const { log, cleanTemp } = require("./utils"); -const { runSpecs } = require("./tests"); +const { runSpecs, runViaApi } = require("./tests"); const { telemetryNotice, sendTelemetry } = require("./telem"); exports.runTests = runTests; @@ -14,18 +16,37 @@ const supportMessage = ` ##########################################################################`; // Run tests defined in specifications and documentation source files. -async function runTests(config) { +async function runTests(config, options = {}) { + let resolvedTests; + + if (options.resolvedTests) { + resolvedTests = options.resolvedTests; + config = resolvedTests.config; + } + // Telemetry notice telemetryNotice(config); - const resolvedTests = await detectAndResolveTests({ config }); - if (!resolvedTests || resolvedTests.specs.length === 0) { - log(config, "warn", "Couldn't resolve any tests."); - return null; + if (!resolvedTests) { + resolvedTests = await detectAndResolveTests({ config }); + if (!resolvedTests || resolvedTests.specs.length === 0) { + log(config, "warn", "Couldn't resolve any tests."); + return null; + } } - // Run test specs - const results = await runSpecs({ resolvedTests }); + let results; + // If config.docDetectiveApi.key is set, run tests via API instead of locally + if (config.docDetectiveApi && config.docDetectiveApi.key) { + // Run test specs via API + results = await runSpecs({ + resolvedTests, + apiKey: config.docDetectiveApi.key, + }); + } else { + // Run test specs locally + results = await runSpecs({ resolvedTests }); + } log(config, "info", "RESULTS:"); log(config, "info", results); log(config, "info", "Cleaning up and finishing post-processing."); diff --git a/src/tests.js b/src/tests.js index d3af7740..b8f73ed1 100644 --- a/src/tests.js +++ b/src/tests.js @@ -28,6 +28,7 @@ const { resolveExpression } = require("./expressions"); const { getEnvironment, getAvailableApps } = require("./config"); exports.runSpecs = runSpecs; +exports.runViaApi = runViaApi; // exports.appiumStart = appiumStart; // exports.appiumIsReady = appiumIsReady; // exports.driverStart = driverStart; @@ -230,6 +231,11 @@ async function allowUnsafeSteps({ config }) { else return false; } +// Run specifications via API. +async function runViaApi({ resolvedTests, apiKey }) { + +} + /** * Orchestrates execution of resolved test specifications and returns a hierarchical run report. * From 488e6dc548d4e188b9de5d11eb17a6a40a811524 Mon Sep 17 00:00:00 2001 From: hawkeyexl Date: Tue, 21 Oct 2025 23:16:01 -0400 Subject: [PATCH 2/5] First API call! --- dev/index.js | 7 ++++++- package-lock.json | 43 +++++++++++++++++++++++++++++++++++-------- package.json | 4 ++-- src/index.js | 10 +++++----- src/tests.js | 16 +++++++++++++++- 5 files changed, 63 insertions(+), 17 deletions(-) diff --git a/dev/index.js b/dev/index.js index af43aa59..f59f7813 100644 --- a/dev/index.js +++ b/dev/index.js @@ -17,7 +17,12 @@ async function main() { name: "firefox", headless: false }] - }] + }], + integrations: { + docDetectiveApi: { + apiKey: process.env.KEY + } + } }; // console.log(json); result = await runTests(json); diff --git a/package-lock.json b/package-lock.json index f0028c13..1477d5f2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,7 +19,7 @@ "appium-geckodriver": "^1.4.3", "appium-safari-driver": "^3.5.25", "axios": "^1.12.2", - "doc-detective-common": "^3.4.0", + "doc-detective-common": "3.4.0-dev.1", "doc-detective-resolver": "^3.4.0", "dotenv": "^17.2.3", "geckodriver": "^5.0.0", @@ -27,7 +27,7 @@ "json-schema-faker": "^0.5.9", "pixelmatch": "^5.3.0", "pngjs": "^7.0.0", - "posthog-node": "^5.10.0", + "posthog-node": "^5.10.2", "sharp": "^0.34.4", "tree-kill": "^1.2.2", "webdriverio": "8.45.0" @@ -14068,9 +14068,9 @@ } }, "node_modules/doc-detective-common": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/doc-detective-common/-/doc-detective-common-3.4.0.tgz", - "integrity": "sha512-du9fVM0UWq8tQ59U11XbtrVBKCAo5C/AH2cjMU0WEZsYHIXHKo0W91ENXT86LYZlb5kcAykFioxDwk3zRqVWLw==", + "version": "3.4.0-dev.1", + "resolved": "https://registry.npmjs.org/doc-detective-common/-/doc-detective-common-3.4.0-dev.1.tgz", + "integrity": "sha512-FZ+tphH8EnNXPWW3sdYjCOnsdB1I22pTvu5IMhUtXBOclTllEVoN2+fObIUitZYXGN2jCrfuBp2OWMq0V7NtAg==", "license": "AGPL-3.0-only", "dependencies": { "@apidevtools/json-schema-ref-parser": "^14.2.1", @@ -14109,6 +14109,33 @@ "posthog-node": "^5.10.0" } }, + "node_modules/doc-detective-resolver/node_modules/doc-detective-common": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/doc-detective-common/-/doc-detective-common-3.4.0.tgz", + "integrity": "sha512-du9fVM0UWq8tQ59U11XbtrVBKCAo5C/AH2cjMU0WEZsYHIXHKo0W91ENXT86LYZlb5kcAykFioxDwk3zRqVWLw==", + "license": "AGPL-3.0-only", + "dependencies": { + "@apidevtools/json-schema-ref-parser": "^14.2.1", + "ajv": "^8.17.1", + "ajv-errors": "^3.0.0", + "ajv-formats": "^3.0.1", + "ajv-keywords": "^5.1.0", + "axios": "^1.12.2", + "yaml": "^2.8.1" + } + }, + "node_modules/doc-detective-resolver/node_modules/yaml": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.1.tgz", + "integrity": "sha512-lcYcMxX2PO9XMGvAJkJ3OsNMw+/7FKes7/hgerGUYWIoWu5j/+YQqcZr5JnPZWzOsEBgMbSbiSTn/dv/69Mkpw==", + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + } + }, "node_modules/dotenv": { "version": "17.2.3", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.3.tgz", @@ -17115,9 +17142,9 @@ } }, "node_modules/posthog-node": { - "version": "5.10.0", - "resolved": "https://registry.npmjs.org/posthog-node/-/posthog-node-5.10.0.tgz", - "integrity": "sha512-uNN+YUuOdbDSbDMGk/Wq57o2YBEH0Unu1kEq2PuYmqFmnu+oYsKyJBrb58VNwEuYsaXVJmk4FtbD+Tl8BT69+w==", + "version": "5.10.2", + "resolved": "https://registry.npmjs.org/posthog-node/-/posthog-node-5.10.2.tgz", + "integrity": "sha512-/C8dlu3bPH4BJ5C5bFJY3jzd2ZvP12N8dqeddTCatRTXYqfWjrLV8fqC1rn8t+zNZz1bx+cC2E4hfYF0vba8jw==", "license": "MIT", "dependencies": { "@posthog/core": "1.3.0" diff --git a/package.json b/package.json index 3f3ef0b0..fa391a0a 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "appium-geckodriver": "^1.4.3", "appium-safari-driver": "^3.5.25", "axios": "^1.12.2", - "doc-detective-common": "^3.4.0", + "doc-detective-common": "3.4.0-dev.1", "doc-detective-resolver": "^3.4.0", "dotenv": "^17.2.3", "geckodriver": "^5.0.0", @@ -44,7 +44,7 @@ "json-schema-faker": "^0.5.9", "pixelmatch": "^5.3.0", "pngjs": "^7.0.0", - "posthog-node": "^5.10.0", + "posthog-node": "^5.10.2", "sharp": "^0.34.4", "tree-kill": "^1.2.2", "webdriverio": "8.45.0" diff --git a/src/index.js b/src/index.js index 07d349f9..6ea1a619 100644 --- a/src/index.js +++ b/src/index.js @@ -18,6 +18,7 @@ const supportMessage = ` // Run tests defined in specifications and documentation source files. async function runTests(config, options = {}) { let resolvedTests; + let results; if (options.resolvedTests) { resolvedTests = options.resolvedTests; @@ -35,13 +36,12 @@ async function runTests(config, options = {}) { } } - let results; - // If config.docDetectiveApi.key is set, run tests via API instead of locally - if (config.docDetectiveApi && config.docDetectiveApi.key) { + // If config.integrations.docDetectiveApi.apiKey is set, run tests via API instead of locally + if (config.integrations && config.integrations.docDetectiveApi && config.integrations.docDetectiveApi.apiKey) { // Run test specs via API - results = await runSpecs({ + results = await runViaApi({ resolvedTests, - apiKey: config.docDetectiveApi.key, + apiKey: config.integrations.docDetectiveApi.apiKey, }); } else { // Run test specs locally diff --git a/src/tests.js b/src/tests.js index b8f73ed1..42658138 100644 --- a/src/tests.js +++ b/src/tests.js @@ -233,7 +233,21 @@ async function allowUnsafeSteps({ config }) { // Run specifications via API. async function runViaApi({ resolvedTests, apiKey }) { - + const baseUrl = process.env.DOC_DETECTIVE_API_URL || "https://api.doc-detective.com/v1"; + // Make an API request to create a test run + const apiUrl = `${baseUrl}/runs`; + const response = await axios.post( + apiUrl, + resolvedTests, + { + headers: { + "X-API-Key": apiKey, + "Content-Type": "application/json" + } + } + ); + + return {}; } /** From d72909c8ec51d52ef7effebdfc93b196127bba50 Mon Sep 17 00:00:00 2001 From: hawkeyexl Date: Thu, 23 Oct 2025 16:13:51 -0400 Subject: [PATCH 3/5] First pass at runViaApi() --- src/tests.js | 136 ++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 114 insertions(+), 22 deletions(-) diff --git a/src/tests.js b/src/tests.js index 42658138..311ccc37 100644 --- a/src/tests.js +++ b/src/tests.js @@ -55,7 +55,9 @@ function getDriverCapabilities({ runnerDetails, name, options }) { // Set Firefox capabilities switch (name) { case "firefox": - firefox = runnerDetails.availableApps.find((app) => app.name === "firefox"); + firefox = runnerDetails.availableApps.find( + (app) => app.name === "firefox" + ); if (!firefox) break; // Set args // Reference: https://wiki.mozilla.org/Firefox/CommandLineOptions @@ -83,7 +85,9 @@ function getDriverCapabilities({ runnerDetails, name, options }) { case "safari": // Set Safari capabilities if (runnerDetails.availableApps.find((app) => app.name === "safari")) { - safari = runnerDetails.availableApps.find((app) => app.name === "safari"); + safari = runnerDetails.availableApps.find( + (app) => app.name === "safari" + ); if (!safari) break; // Set capabilities capabilities = { @@ -182,7 +186,7 @@ function getDefaultBrowser({ runnerDetails }) { let browser = {}; const browserNames = ["firefox", "chrome", "safari"]; for (const name of browserNames) { - if (runnerDetails.availableApps.find(app => app.name === name)) { + if (runnerDetails.availableApps.find((app) => app.name === name)) { browser = { name }; break; } @@ -226,28 +230,111 @@ async function allowUnsafeSteps({ config }) { // If allowUnsafeSteps is set to false, return false else if (config.allowUnsafeSteps === false) return false; // if DOC_DETECTIVE.container is set to true, return true - else if (process.env.DOC_DETECTIVE && JSON.parse(process.env.DOC_DETECTIVE).container) return true; + else if ( + process.env.DOC_DETECTIVE && + JSON.parse(process.env.DOC_DETECTIVE).container + ) + return true; // If allowUnsafeSteps is not set, return false by default else return false; } // Run specifications via API. -async function runViaApi({ resolvedTests, apiKey }) { - const baseUrl = process.env.DOC_DETECTIVE_API_URL || "https://api.doc-detective.com/v1"; +async function runViaApi({ resolvedTests, apiKey, config = {} }) { + const baseUrl = + process.env.DOC_DETECTIVE_API_URL || "https://api.doc-detective.com/v1"; // Make an API request to create a test run const apiUrl = `${baseUrl}/runs`; - const response = await axios.post( - apiUrl, - resolvedTests, - { - headers: { - "X-API-Key": apiKey, - "Content-Type": "application/json" - } + + // Configure axios with proper timeout and connection handling + const axiosConfig = { + headers: { + "X-API-Key": apiKey, + "Content-Type": "application/json", + }, + // Prevent connection reuse issues with keep-alive + httpAgent: new (require("http").Agent)({ keepAlive: false }), + httpsAgent: new (require("https").Agent)({ keepAlive: false }), + }; + + // Create run + let createResponse; + try { + createResponse = await axios.post(apiUrl, resolvedTests, axiosConfig); + } catch (error) { + return { status: error.response.status, error: error.response.data.error }; + } + if (createResponse.status !== 201) { + return { status: createResponse.status, error: createResponse.data.error }; + } + const runId = createResponse.data.run.runId; + + // TODO: Add file uploads, if any + + // Start run + let startResponse; + try { + startResponse = await axios.post( + `${apiUrl}/${runId}/start`, + {}, + axiosConfig + ); + } catch (error) { + return { status: error.response.status, error: error.response.data.error }; + } + if (startResponse.status !== 200) { + return { status: startResponse.status, error: startResponse.data.error }; + } + + // Poll for results + const pollInterval = 5000; // 5 seconds in milliseconds + const pollIntervalVariance = 2000; // +/- 2 seconds + const maxWaitTime = (config.apiMaxWaitTime || 600) * 1000; // Default 10 minutes, converted to milliseconds + const startTime = Date.now(); + + let response; + while (true) { + // Check if we've exceeded the max wait time + if (Date.now() - startTime > maxWaitTime) { + return { + status: "TIMEOUT", + error: `Test execution exceeded maximum wait time of ${ + maxWaitTime / 1000 + } seconds`, + }; } - ); - return {}; + // Poll for results + try { + response = await axios.get(`${apiUrl}/${runId}`, axiosConfig); + } catch (error) { + return { + status: error.response.status, + error: error.response.data.error, + }; + } + + if (response.status !== 200) { + return { status: response.status, error: response.data.error }; + } + + // Check if the test run is complete + if (response.data.status === "completed") { + break; + } + + // Wait before polling again (with variance) + const variance = + Math.random() * pollIntervalVariance * 2 - pollIntervalVariance; + const waitTime = pollInterval + variance; + await new Promise((resolve) => setTimeout(resolve, waitTime)); + } + + // TODO: Handle file downloads/placement, if any + + const results = JSON.parse(response.data.report); + + return results; } /** @@ -400,7 +487,9 @@ async function runSpecs({ resolvedTests }) { log( config, "warning", - `Skipping context. The current system doesn't support this context: {"platform": "${context.platform}", "apps": ${JSON.stringify(context.apps)}}` + `Skipping context. The current system doesn't support this context: {"platform": "${ + context.platform + }", "apps": ${JSON.stringify(context.apps)}}` ); contextReport = { result: { status: "SKIPPED" }, ...contextReport }; report.summary.contexts.skipped++; @@ -496,7 +585,6 @@ async function runSpecs({ resolvedTests }) { if (!step.stepId) step.stepId = randomUUID(); log(config, "debug", `STEP:\n${JSON.stringify(step, null, 2)}`); - if (step.unsafe && runnerDetails.allowUnsafeSteps === false) { log( config, @@ -507,7 +595,7 @@ async function runSpecs({ resolvedTests }) { const stepReport = { ...step, result: "SKIPPED", - resultDescription: "Skipped because unsafe steps aren't allowed." + resultDescription: "Skipped because unsafe steps aren't allowed.", }; contextReport.steps.push(stepReport); report.summary.steps.skipped++; @@ -519,7 +607,7 @@ async function runSpecs({ resolvedTests }) { const stepReport = { ...step, result: "SKIPPED", - resultDescription: "Skipped due to previous failure in context." + resultDescription: "Skipped due to previous failure in context.", }; contextReport.steps.push(stepReport); report.summary.steps.skipped++; @@ -545,7 +633,11 @@ async function runSpecs({ resolvedTests }) { log( config, "debug", - `RESULT: ${stepResult.status}\n${JSON.stringify(stepResult, null, 2)}` + `RESULT: ${stepResult.status}\n${JSON.stringify( + stepResult, + null, + 2 + )}` ); stepResult.result = stepResult.status; @@ -830,4 +922,4 @@ async function driverStart(capabilities) { }); driver.state = { url: "", x: null, y: null }; return driver; -} \ No newline at end of file +} From 98e1f6f8ce004b64a65894a4c6dc86a756800343 Mon Sep 17 00:00:00 2001 From: hawkeyexl Date: Fri, 24 Oct 2025 23:05:03 -0400 Subject: [PATCH 4/5] Review feedback --- src/tests.js | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/tests.js b/src/tests.js index ed2bc6ad..5180794b 100644 --- a/src/tests.js +++ b/src/tests.js @@ -262,7 +262,7 @@ async function runViaApi({ resolvedTests, apiKey, config = {} }) { try { createResponse = await axios.post(apiUrl, resolvedTests, axiosConfig); } catch (error) { - return { status: error.response.status, error: error.response.data.error }; + return { status: error.response?.status, error: error.response?.data?.error }; } if (createResponse.status !== 201) { return { status: createResponse.status, error: createResponse.data.error }; @@ -280,7 +280,7 @@ async function runViaApi({ resolvedTests, apiKey, config = {} }) { axiosConfig ); } catch (error) { - return { status: error.response.status, error: error.response.data.error }; + return { status: error.response?.status, error: error.response?.data?.error }; } if (startResponse.status !== 200) { return { status: startResponse.status, error: startResponse.data.error }; @@ -289,7 +289,7 @@ async function runViaApi({ resolvedTests, apiKey, config = {} }) { // Poll for results const pollInterval = 5000; // 5 seconds in milliseconds const pollIntervalVariance = 2000; // +/- 2 seconds - const maxWaitTime = (config.apiMaxWaitTime || 600) * 1000; // Default 10 minutes, converted to milliseconds + const maxWaitTime = (config.apiMaxWaitTime || 600) * 1000; // Default 600 seconds (10 minutes), converted to milliseconds const startTime = Date.now(); let response; @@ -297,10 +297,9 @@ async function runViaApi({ resolvedTests, apiKey, config = {} }) { // Check if we've exceeded the max wait time if (Date.now() - startTime > maxWaitTime) { return { - status: "TIMEOUT", - error: `Test execution exceeded maximum wait time of ${ - maxWaitTime / 1000 - } seconds`, + status: 408, + type: "TIMEOUT", + error: `Test execution exceeded maximum wait time of ${maxWaitTime / 1000} seconds`, }; } @@ -309,8 +308,8 @@ async function runViaApi({ resolvedTests, apiKey, config = {} }) { response = await axios.get(`${apiUrl}/${runId}`, axiosConfig); } catch (error) { return { - status: error.response.status, - error: error.response.data.error, + status: error.response?.status, + error: error.response?.data?.error, }; } From 6661512083a798e64904ceceaf48d9b0ad9bb80f Mon Sep 17 00:00:00 2001 From: Manny Silva Date: Fri, 24 Oct 2025 23:07:16 -0400 Subject: [PATCH 5/5] Apply suggestion from @coderabbitai[bot] Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- src/tests.js | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/tests.js b/src/tests.js index 5180794b..caa08d69 100644 --- a/src/tests.js +++ b/src/tests.js @@ -331,9 +331,15 @@ async function runViaApi({ resolvedTests, apiKey, config = {} }) { // TODO: Handle file downloads/placement, if any - const results = JSON.parse(response.data.report); - - return results; + try { + const results = JSON.parse(response.data.report); + return results; + } catch (error) { + return { + status: "PARSE_ERROR", + error: `Failed to parse API response: ${error.message}`, + }; + } } /**