|
| 1 | +/* |
| 2 | +Toolbox Aid |
| 3 | +David Quesenberry |
| 4 | +05/03/2026 |
| 5 | +update-tool-completion-audit-from-playwright.mjs |
| 6 | +*/ |
| 7 | +import fs from "node:fs"; |
| 8 | +import path from "node:path"; |
| 9 | +import { fileURLToPath } from "node:url"; |
| 10 | + |
| 11 | +const __filename = fileURLToPath(import.meta.url); |
| 12 | +const __dirname = path.dirname(__filename); |
| 13 | +const repoRoot = path.resolve(__dirname, ".."); |
| 14 | +const resultsPath = path.join(repoRoot, "tests", "results", "playwright-results.json"); |
| 15 | +const auditPath = path.join(repoRoot, "docs", "dev", "reports", "tool_completion_audit.md"); |
| 16 | +const validationReportPath = path.join(repoRoot, "docs", "dev", "reports", "tool_validation_results.md"); |
| 17 | + |
| 18 | +function readJsonFile(filePath) { |
| 19 | + return JSON.parse(fs.readFileSync(filePath, "utf8")); |
| 20 | +} |
| 21 | + |
| 22 | +function readTextFile(filePath) { |
| 23 | + return fs.readFileSync(filePath, "utf8"); |
| 24 | +} |
| 25 | + |
| 26 | +function writeTextFile(filePath, text) { |
| 27 | + fs.writeFileSync(filePath, text); |
| 28 | +} |
| 29 | + |
| 30 | +function collectToolIdsFromAudit(auditText) { |
| 31 | + const toolMatches = [...auditText.matchAll(/###\s+([a-z0-9-]+-v2)\s*$/gm)]; |
| 32 | + const toolIds = []; |
| 33 | + for (const match of toolMatches) { |
| 34 | + const toolId = match[1].trim(); |
| 35 | + if (!toolIds.includes(toolId)) { |
| 36 | + toolIds.push(toolId); |
| 37 | + } |
| 38 | + } |
| 39 | + return toolIds; |
| 40 | +} |
| 41 | + |
| 42 | +function collectTestsFromPlaywrightSuite(suite, tests) { |
| 43 | + if (suite.specs && Array.isArray(suite.specs)) { |
| 44 | + for (const spec of suite.specs) { |
| 45 | + const specTitle = typeof spec.title === "string" ? spec.title : ""; |
| 46 | + const titlePath = Array.isArray(spec.titlePath) ? spec.titlePath : []; |
| 47 | + if (!spec.tests || !Array.isArray(spec.tests)) { |
| 48 | + continue; |
| 49 | + } |
| 50 | + for (const test of spec.tests) { |
| 51 | + const resultStatus = Array.isArray(test.results) |
| 52 | + ? test.results.map((result) => result.status).find((status) => status && status !== "skipped") |
| 53 | + : null; |
| 54 | + tests.push({ |
| 55 | + title: typeof test.title === "string" ? test.title : specTitle, |
| 56 | + fullTitle: [...titlePath, test.title || specTitle].filter(Boolean).join(" > "), |
| 57 | + status: resultStatus || (test.outcome === "expected" ? "passed" : "failed"), |
| 58 | + error: Array.isArray(test.results) |
| 59 | + ? test.results.map((result) => result.error && result.error.message).find(Boolean) || "" |
| 60 | + : "" |
| 61 | + }); |
| 62 | + } |
| 63 | + } |
| 64 | + } |
| 65 | + if (suite.suites && Array.isArray(suite.suites)) { |
| 66 | + for (const childSuite of suite.suites) { |
| 67 | + collectTestsFromPlaywrightSuite(childSuite, tests); |
| 68 | + } |
| 69 | + } |
| 70 | +} |
| 71 | + |
| 72 | +function collectAllPlaywrightTests(playwrightJson) { |
| 73 | + const tests = []; |
| 74 | + if (playwrightJson.suites && Array.isArray(playwrightJson.suites)) { |
| 75 | + for (const suite of playwrightJson.suites) { |
| 76 | + collectTestsFromPlaywrightSuite(suite, tests); |
| 77 | + } |
| 78 | + } |
| 79 | + return tests; |
| 80 | +} |
| 81 | + |
| 82 | +function getToolStatusFromTests(toolId, tests) { |
| 83 | + const matchingTests = tests.filter((test) => test.fullTitle.includes(`@${toolId}`) || test.title.includes(`@${toolId}`)); |
| 84 | + if (matchingTests.length === 0) { |
| 85 | + return { |
| 86 | + status: "FAIL", |
| 87 | + reason: "No Playwright tool-level tests found for this tool." |
| 88 | + }; |
| 89 | + } |
| 90 | + const failedTests = matchingTests.filter((test) => test.status !== "passed"); |
| 91 | + if (failedTests.length > 0) { |
| 92 | + const firstFailure = failedTests[0]; |
| 93 | + const failureMessage = firstFailure.error ? firstFailure.error.replace(/\s+/g, " ").trim() : "Playwright reported a failing tool-level test."; |
| 94 | + return { |
| 95 | + status: "FAIL", |
| 96 | + reason: `${firstFailure.fullTitle}${failureMessage ? ` -> ${failureMessage}` : ""}` |
| 97 | + }; |
| 98 | + } |
| 99 | + return { |
| 100 | + status: "PASS", |
| 101 | + reason: "All mapped Playwright tool-level tests passed." |
| 102 | + }; |
| 103 | +} |
| 104 | + |
| 105 | +function updateToolSectionStatus(auditText, toolId, status, reason) { |
| 106 | + const sectionPattern = new RegExp(`(###\\s+${toolId}[\\s\\S]*?)(?=\\n###\\s+[a-z0-9-]+-v2\\s*$|\\s*$)`); |
| 107 | + const sectionMatch = auditText.match(sectionPattern); |
| 108 | + if (!sectionMatch) { |
| 109 | + return auditText; |
| 110 | + } |
| 111 | + const section = sectionMatch[1]; |
| 112 | + const sectionLines = section.split(/\r?\n/); |
| 113 | + const headingLine = sectionLines[0]; |
| 114 | + const remainingLines = sectionLines |
| 115 | + .slice(1) |
| 116 | + .filter((line) => !/^\s*-\s+\*\*Status:\*\*/.test(line)) |
| 117 | + .filter((line) => !/^\s*-\s+Exact failure reason:/.test(line)); |
| 118 | + const rebuiltLines = [ |
| 119 | + headingLine, |
| 120 | + `- **Status:** ${status}`, |
| 121 | + `- Exact failure reason: ${reason}`, |
| 122 | + ...remainingLines |
| 123 | + ]; |
| 124 | + const rebuiltSection = `${rebuiltLines.join("\n").replace(/\n{3,}/g, "\n\n").trimEnd()}\n`; |
| 125 | + return auditText.replace(sectionPattern, rebuiltSection); |
| 126 | +} |
| 127 | + |
| 128 | +function buildValidationReport(toolIds, toolStatuses) { |
| 129 | + const lines = []; |
| 130 | + lines.push("# Tool Validation Results"); |
| 131 | + lines.push(""); |
| 132 | + lines.push("Derived from `tests/results/playwright-results.json` generated by `npm run test:workspace-v2`."); |
| 133 | + lines.push(""); |
| 134 | + lines.push("| Tool | Status | Failing Reason |"); |
| 135 | + lines.push("| --- | --- | --- |"); |
| 136 | + for (const toolId of toolIds) { |
| 137 | + const entry = toolStatuses[toolId]; |
| 138 | + const reasonText = entry.status === "FAIL" ? entry.reason : "n/a"; |
| 139 | + lines.push(`| \`${toolId}\` | ${entry.status} | ${reasonText.replace(/\|/g, "\\|")} |`); |
| 140 | + } |
| 141 | + lines.push(""); |
| 142 | + return `${lines.join("\n")}\n`; |
| 143 | +} |
| 144 | + |
| 145 | +function updateGateEvidenceLine(auditText, passedCount, failedCount) { |
| 146 | + const gateStatus = failedCount > 0 ? "FAIL" : "PASS"; |
| 147 | + const gateLine = `- \`npm run test:workspace-v2\` -> ${gateStatus} (\`${passedCount} passed\`, \`${failedCount} failed\`).`; |
| 148 | + if (/^- `npm run test:workspace-v2` -> .*/m.test(auditText)) { |
| 149 | + return auditText.replace(/^- `npm run test:workspace-v2` -> .*/m, gateLine); |
| 150 | + } |
| 151 | + return auditText; |
| 152 | +} |
| 153 | + |
| 154 | +function main() { |
| 155 | + if (!fs.existsSync(resultsPath)) { |
| 156 | + const reportText = "# Tool Validation Results\n\nPlaywright results file is missing. Run `npm run test:workspace-v2` first.\n"; |
| 157 | + writeTextFile(validationReportPath, reportText); |
| 158 | + console.error(`Missing Playwright results: ${resultsPath}`); |
| 159 | + process.exitCode = 1; |
| 160 | + return; |
| 161 | + } |
| 162 | + |
| 163 | + const playwrightJson = readJsonFile(resultsPath); |
| 164 | + const auditText = readTextFile(auditPath); |
| 165 | + const toolIds = collectToolIdsFromAudit(auditText); |
| 166 | + const tests = collectAllPlaywrightTests(playwrightJson); |
| 167 | + const passedCount = tests.filter((test) => test.status === "passed").length; |
| 168 | + const failedCount = tests.filter((test) => test.status !== "passed").length; |
| 169 | + const toolStatuses = {}; |
| 170 | + |
| 171 | + let updatedAuditText = updateGateEvidenceLine(auditText, passedCount, failedCount); |
| 172 | + for (const toolId of toolIds) { |
| 173 | + toolStatuses[toolId] = getToolStatusFromTests(toolId, tests); |
| 174 | + updatedAuditText = updateToolSectionStatus( |
| 175 | + updatedAuditText, |
| 176 | + toolId, |
| 177 | + toolStatuses[toolId].status, |
| 178 | + toolStatuses[toolId].reason |
| 179 | + ); |
| 180 | + } |
| 181 | + |
| 182 | + writeTextFile(auditPath, updatedAuditText); |
| 183 | + writeTextFile(validationReportPath, buildValidationReport(toolIds, toolStatuses)); |
| 184 | + console.log(`Updated tool audit and validation report from Playwright results for ${toolIds.length} tools.`); |
| 185 | +} |
| 186 | + |
| 187 | +main(); |
0 commit comments