Add dry_run option to the accessibility scanner

.github/actions/file/action.yml

@@ -24,6 +24,10 @@ inputs:
     description: "In the 'file' step, also open grouped issues which link to all issues with the same root cause"
     required: false
     default: "false"
+  dry_run:
+    description: "When true, log the issues that would be filed without opening, closing, or reopening any issues."
+    required: false
+    default: "false"
 
 outputs:
   filings_file:

.github/actions/file/src/index.ts

@@ -29,12 +29,14 @@ export default async function () {
     ? JSON.parse(fs.readFileSync(cachedFilingsFile, 'utf8'))
     : []
   const shouldOpenGroupedIssues = core.getBooleanInput('open_grouped_issues')
+  const dryRun = core.getBooleanInput('dry_run')
   core.debug(`Input: 'findings_file: ${findingsFile}'`)
   core.debug(`Input: 'repository: ${repoWithOwner}'`)
   core.debug(`Input: 'base_url: ${baseUrl ?? '(default)'}'`)
   core.debug(`Input: 'screenshot_repository: ${screenshotRepo}'`)
   core.debug(`Input: 'cached_filings_file: ${cachedFilingsFile}'`)
   core.debug(`Input: 'open_grouped_issues: ${shouldOpenGroupedIssues}'`)
+  core.debug(`Input: 'dry_run: ${dryRun}'`)
 
   const octokit = new OctokitWithThrottling({
     auth: token,
@@ -61,50 +63,69 @@ export default async function () {
   // Track new issues for grouping
   const newIssuesByProblemShort: Record<string, FindingGroupIssue[]> = {}
   const trackingIssueUrls: Record<string, string> = {}
+  const dryRunCounts = {open: 0, reopen: 0, close: 0}
 
   for (const filing of filings) {
     let response: OctokitResponse<IssueResponse> | undefined
     try {
-      if (isResolvedFiling(filing)) {
-        // Close the filing’s issue (if necessary)
-        response = await closeIssue(octokit, new Issue(filing.issue))
-        filing.issue.state = 'closed'
-      } else if (isNewFiling(filing)) {
-        // Open a new issue for the filing
-        response = await openIssue(octokit, repoWithOwner, filing.findings[0], screenshotRepo)
-        ;(filing as Filing).issue = {state: 'open'} as Issue
+      if (dryRun) {
+        if (isResolvedFiling(filing)) {
+          dryRunCounts.close++
+          filing.issue.state = 'closed'
+          core.info(`[dry run] Would CLOSE issue: ${filing.issue.url}`)
+        } else if (isNewFiling(filing)) {
+          dryRunCounts.open++
+          ;(filing as Filing).issue = {state: 'open'} as Issue
+          core.info(
+            `[dry run] Would OPEN a new issue for: ${filing.findings[0].problemShort} (${filing.findings[0].url})`,
+          )
+        } else if (isRepeatedFiling(filing)) {
+          dryRunCounts.reopen++
+          filing.issue.state = 'reopened'
+          core.info(`[dry run] Would REOPEN issue: ${filing.issue.url}`)
+        }
+      } else {
+        if (isResolvedFiling(filing)) {
+          // Close the filing's issue (if necessary)
+          response = await closeIssue(octokit, new Issue(filing.issue))
+          filing.issue.state = 'closed'
+        } else if (isNewFiling(filing)) {
+          // Open a new issue for the filing
+          response = await openIssue(octokit, repoWithOwner, filing.findings[0], screenshotRepo)
+          ;(filing as Filing).issue = {state: 'open'} as Issue
 
-        // Track for grouping
-        if (shouldOpenGroupedIssues) {
-          const problemShort: string = filing.findings[0].problemShort
-          if (!newIssuesByProblemShort[problemShort]) {
-            newIssuesByProblemShort[problemShort] = []
+          // Track for grouping
+          if (shouldOpenGroupedIssues) {
+            const problemShort: string = filing.findings[0].problemShort
+            if (!newIssuesByProblemShort[problemShort]) {
+              newIssuesByProblemShort[problemShort] = []
+            }
+            newIssuesByProblemShort[problemShort].push({
+              url: response.data.html_url,
+              id: response.data.number,
+            })
           }
-          newIssuesByProblemShort[problemShort].push({
-            url: response.data.html_url,
-            id: response.data.number,
-          })
+        } else if (isRepeatedFiling(filing)) {
+          // Reopen the filing's issue (if necessary) and update the body with the latest finding
+          response = await reopenIssue(
+            octokit,
+            new Issue(filing.issue),
+            filing.findings[0],
+            repoWithOwner,
+            screenshotRepo,
+          )
+          filing.issue.state = 'reopened'
+        }
+        if (response?.data && filing.issue) {
+          // Update the filing with the latest issue data
+          filing.issue.id = response.data.id
+          filing.issue.nodeId = response.data.node_id
+          filing.issue.url = response.data.html_url
+          filing.issue.title = response.data.title
+          core.info(
+            `Set issue ${response.data.title} (${repoWithOwner}#${response.data.number}) state to ${filing.issue.state}`,
+          )
         }
-      } else if (isRepeatedFiling(filing)) {
-        // Reopen the filing's issue (if necessary) and update the body with the latest finding
-        response = await reopenIssue(
-          octokit,
-          new Issue(filing.issue),
-          filing.findings[0],
-          repoWithOwner,
-          screenshotRepo,
-        )
-        filing.issue.state = 'reopened'
-      }
-      if (response?.data && filing.issue) {
-        // Update the filing with the latest issue data
-        filing.issue.id = response.data.id
-        filing.issue.nodeId = response.data.node_id
-        filing.issue.url = response.data.html_url
-        filing.issue.title = response.data.title
-        core.info(
-          `Set issue ${response.data.title} (${repoWithOwner}#${response.data.number}) state to ${filing.issue.state}`,
-        )
       }
     } catch (error) {
       core.setFailed(`Failed on filing: ${JSON.stringify(filing, null, 2)}\n${error}`)
@@ -114,7 +135,7 @@ export default async function () {
 
   // Open tracking issues for groups with >1 new issue and link back from each
   // new issue
-  if (shouldOpenGroupedIssues) {
+  if (shouldOpenGroupedIssues && !dryRun) {
     for (const [problemShort, issues] of Object.entries(newIssuesByProblemShort)) {
       if (issues.length > 1) {
         const capitalizedProblemShort = problemShort[0].toUpperCase() + problemShort.slice(1)
@@ -138,6 +159,16 @@ export default async function () {
     }
   }
 
+  if (dryRun) {
+    core.info('[dry run] Summary of actions that would be taken:')
+    console.table({
+      open: dryRunCounts.open,
+      reopen: dryRunCounts.reopen,
+      close: dryRunCounts.close,
+      total: dryRunCounts.open + dryRunCounts.reopen + dryRunCounts.close,
+    })
+  }
+
   const filingsPath = path.join(process.env.RUNNER_TEMP || '/tmp', `filings-${crypto.randomUUID()}.json`)
   fs.writeFileSync(filingsPath, JSON.stringify(filings))
   core.setOutput('filings_file', filingsPath)

.github/actions/file/tests/dryRun.test.ts

@@ -0,0 +1,195 @@
+import {describe, it, expect, vi, beforeEach, afterEach} from 'vitest'
+
+// --- Mock the issue-mutating helpers so we can assert they are NEVER called in dry run ---
+const openIssue = vi.fn()
+const reopenIssue = vi.fn()
+const closeIssue = vi.fn()
+vi.mock('../src/openIssue.js', () => ({openIssue: (...args: unknown[]) => openIssue(...args)}))
+vi.mock('../src/reopenIssue.js', () => ({reopenIssue: (...args: unknown[]) => reopenIssue(...args)}))
+vi.mock('../src/closeIssue.js', () => ({closeIssue: (...args: unknown[]) => closeIssue(...args)}))
+
+// --- Mock @actions/core: control inputs, capture logs/outputs ---
+const inputs: Record<string, string> = {}
+const infoLines: string[] = []
+const outputs: Record<string, string> = {}
+vi.mock('@actions/core', () => ({
+  getInput: (name: string) => inputs[name] ?? '',
+  getBooleanInput: (name: string) => (inputs[name] ?? 'false') === 'true',
+  setOutput: (name: string, value: string) => {
+    outputs[name] = value
+  },
+  info: (msg: string) => {
+    infoLines.push(msg)
+  },
+  debug: () => {},
+  warning: () => {},
+  setFailed: () => {},
+}))
+
+// --- Mock fs: feed findings/cached filings in, swallow the output write ---
+const files: Record<string, string> = {}
+vi.mock('node:fs', () => ({
+  default: {
+    readFileSync: (p: string) => files[p],
+    writeFileSync: (p: string, data: string) => {
+      files[p] = data
+    },
+  },
+}))
+
+// --- Stub Octokit so constructing it in index.ts doesn't do anything real ---
+const octokitRequest = vi.fn()
+vi.mock('@octokit/core', () => ({
+  Octokit: {
+    plugin: () =>
+      class {
+        request = octokitRequest
+      },
+  },
+}))
+vi.mock('@octokit/plugin-throttling', () => ({throttling: {}}))
+
+import runFileAction from '../src/index.ts'
+
+const finding = {
+  scannerType: 'axe',
+  ruleId: 'color-contrast',
+  url: 'https://example.com/page',
+  html: '<span>Low contrast</span>',
+  problemShort: 'elements must meet minimum color contrast ratio thresholds',
+  problemUrl: 'https://dequeuniversity.com/rules/axe/4.10/color-contrast',
+  solutionShort: 'ensure sufficient contrast',
+}
+
+// A second finding with no matching cached filing -> NEW (open)
+const newFinding = {...finding, ruleId: 'heading-order', html: '<h3>Skipped</h3>'}
+
+// A cached filing whose finding matches `finding` -> REPEATED (reopen)
+const repeatedCached = {
+  issue: {id: 1, nodeId: 'N1', url: 'https://github.com/org/repo/issues/1', title: 'repeat'},
+  findings: [finding],
+}
+
+// A cached filing with NO matching finding this run -> RESOLVED (close)
+const resolvedCached = {
+  issue: {id: 2, nodeId: 'N2', url: 'https://github.com/org/repo/issues/2', title: 'resolved'},
+  findings: [{...finding, ruleId: 'landmark-one-main', html: '<div>old</div>'}],
+}
+
+function setup() {
+  // findings file: includes `finding` (matches repeatedCached) and `newFinding` (brand new)
+  files['/tmp/findings.json'] = JSON.stringify([finding, newFinding])
+  // cached filings: one repeated, one resolved (its finding is absent from findings file)
+  files['/tmp/cached.json'] = JSON.stringify([repeatedCached, resolvedCached])
+  inputs.findings_file = '/tmp/findings.json'
+  inputs.cached_filings_file = '/tmp/cached.json'
+  inputs.repository = 'org/repo'
+  inputs.token = 'fake-token'
+}
+
+describe('file action — dry_run', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    infoLines.length = 0
+    for (const k of Object.keys(inputs)) delete inputs[k]
+    for (const k of Object.keys(outputs)) delete outputs[k]
+    vi.spyOn(console, 'table').mockImplementation(() => {})
+  })
+  afterEach(() => {
+    vi.restoreAllMocks()
+  })
+
+  it('does not open, reopen, or close any issues when dry_run is true', async () => {
+    setup()
+    inputs.dry_run = 'true'
+
+    await runFileAction()
+
+    expect(openIssue).not.toHaveBeenCalled()
+    expect(reopenIssue).not.toHaveBeenCalled()
+    expect(closeIssue).not.toHaveBeenCalled()
+    expect(octokitRequest).not.toHaveBeenCalled()
+  })
+
+  it('logs the intended action for each filing type', async () => {
+    setup()
+    inputs.dry_run = 'true'
+
+    await runFileAction()
+
+    const log = infoLines.join('\n')
+    expect(log).toContain(
+      '[dry run] Would OPEN a new issue for: elements must meet minimum color contrast ratio thresholds (https://example.com/page)',
+    )
+    expect(log).toContain('[dry run] Would REOPEN issue: https://github.com/org/repo/issues/1')
+    expect(log).toContain('[dry run] Would CLOSE issue: https://github.com/org/repo/issues/2')
+  })
+
+  it('logs a summary table with counts', async () => {
+    setup()
+    inputs.dry_run = 'true'
+
+    await runFileAction()
+
+    expect(vi.mocked(console.table)).toHaveBeenCalledWith(
+      expect.objectContaining({open: 1, reopen: 1, close: 1, total: 3}),
+    )
+  })
+
+  it('still writes the filings_file output in dry run', async () => {
+    setup()
+    inputs.dry_run = 'true'
+
+    await runFileAction()
+
+    expect(outputs.filings_file).toBeDefined()
+  })
+
+  it('updates in-memory issue state for an accurate preview without mutating remotely', async () => {
+    setup()
+    inputs.dry_run = 'true'
+
+    await runFileAction()
+
+    // The path written is `${RUNNER_TEMP||'/tmp'}/filings-<uuid>.json`; grab it from the output.
+    const writtenPath = outputs.filings_file
+    const writtenFilings = JSON.parse(files[writtenPath])
+
+    // Resolved cached filing (issues/2) -> would be CLOSED
+    const resolved = writtenFilings.find(
+      (f: {issue?: {url?: string}}) => f.issue?.url === 'https://github.com/org/repo/issues/2',
+    )
+    expect(resolved?.issue.state).toBe('closed')
+
+    // Repeated cached filing (issues/1) -> would be REOPENED
+    const repeated = writtenFilings.find(
+      (f: {issue?: {url?: string}}) => f.issue?.url === 'https://github.com/org/repo/issues/1',
+    )
+    expect(repeated?.issue.state).toBe('reopened')
+
+    // New filing -> issue object created with state 'open'
+    const opened = writtenFilings.find((f: {issue?: {state?: string}}) => f.issue?.state === 'open')
+    expect(opened).toBeDefined()
+
+    // And confirm we still didn't actually mutate anything remotely
+    expect(openIssue).not.toHaveBeenCalled()
+    expect(reopenIssue).not.toHaveBeenCalled()
+    expect(closeIssue).not.toHaveBeenCalled()
+  })
+
+  it('does call the mutating helpers when dry_run is false (regression guard)', async () => {
+    setup()
+    inputs.dry_run = 'false'
+    // helpers return a minimal Octokit-style response so index.ts can read response.data
+    const resp = {data: {id: 9, node_id: 'N', number: 9, html_url: 'https://github.com/org/repo/issues/9', title: 't'}}
+    openIssue.mockResolvedValue(resp)
+    reopenIssue.mockResolvedValue(resp)
+    closeIssue.mockResolvedValue(resp)
+
+    await runFileAction()
+
+    expect(openIssue).toHaveBeenCalled()
+    expect(reopenIssue).toHaveBeenCalled()
+    expect(closeIssue).toHaveBeenCalled()
+  })
+})

FAQ.md

@@ -60,6 +60,17 @@ Just keep in mind that resetting the cache means the Action will "forget" what
 it's already seen, so it may reopen issues that were previously tracked or
 closed.
 
+### How can I preview what the scanner would do without filing issues?
+
+Set the `dry_run` input to `true`. The scanner will run a normal scan and log the
+issues it _would_ open, reopen, or close — but it won't actually mutate any data or write to the `gh-cache` branch
+assign any issues, and it won't write to the `gh-cache` branch.
+
+This is handy for trying out a new configuration or seeing how many issues a scan
+would file, without making any changes to your repository. Because dry runs don't
+update the cache, your next real run behaves exactly as if the dry run never
+happened.
+
 ### Does this work with private repositories?
 
 Yes! The Action works with both public and private repositories. Since it runs

README.md

@@ -55,6 +55,7 @@ jobs:
           # skip_copilot_assignment: false # Optional: Set to true to skip assigning issues to GitHub Copilot (or if you don't have GitHub Copilot)
           # include_screenshots: false # Optional: Set to true to capture screenshots and include links to them in filed issues
           # open_grouped_issues: false # Optional: Set to true to open an issue grouping individual issues per violation
+          # dry_run: false # Optional: Set to true to scan and log what would be filed without creating/closing issues or writing the cache
           # reduced_motion: no-preference # Optional: Playwright reduced motion configuration option
           # color_scheme: light # Optional: Playwright color scheme configuration option
           # scans: '["axe","reflow-scan"]' # Optional: An array of scans (or plugins) to be performed. If not provided, only Axe will be performed.
@@ -131,6 +132,7 @@ Trigger the workflow manually or automatically based on your configuration. The
 | `reduced_motion`          | No       | Playwright `reducedMotion` setting for scan contexts. Allowed values: `reduce`, `no-preference`                                                                                                                                                                  | `reduce`                                                                    |
 | `color_scheme`            | No       | Playwright `colorScheme` setting for scan contexts. Allowed values: `light`, `dark`, `no-preference`                                                                                                                                                             | `dark`                                                                      |
 | `scans`                   | No       | An array of scans (or plugins) to be performed. If not provided, only Axe will be performed.                                                                                                                                                                     | `'["axe", "reflow-scan", ...other plugins]'`                                |
+| `dry_run`                 | No       | When `true`, scan and log the issues that _would_ be filed without opening, closing, reopening, or assigning any issues — and without writing to the `gh-cache` branch. Useful for safely previewing results. Default: `false`                                   | `true`                                                                      |
 | `url_configs`             | No       | A stringified JSON array of URL config objects. Each object must have a `url` field and may have an optional `excludeSelectors` field (array of CSS selectors to exclude from the Axe scan for that URL). When provided, takes precedence over the `urls` input. | `'[{"url":"https://example.com","excludeSelectors":["iframe","#widget"]}]'` |
 
 ---