From c090cb951f96ed8af59cc5cf6f3f0631842a7e79 Mon Sep 17 00:00:00 2001 From: Toni Date: Wed, 17 Jun 2026 17:05:43 +0100 Subject: [PATCH] feat: add acceptance tests for comparison [CMPA-595] --- .../Deps/LocalDep/Package.swift | 12 ++ .../LocalDep/Sources/LocalDep/LocalDep.swift | 3 + .../workspaces/swift-local-dep/Package.swift | 21 ++++ .../swift-local-dep/Sources/App/main.swift | 3 + .../snyk-test/equivalenceHelpers.ts | 33 +++++ .../unified-test-api-equivalence.spec.ts | 117 ++++++++++++++++-- 6 files changed, 176 insertions(+), 13 deletions(-) create mode 100644 test/acceptance/workspaces/swift-local-dep/Deps/LocalDep/Package.swift create mode 100644 test/acceptance/workspaces/swift-local-dep/Deps/LocalDep/Sources/LocalDep/LocalDep.swift create mode 100644 test/acceptance/workspaces/swift-local-dep/Package.swift create mode 100644 test/acceptance/workspaces/swift-local-dep/Sources/App/main.swift diff --git a/test/acceptance/workspaces/swift-local-dep/Deps/LocalDep/Package.swift b/test/acceptance/workspaces/swift-local-dep/Deps/LocalDep/Package.swift new file mode 100644 index 0000000000..a31605d86b --- /dev/null +++ b/test/acceptance/workspaces/swift-local-dep/Deps/LocalDep/Package.swift @@ -0,0 +1,12 @@ +// swift-tools-version:5.6 +import PackageDescription + +let package = Package( + name: "LocalDep", + products: [ + .library(name: "LocalDep", targets: ["LocalDep"]), + ], + targets: [ + .target(name: "LocalDep"), + ] +) diff --git a/test/acceptance/workspaces/swift-local-dep/Deps/LocalDep/Sources/LocalDep/LocalDep.swift b/test/acceptance/workspaces/swift-local-dep/Deps/LocalDep/Sources/LocalDep/LocalDep.swift new file mode 100644 index 0000000000..e2e79130d7 --- /dev/null +++ b/test/acceptance/workspaces/swift-local-dep/Deps/LocalDep/Sources/LocalDep/LocalDep.swift @@ -0,0 +1,3 @@ +public enum LocalDep { + public static let greeting = "hello" +} diff --git a/test/acceptance/workspaces/swift-local-dep/Package.swift b/test/acceptance/workspaces/swift-local-dep/Package.swift new file mode 100644 index 0000000000..f7a674114c --- /dev/null +++ b/test/acceptance/workspaces/swift-local-dep/Package.swift @@ -0,0 +1,21 @@ +// swift-tools-version:5.6 +// SwiftPM fixture for the unified-test-api equivalence suite. +// +// Unlike most lockfile ecosystems, the snyk-swiftpm-plugin resolves by shelling +// out to `swift package show-dependencies`, which needs the dependencies on disk. +// To stay network-free and deterministic we depend on a sibling package by path +// (Deps/LocalDep) instead of a remote git URL — no fetch, no vendored checkout, +// only the `swift` toolchain itself (gated via requiresCmd in the spec). +import PackageDescription + +let package = Package( + name: "App", + dependencies: [ + .package(path: "Deps/LocalDep"), + ], + targets: [ + .executableTarget(name: "App", dependencies: [ + .product(name: "LocalDep", package: "LocalDep"), + ]), + ] +) diff --git a/test/acceptance/workspaces/swift-local-dep/Sources/App/main.swift b/test/acceptance/workspaces/swift-local-dep/Sources/App/main.swift new file mode 100644 index 0000000000..b4651167e6 --- /dev/null +++ b/test/acceptance/workspaces/swift-local-dep/Sources/App/main.swift @@ -0,0 +1,3 @@ +import LocalDep + +print(LocalDep.greeting) diff --git a/test/jest/acceptance/snyk-test/equivalenceHelpers.ts b/test/jest/acceptance/snyk-test/equivalenceHelpers.ts index 32ba982047..c7b1e6c526 100644 --- a/test/jest/acceptance/snyk-test/equivalenceHelpers.ts +++ b/test/jest/acceptance/snyk-test/equivalenceHelpers.ts @@ -64,6 +64,14 @@ export async function runBothFlows( server: FakeServer, env: Record, runOptions: RunCommandOptions = {}, + /** + * Extra feature flags to enable ONLY in the unified (FF-on) run — e.g. a + * native resolver gate like 'internal_new_gradle_resolver'. The legacy run + * never sees these, so the comparison becomes "native resolver vs legacy + * CLI". Empty (the default) keeps both runs on the legacy resolver, which is + * the Phase 1 endpoint-parity comparison. + */ + unifiedFlags: string[] = [], ): Promise { const argsWithJson = argsString.includes('--json') ? argsString @@ -83,6 +91,9 @@ export async function runBothFlows( server.restore(); server.setFeatureFlag(UNIFIED_TEST_API_FF, true); + for (const flag of unifiedFlags) { + server.setFeatureFlag(flag, true); + } const unifiedRun = await runSnykCLI(argsWithJson, { ...runOptions, cwd, @@ -162,6 +173,11 @@ export type AssertOptions = { /** Set true for fixtures that legitimately produce no submissions * (e.g. "no supported target files"). */ expectNoSubmissions?: boolean; + /** Set true for fixtures expected to be REJECTED by both flows (e.g. an + * out-of-sync lockfile). The two flows have intentional exit-code + * differences for failures (TS CLI exits 3, os-flows exits 2), so this + * asserts the tolerance invariant "both fail" rather than equal codes. */ + expectError?: boolean; }; export function assertEquivalent( @@ -170,6 +186,23 @@ export function assertEquivalent( ): EquivalenceDiff { const { legacy, unified } = result; + // Error-parity mode: the fixture should be rejected by BOTH flows. Exit-code + // values legitimately differ between the TS CLI and os-flows, so we assert + // only the invariant that neither flow reported success. + if (options.expectError) { + if (legacy.code !== 0 && unified.code !== 0) { + return { ok: true }; + } + return { + ok: false, + reason: `expected both flows to fail: legacy=${legacy.code} unified=${unified.code}`, + detail: { + legacyStderr: legacy.stderr, + unifiedStderr: unified.stderr, + }, + }; + } + const bothEmpty = legacy.submissionCount === 0 && unified.submissionCount === 0; diff --git a/test/jest/acceptance/snyk-test/unified-test-api-equivalence.spec.ts b/test/jest/acceptance/snyk-test/unified-test-api-equivalence.spec.ts index 52a6dc2a0d..2bdeaf3e4a 100644 --- a/test/jest/acceptance/snyk-test/unified-test-api-equivalence.spec.ts +++ b/test/jest/acceptance/snyk-test/unified-test-api-equivalence.spec.ts @@ -10,9 +10,25 @@ * FF on → Go binary's os-flows extension resolves dep graphs via the plugin * orchestrator and posts to /rest/orgs/:orgId/tests. * - * Starter corpus. Expand per resolveDepgraphs-rollout-plan.md. + * Phase 1 of the rollout: breadth across every ecosystem that resolves through + * the legacy-CLI fallback. The native resolver flags (cargo, pnpm, gradle, …) + * stay OFF here, so BOTH runs resolve via the same legacy CLI and only the + * submission endpoint differs — that endpoint/serialization parity is what this + * phase proves. Native-resolver parity (gradle, pnpm) and exit-1/error parity + * arrive in later phases. + * + * Fixtures that need an external build tool to resolve (go, sbt, swift, …) are + * tagged with `requiresCmd` and skip when that tool is absent rather than + * failing "inconclusive". Pure lockfile ecosystems (yarn, ruby, composer, + * poetry, …) need no tool beyond what the harness already has. + * + * Note: pip/pipenv and hex/mix are deliberately excluded — their snyk plugins + * resolve by introspecting an *installed* environment (site-packages / fetched + * mix deps), so they need an install step and can't resolve from a clean + * checkout. Python is covered via Poetry, which resolves offline from its lock. */ +import { execFileSync } from 'child_process'; import { fakeServer } from '../../../acceptance/fake-server'; import { createProjectFromWorkspace } from '../../util/createProject'; import { getServerPort } from '../../util/getServerPort'; @@ -20,6 +36,18 @@ import { assertEquivalent, runBothFlows } from './equivalenceHelpers'; jest.setTimeout(1000 * 60 * 3); +/** True when `cmd` resolves on PATH — used to skip fixtures whose toolchain is absent. */ +function commandAvailable(cmd: string): boolean { + try { + execFileSync(process.platform === 'win32' ? 'where' : 'which', [cmd], { + stdio: 'ignore', + }); + return true; + } catch { + return false; + } +} + describe('snyk test — unified test API equivalence (FF off vs on)', () => { let server; let env: Record; @@ -47,9 +75,15 @@ describe('snyk test — unified test API equivalence (FF off vs on)', () => { name: string; args: string; expectNoSubmissions?: boolean; + /** Expected to be rejected by both flows (e.g. out-of-sync lockfile). */ + expectError?: boolean; + /** Binary that must be on PATH for the legacy CLI to resolve this fixture. + * Omitted for lockfile-only ecosystems that need no external build tool. */ + requiresCmd?: string; }; const fixtures: Fixture[] = [ + // --- Starter corpus (unchanged) --- { name: 'npm-package', args: 'test' }, { name: 'maven-app', args: 'test' }, { name: 'mono-repo-project', args: 'test --all-projects' }, @@ -58,24 +92,81 @@ describe('snyk test — unified test API equivalence (FF off vs on)', () => { args: 'test', expectNoSubmissions: true, }, + + // --- Phase 1: breadth via the legacy fallback --- + // JavaScript (yarn) — resolved from yarn.lock; no tool beyond node. + { name: 'yarn-package', args: 'test' }, + { name: 'yarn-workspaces', args: 'test --all-projects' }, + // Ruby — resolved from Gemfile.lock. + { name: 'ruby-app', args: 'test' }, + // PHP (Composer) — resolved from composer.lock. + { name: 'composer-app', args: 'test' }, + // CocoaPods — resolved from Podfile.lock. + { name: 'cocoapods-app', args: 'test' }, + // Swift (SwiftPM) — the plugin shells out to `swift package show-dependencies`, + // so it needs the swift toolchain (gated) and the deps on disk. The fixture + // depends on a sibling package by path, keeping resolution network-free. + { name: 'swift-local-dep', args: 'test', requiresCmd: 'swift' }, + // Go modules — needs the go toolchain. + { name: 'golang-gomodules', args: 'test', requiresCmd: 'go' }, + // Python (Poetry) — resolved from poetry.lock; no tool beyond node. + // (pip/pipenv are intentionally NOT used here: their snyk plugins resolve by + // introspecting *installed* site-packages, so they require a pip/pipenv + // install step and can't resolve from a clean checkout the way a lockfile can.) + { name: 'poetry-app', args: 'test' }, + // .NET (NuGet) — resolved from project.assets.json; no tool beyond node. + { name: 'nuget-app-2', args: 'test' }, + // Scala (sbt) — needs sbt. + { name: 'sbt-app', args: 'test', requiresCmd: 'sbt' }, + + // --- Phase 2: option parity (same flag applied to BOTH flows) --- + // Each option is passed to legacy and unified alike; the resolved graph and + // project metadata must still match. Clean projects, so exit code stays 0. + { name: 'npm-package', args: 'test --dev' }, + { name: 'npm-package', args: 'test --file=package.json' }, + { + name: 'npm-package-pruneable', + args: 'test --prune-repeated-subdependencies', + }, + + // --- Phase 2: partial-failure parity under --all-projects --- + // monorepo-bad-project mixes resolvable and unresolvable projects. This is + // the exact scenario the os-flows change targeted ("tolerate per-project + // resolution failures in the --all-projects orchestrator path"): both flows + // should resolve the good projects, skip the bad one, and agree on the + // submitted-project count and exit code. + { name: 'monorepo-bad-project', args: 'test --all-projects' }, + + // --- Phase 2: error parity (both flows must REJECT the bad input) --- + // Exit-code values legitimately differ (TS CLI 3 vs os-flows 2), so these + // assert the tolerance invariant "both fail" rather than equal codes. + { name: 'npm-out-of-sync', args: 'test', expectError: true }, + { name: 'yarn-out-of-sync', args: 'test', expectError: true }, ]; describe.each(fixtures)( 'fixture: $name ($args)', - ({ name, args, expectNoSubmissions }) => { - test('FF off vs on produces equivalent dep graphs and exit code', async () => { - const project = await createProjectFromWorkspace(name); + ({ name, args, expectNoSubmissions, expectError, requiresCmd }) => { + const runnable = !requiresCmd || commandAvailable(requiresCmd); + (runnable ? test : test.skip)( + 'FF off vs on produces equivalent dep graphs and exit code', + async () => { + const project = await createProjectFromWorkspace(name); - const result = await runBothFlows(project.path(), args, server, env); - const diff = assertEquivalent(result, { expectNoSubmissions }); + const result = await runBothFlows(project.path(), args, server, env); + const diff = assertEquivalent(result, { + expectNoSubmissions, + expectError, + }); - if (!diff.ok) { - throw new Error( - `Equivalence failed for ${name} (${args}): ${diff.reason}\n` + - `detail=${JSON.stringify(diff.detail, null, 2)}`, - ); - } - }); + if (!diff.ok) { + throw new Error( + `Equivalence failed for ${name} (${args}): ${diff.reason}\n` + + `detail=${JSON.stringify(diff.detail, null, 2)}`, + ); + } + }, + ); }, ); });