diff --git a/src/commands/fix/agent-fix.mts b/src/commands/fix/agent-fix.mts new file mode 100644 index 000000000..dbccd9172 --- /dev/null +++ b/src/commands/fix/agent-fix.mts @@ -0,0 +1,630 @@ +import { existsSync } from 'node:fs' +import path from 'node:path' + +import semver from 'semver' + +import { getManifestData } from '@socketsecurity/registry' +import { arrayUnique } from '@socketsecurity/registry/lib/arrays' +import { debugFn, isDebug } from '@socketsecurity/registry/lib/debug' +import { logger } from '@socketsecurity/registry/lib/logger' +import { runScript } from '@socketsecurity/registry/lib/npm' +import { + fetchPackagePackument, + readPackageJson, + resolvePackageName, +} from '@socketsecurity/registry/lib/packages' +import { naturalCompare } from '@socketsecurity/registry/lib/sorts' + +import { getActiveBranchesForPackage } from './fix-branch-helpers.mts' +import { getActualTree } from './get-actual-tree.mts' +import { + getSocketBranchName, + getSocketBranchWorkspaceComponent, + getSocketCommitMessage, + gitCreateAndPushBranch, + gitRemoteBranchExists, + gitResetAndClean, + gitUnstagedModifiedFiles, +} from './git.mts' +import { + cleanupOpenPrs, + enablePrAutoMerge, + openPr, + prExistForBranch, + setGitRemoteGithubRepoUrl, +} from './open-pr.mts' +import constants from '../../constants.mts' +import { + findBestPatchVersion, + findPackageNode, + findPackageNodes, + updatePackageJsonFromNode, +} from '../../shadow/npm/arborist-helpers.mts' +import { removeNodeModules } from '../../utils/fs.mts' +import { globWorkspace } from '../../utils/glob.mts' +import { readLockfile } from '../../utils/lockfile.mts' +import { getPurlObject } from '../../utils/purl.mts' +import { applyRange } from '../../utils/semver.mts' +import { getCveInfoFromAlertsMap } from '../../utils/socket-package-alert.mts' +import { idToPurl } from '../../utils/spec.mts' +import { overridesDataByAgent } from '../optimize/get-overrides-by-agent.mts' + +import type { CiEnv } from './fix-env-helpers.mts' +import type { PrMatch } from './open-pr.mts' +import type { NodeClass } from '../../shadow/npm/arborist/types.mts' +import type { CResult } from '../../types.mts' +import type { EnvDetails } from '../../utils/package-environment.mts' +import type { RangeStyle } from '../../utils/semver.mts' +import type { AlertsByPurl } from '../../utils/socket-package-alert.mts' +import type { EditablePackageJson } from '@socketsecurity/registry/lib/packages' +import type { Spinner } from '@socketsecurity/registry/lib/spinner' + +export type FixOptions = { + autoMerge: boolean + cwd: string + limit: number + purls: string[] + rangeStyle: RangeStyle + spinner?: Spinner | undefined + test: boolean + testScript: string +} + +export type InstallOptions = { + args?: string[] | undefined + cwd?: string | undefined + spinner?: Spinner | undefined +} + +export type InstallPhaseHandler = ( + editablePkgJson: EditablePackageJson, + name: string, + oldVersion: string, + newVersion: string, + vulnerableVersionRange: string, + options: FixOptions, +) => Promise + +export type Installer = ( + pkgEnvDetails: EnvDetails, + options: InstallOptions, +) => Promise + +const noopHandler = (() => {}) as unknown as InstallPhaseHandler + +export async function agentFix( + pkgEnvDetails: EnvDetails, + actualTree: NodeClass | undefined, + alertsMap: AlertsByPurl, + installer: Installer, + { + beforeInstall = noopHandler, + // eslint-disable-next-line sort-destructure-keys/sort-destructure-keys + afterInstall = noopHandler, + revertInstall = noopHandler, + }: { + beforeInstall?: InstallPhaseHandler | undefined + afterInstall?: InstallPhaseHandler | undefined + revertInstall?: InstallPhaseHandler | undefined + }, + ciEnv: CiEnv | null, + openPrs: PrMatch[], + options: FixOptions, +): Promise> { + const { autoMerge, cwd, limit, rangeStyle, test, testScript } = options + const { spinner } = constants + const { pkgPath: rootPath } = pkgEnvDetails + + let count = 0 + + const infoByPartialPurl = getCveInfoFromAlertsMap(alertsMap, { + limit: Math.max(limit, openPrs.length), + }) + if (!infoByPartialPurl) { + spinner?.stop() + logger.info('No fixable vulns found.') + return { ok: true, data: { fixed: false } } + } + + if (isDebug()) { + debugFn('found: cves for', Array.from(infoByPartialPurl.keys())) + } + + // Lazily access constants.packumentCache. + const { packumentCache } = constants + + const workspacePkgJsonPaths = await globWorkspace( + pkgEnvDetails.agent, + rootPath, + ) + const pkgJsonPaths = [ + ...workspacePkgJsonPaths, + // Process the workspace root last since it will add an override to package.json. + pkgEnvDetails.editablePkgJson.filename!, + ] + const sortedInfoEntries = Array.from(infoByPartialPurl.entries()).sort( + (a, b) => naturalCompare(a[0], b[0]), + ) + + const getOverridesData = overridesDataByAgent.get(pkgEnvDetails.agent)! + + const cleanupInfoEntriesLoop = () => { + logger.dedent() + spinner?.dedent() + packumentCache.clear() + } + + const handleInstallFail = (): CResult<{ fixed: boolean }> => { + cleanupInfoEntriesLoop() + return { + ok: false, + message: 'Install failed', + cause: `Unexpected condition: ${pkgEnvDetails.agent} install failed`, + } + } + + spinner?.stop() + + infoEntriesLoop: for ( + let i = 0, { length } = sortedInfoEntries; + i < length; + i += 1 + ) { + const isLastInfoEntry = i === length - 1 + const infoEntry = sortedInfoEntries[i]! + const partialPurlObj = getPurlObject(infoEntry[0]) + const name = resolvePackageName(partialPurlObj) + + const infos = Array.from(infoEntry[1].values()) + if (!infos.length) { + continue infoEntriesLoop + } + + logger.log(`Processing vulns for ${name}:`) + logger.indent() + spinner?.indent() + + if (getManifestData(partialPurlObj.type, name)) { + debugFn(`found: Socket Optimize variant for ${name}`) + } + // eslint-disable-next-line no-await-in-loop + const packument = await fetchPackagePackument(name) + if (!packument) { + logger.warn(`Unexpected condition: No packument found for ${name}.\n`) + cleanupInfoEntriesLoop() + continue infoEntriesLoop + } + + const activeBranches = getActiveBranchesForPackage( + ciEnv, + infoEntry[0], + openPrs, + ) + const availableVersions = Object.keys(packument.versions) + const warningsForAfter = new Set() + + // eslint-disable-next-line no-unused-labels + pkgJsonPathsLoop: for ( + let j = 0, { length: length_j } = pkgJsonPaths; + j < length_j; + j += 1 + ) { + const isLastPkgJsonPath = j === length_j - 1 + const pkgJsonPath = pkgJsonPaths[j]! + const pkgPath = path.dirname(pkgJsonPath) + const isWorkspaceRoot = + pkgJsonPath === pkgEnvDetails.editablePkgJson.filename + const workspace = isWorkspaceRoot + ? 'root' + : path.relative(rootPath, pkgPath) + const branchWorkspace = ciEnv + ? getSocketBranchWorkspaceComponent(workspace) + : '' + + // actualTree may not be defined on the first iteration of pkgJsonPathsLoop. + if (!actualTree) { + if (!ciEnv) { + // eslint-disable-next-line no-await-in-loop + await removeNodeModules(cwd) + } + const maybeActualTree = + ciEnv && existsSync(path.join(rootPath, 'node_modules')) + ? // eslint-disable-next-line no-await-in-loop + await getActualTree(cwd) + : // eslint-disable-next-line no-await-in-loop + await installer(pkgEnvDetails, { cwd, spinner }) + const maybeLockSrc = maybeActualTree + ? // eslint-disable-next-line no-await-in-loop + await readLockfile(pkgEnvDetails.lockPath) + : null + if (maybeActualTree && maybeLockSrc) { + actualTree = maybeActualTree + } + } + if (!actualTree) { + // Exit early if install fails. + return handleInstallFail() + } + + const oldVersions = arrayUnique( + findPackageNodes(actualTree, name) + .map(n => n.version) + .filter(Boolean), + ) + + if (!oldVersions.length) { + debugFn(`skip: ${name} not found\n`) + // Skip to next package. + cleanupInfoEntriesLoop() + continue infoEntriesLoop + } + + // Always re-read the editable package.json to avoid stale mutations + // across iterations. + // eslint-disable-next-line no-await-in-loop + const editablePkgJson = await readPackageJson(pkgJsonPath, { + editable: true, + }) + const seenVersions = new Set() + + let hasAnnouncedWorkspace = false + let workspaceLogCallCount = logger.logCallCount + if (isDebug()) { + debugFn(`check: workspace ${workspace}`) + hasAnnouncedWorkspace = true + workspaceLogCallCount = logger.logCallCount + } + + oldVersionsLoop: for (const oldVersion of oldVersions) { + const oldId = `${name}@${oldVersion}` + const oldPurl = idToPurl(oldId, partialPurlObj.type) + + const node = findPackageNode(actualTree, name, oldVersion) + if (!node) { + debugFn(`skip: ${oldId} not found`) + continue oldVersionsLoop + } + infosLoop: for (const { + firstPatchedVersionIdentifier, + vulnerableVersionRange, + } of infos) { + const newVersion = findBestPatchVersion( + node, + availableVersions, + vulnerableVersionRange, + ) + const newVersionPackument = newVersion + ? packument.versions[newVersion] + : undefined + + if (!(newVersion && newVersionPackument)) { + warningsForAfter.add( + `${oldId} not updated: requires >=${firstPatchedVersionIdentifier}`, + ) + continue infosLoop + } + if (seenVersions.has(newVersion)) { + continue infosLoop + } + if (semver.gte(oldVersion, newVersion)) { + debugFn(`skip: ${oldId} is >= ${newVersion}`) + continue infosLoop + } + if ( + activeBranches.find( + b => + b.workspace === branchWorkspace && b.newVersion === newVersion, + ) + ) { + debugFn(`skip: open PR found for ${name}@${newVersion}`) + if (++count >= limit) { + cleanupInfoEntriesLoop() + break infoEntriesLoop + } + continue infosLoop + } + + const oldOverrides = getOverridesData( + pkgEnvDetails, + editablePkgJson.content, + ) + const overrideKey = `${name}@${vulnerableVersionRange}` + + const newVersionRange = applyRange( + (oldOverrides as any)?.[overrideKey] ?? oldVersion, + newVersion, + rangeStyle, + ) + const newId = `${name}@${newVersionRange}` + + // eslint-disable-next-line no-await-in-loop + await beforeInstall( + editablePkgJson, + name, + oldVersion, + newVersion, + vulnerableVersionRange, + options, + ) + + updatePackageJsonFromNode( + editablePkgJson, + actualTree, + node, + newVersion, + rangeStyle, + ) + // eslint-disable-next-line no-await-in-loop + if (!(await editablePkgJson.save({ ignoreWhitespace: true }))) { + debugFn(`skip: ${workspace}/package.json unchanged`) + // Reset things just in case. + if (ciEnv) { + // eslint-disable-next-line no-await-in-loop + await gitResetAndClean(ciEnv.baseBranch, cwd) + } + continue infosLoop + } + + if (!hasAnnouncedWorkspace) { + hasAnnouncedWorkspace = true + workspaceLogCallCount = logger.logCallCount + } + + spinner?.start() + spinner?.info(`Installing ${newId} in ${workspace}.`) + + let error + let errored = false + try { + // eslint-disable-next-line no-await-in-loop + const maybeActualTree = await installer(pkgEnvDetails, { + cwd, + spinner, + }) + const maybeLockSrc = maybeActualTree + ? // eslint-disable-next-line no-await-in-loop + await readLockfile(pkgEnvDetails.lockPath) + : null + if (maybeActualTree && maybeLockSrc) { + actualTree = maybeActualTree + // eslint-disable-next-line no-await-in-loop + await afterInstall( + editablePkgJson, + name, + oldVersion, + newVersion, + vulnerableVersionRange, + options, + ) + if (test) { + spinner?.info(`Testing ${newId} in ${workspace}.`) + // eslint-disable-next-line no-await-in-loop + await runScript(testScript, [], { spinner, stdio: 'ignore' }) + } + spinner?.success(`Fixed ${name} in ${workspace}.`) + seenVersions.add(newVersion) + } else { + errored = true + } + } catch (e) { + error = e + errored = true + } + + spinner?.stop() + + // Check repoInfo to make TypeScript happy. + if (!errored && ciEnv?.repoInfo) { + try { + // eslint-disable-next-line no-await-in-loop + const result = await gitUnstagedModifiedFiles(cwd) + if (!result.ok) { + logger.warn( + 'Unexpected condition: Nothing to commit, skipping PR creation.', + ) + continue + } + const moddedFilepaths = result.data.filter(filepath => { + const basename = path.basename(filepath) + return ( + basename === 'package.json' || + basename === pkgEnvDetails.lockName + ) + }) + if (!moddedFilepaths.length) { + logger.warn( + 'Unexpected condition: Nothing to commit, skipping PR creation.', + ) + continue infosLoop + } + + const branch = getSocketBranchName(oldPurl, newVersion, workspace) + let skipPr = false + if ( + // eslint-disable-next-line no-await-in-loop + await prExistForBranch( + ciEnv.repoInfo.owner, + ciEnv.repoInfo.repo, + branch, + ) + ) { + skipPr = true + debugFn(`skip: branch "${branch}" exists`) + } + // eslint-disable-next-line no-await-in-loop + else if (await gitRemoteBranchExists(branch, cwd)) { + skipPr = true + debugFn(`skip: remote branch "${branch}" exists`) + } else if ( + // eslint-disable-next-line no-await-in-loop + !(await gitCreateAndPushBranch( + branch, + getSocketCommitMessage(oldPurl, newVersion, workspace), + moddedFilepaths, + { + cwd, + email: ciEnv.gitEmail, + user: ciEnv.gitUser, + }, + )) + ) { + skipPr = true + logger.warn( + 'Unexpected condition: Push failed, skipping PR creation.', + ) + } + if (skipPr) { + // eslint-disable-next-line no-await-in-loop + await gitResetAndClean(ciEnv.baseBranch, cwd) + // eslint-disable-next-line no-await-in-loop + const maybeActualTree = await installer(pkgEnvDetails, { + cwd, + spinner, + }) + const maybeLockSrc = maybeActualTree + ? // eslint-disable-next-line no-await-in-loop + await readLockfile(pkgEnvDetails.lockPath) + : null + if (maybeActualTree && maybeLockSrc) { + actualTree = maybeActualTree + continue infosLoop + } + // Exit early if install fails. + return handleInstallFail() + } + + // eslint-disable-next-line no-await-in-loop + await Promise.allSettled([ + setGitRemoteGithubRepoUrl( + ciEnv.repoInfo.owner, + ciEnv.repoInfo.repo, + ciEnv.githubToken!, + cwd, + ), + cleanupOpenPrs(ciEnv.repoInfo.owner, ciEnv.repoInfo.repo, { + newVersion, + purl: oldPurl, + workspace, + }), + ]) + // eslint-disable-next-line no-await-in-loop + const prResponse = await openPr( + ciEnv.repoInfo.owner, + ciEnv.repoInfo.repo, + branch, + oldPurl, + newVersion, + { + baseBranch: ciEnv.baseBranch, + cwd, + workspace, + }, + ) + if (prResponse) { + const { data } = prResponse + const prRef = `PR #${data.number}` + logger.success(`Opened ${prRef}.`) + if (autoMerge) { + logger.indent() + spinner?.indent() + // eslint-disable-next-line no-await-in-loop + const { details, enabled } = await enablePrAutoMerge(data) + if (enabled) { + logger.info(`Auto-merge enabled for ${prRef}.`) + } else { + const message = `Failed to enable auto-merge for ${prRef}${ + details + ? `:\n${details.map(d => ` - ${d}`).join('\n')}` + : '.' + }` + logger.error(message) + } + logger.dedent() + spinner?.dedent() + } + } + } catch (e) { + error = e + errored = true + } + } + + if (ciEnv) { + spinner?.start() + // eslint-disable-next-line no-await-in-loop + await gitResetAndClean(ciEnv.baseBranch, cwd) + // eslint-disable-next-line no-await-in-loop + const maybeActualTree = await installer(pkgEnvDetails, { + cwd, + spinner, + }) + spinner?.stop() + if (maybeActualTree) { + actualTree = maybeActualTree + } else { + errored = true + } + } + if (errored) { + if (!ciEnv) { + spinner?.start() + // eslint-disable-next-line no-await-in-loop + await revertInstall( + editablePkgJson, + name, + oldVersion, + newVersion, + vulnerableVersionRange, + options, + ) + // eslint-disable-next-line no-await-in-loop + await Promise.all([ + removeNodeModules(cwd), + editablePkgJson.save({ ignoreWhitespace: true }), + ]) + // eslint-disable-next-line no-await-in-loop + const maybeActualTree = await installer(pkgEnvDetails, { + cwd, + spinner, + }) + spinner?.stop() + if (maybeActualTree) { + actualTree = maybeActualTree + } else { + // Exit early if install fails. + return handleInstallFail() + } + } + return { + ok: false, + message: 'Update failed', + cause: `Update failed for ${oldId} in ${workspace}${error ? '; ' + error : ''}`, + } + } + debugFn('name:', name) + debugFn('increment: count', count + 1) + if (++count >= limit) { + cleanupInfoEntriesLoop() + break infoEntriesLoop + } + } + } + if (!isLastPkgJsonPath && logger.logCallCount > workspaceLogCallCount) { + logger.logNewline() + } + } + + for (const warningText of warningsForAfter) { + logger.warn(warningText) + } + if (!isLastInfoEntry) { + logger.logNewline() + } + cleanupInfoEntriesLoop() + } + + spinner?.stop() + + // Or, did we change anything? + return { ok: true, data: { fixed: true } } +} diff --git a/src/commands/fix/get-actual-tree.mts b/src/commands/fix/get-actual-tree.mts new file mode 100644 index 000000000..ee7d3ebd4 --- /dev/null +++ b/src/commands/fix/get-actual-tree.mts @@ -0,0 +1,20 @@ +import { + Arborist, + SAFE_ARBORIST_REIFY_OPTIONS_OVERRIDES, +} from '../../shadow/npm/arborist/index.mts' + +import type { NodeClass } from '../../shadow/npm/arborist/types.mts' + +export async function getActualTree( + cwd: string = process.cwd(), +): Promise { + // @npmcli/arborist DOES have partial support for pnpm structured node_modules + // folders. However, support is iffy resulting in unhappy path errors and hangs. + // So, to avoid the unhappy path, we restrict our usage to --dry-run loading + // of the node_modules folder. + const arb = new Arborist({ + path: cwd, + ...SAFE_ARBORIST_REIFY_OPTIONS_OVERRIDES, + }) + return await arb.loadActual() +} diff --git a/src/commands/fix/handle-fix.mts b/src/commands/fix/handle-fix.mts index 235f7cd75..aef95c0e8 100644 --- a/src/commands/fix/handle-fix.mts +++ b/src/commands/fix/handle-fix.mts @@ -1,9 +1,17 @@ +import { logger } from '@socketsecurity/registry/lib/logger' + +import { npmFix } from './npm-fix.mts' import { outputFixResult } from './output-fix-result.mts' -import { runFix } from './run-fix.mts' +import { pnpmFix } from './pnpm-fix.mts' +import { CMD_NAME } from './shared.mts' +import constants from '../../constants.mts' +import { detectAndValidatePackageEnvironment } from '../../utils/package-environment.mts' import type { OutputKind } from '../../types.mts' import type { RangeStyle } from '../../utils/semver.mts' +const { NPM, PNPM } = constants + export async function handleFix({ autoMerge, cwd, @@ -23,12 +31,47 @@ export async function handleFix({ test: boolean testScript: string }) { - const result = await runFix({ + const pkgEnvResult = await detectAndValidatePackageEnvironment(cwd, { + cmdName: CMD_NAME, + logger, + }) + if (!pkgEnvResult.ok) { + return pkgEnvResult + } + + const pkgEnvDetails = pkgEnvResult.data + if (!pkgEnvDetails) { + return { + ok: false, + message: 'No package found', + cause: `No valid package environment was found in given cwd (${cwd})`, + } + } + + logger.info( + `Fixing packages for ${pkgEnvDetails.agent} v${pkgEnvDetails.agentVersion}.\n`, + ) + + const { agent } = pkgEnvDetails + if (agent !== NPM && agent !== PNPM) { + return { + ok: false, + message: 'Not supported', + cause: `${agent} is not supported by this command at the moment.`, + } + } + + // Lazily access spinner. + const { spinner } = constants + const fixer = agent === NPM ? npmFix : pnpmFix + + const result = await fixer(pkgEnvDetails, { autoMerge, cwd, limit, purls, rangeStyle, + spinner, test, testScript, }) diff --git a/src/commands/fix/npm-fix.mts b/src/commands/fix/npm-fix.mts index f0ad39c0e..91cec5688 100644 --- a/src/commands/fix/npm-fix.mts +++ b/src/commands/fix/npm-fix.mts @@ -1,165 +1,74 @@ -import path from 'node:path' - -import Config from '@npmcli/config' -import { - definitions, - flatten, - shorthands, - // @ts-ignore -} from '@npmcli/config/lib/definitions' -import semver from 'semver' - -import { getManifestData } from '@socketsecurity/registry' -import { arrayUnique } from '@socketsecurity/registry/lib/arrays' import { debugFn, isDebug } from '@socketsecurity/registry/lib/debug' -import { logger } from '@socketsecurity/registry/lib/logger' -import { runScript } from '@socketsecurity/registry/lib/npm' -import { - fetchPackagePackument, - readPackageJson, - resolvePackageName, -} from '@socketsecurity/registry/lib/packages' -import { naturalCompare } from '@socketsecurity/registry/lib/sorts' -import { getActiveBranchesForPackage } from './fix-branch-helpers.mts' +import { agentFix } from './agent-fix.mts' import { getCiEnv, getOpenPrsForEnvironment } from './fix-env-helpers.mts' -import { - getSocketBranchName, - getSocketBranchWorkspaceComponent, - getSocketCommitMessage, - gitCreateAndPushBranch, - gitRemoteBranchExists, - gitResetAndClean, - gitUnstagedModifiedFiles, -} from './git.mts' -import { - cleanupOpenPrs, - enablePrAutoMerge, - openPr, - prExistForBranch, - setGitRemoteGithubRepoUrl, -} from './open-pr.mts' +import { getActualTree } from './get-actual-tree.mts' import { getAlertsMapOptions } from './shared.mts' -import constants from '../../constants.mts' import { Arborist, SAFE_ARBORIST_REIFY_OPTIONS_OVERRIDES, } from '../../shadow/npm/arborist/index.mts' -import { - findBestPatchVersion, - findPackageNode, - findPackageNodes, - getAlertsMapFromArborist, - updateNode, - updatePackageJsonFromNode, -} from '../../shadow/npm/arborist-helpers.mts' +import { getAlertsMapFromArborist } from '../../shadow/npm/arborist-helpers.mts' +import { runAgentInstall } from '../../utils/agent.mts' import { getAlertsMapFromPurls } from '../../utils/alerts-map.mts' -import { removeNodeModules } from '../../utils/fs.mts' -import { globWorkspace } from '../../utils/glob.mts' -import { getPurlObject } from '../../utils/purl.mts' -import { applyRange } from '../../utils/semver.mts' -import { getCveInfoFromAlertsMap } from '../../utils/socket-package-alert.mts' -import { idToPurl } from '../../utils/spec.mts' -import type { - ArboristInstance, - NodeClass, -} from '../../shadow/npm/arborist/types.mts' +import type { FixOptions, InstallOptions } from './agent-fix.mts' +import type { NodeClass } from '../../shadow/npm/arborist/types.mts' import type { CResult } from '../../types.mts' import type { EnvDetails } from '../../utils/package-environment.mts' -import type { RangeStyle } from '../../utils/semver.mts' import type { PackageJson } from '@socketsecurity/registry/lib/packages' -type InstallOptions = { - cwd?: string | undefined -} - async function install( pkgEnvDetails: EnvDetails, - arb: ArboristInstance, options: InstallOptions, ): Promise { - const { cwd = process.cwd() } = { + const { args, cwd, spinner } = { __proto__: null, ...options, } as InstallOptions try { - const config = new Config({ - argv: [], - cwd, - definitions, - flatten, - npmPath: pkgEnvDetails.agentExecPath, - shorthands, + await runAgentInstall(pkgEnvDetails, { + args, + spinner, + stdio: isDebug() ? 'inherit' : 'ignore', }) - await config.load() - - const legacyPeerDeps = config.get('legacy-peer-deps') - const newArb = new Arborist({ - legacyPeerDeps, - path: cwd, - }) - newArb.idealTree = await arb.buildIdealTree() - await newArb.reify() - arb.actualTree = null - await arb.loadActual() - return arb.actualTree + return await getActualTree(cwd) } catch {} return null } export async function npmFix( pkgEnvDetails: EnvDetails, - { - autoMerge, - cwd, - limit, - purls, - rangeStyle, - test, - testScript, - }: { - autoMerge: boolean - cwd: string - limit: number - purls: string[] - rangeStyle: RangeStyle - test: boolean - testScript: string - }, + options: FixOptions, ): Promise> { - // Lazily access constants.spinner. - const { spinner } = constants - const { pkgPath: rootPath } = pkgEnvDetails + const { limit, purls, spinner } = options spinner?.start() const ciEnv = await getCiEnv() const openPrs = ciEnv ? await getOpenPrsForEnvironment(ciEnv) : [] - let count = 0 - - let arb = new Arborist({ - path: rootPath, - ...SAFE_ARBORIST_REIFY_OPTIONS_OVERRIDES, - }) - // Calling arb.reify() creates the arb.diff object, nulls-out arb.idealTree, - // and populates arb.actualTree. - await arb.reify() - await arb.loadActual() - let actualTree = arb.actualTree! - + let actualTree: NodeClass | undefined let alertsMap try { - alertsMap = purls.length - ? await getAlertsMapFromPurls( - purls, - getAlertsMapOptions({ limit: Math.max(limit, openPrs.length) }), - ) - : await getAlertsMapFromArborist( - arb, - getAlertsMapOptions({ limit: Math.max(limit, openPrs.length) }), - ) + if (purls.length) { + alertsMap = await getAlertsMapFromPurls( + purls, + getAlertsMapOptions({ limit: Math.max(limit, openPrs.length) }), + ) + } else { + const arb = new Arborist({ + path: pkgEnvDetails.pkgPath, + ...SAFE_ARBORIST_REIFY_OPTIONS_OVERRIDES, + }) + actualTree = await arb.reify() + // Calling arb.reify() creates the arb.diff object, nulls-out arb.idealTree, + // and populates arb.actualTree. + alertsMap = await getAlertsMapFromArborist( + arb, + getAlertsMapOptions({ limit: Math.max(limit, openPrs.length) }), + ) + } } catch (e) { spinner?.stop() debugFn('catch: PURL API\n', e) @@ -170,450 +79,37 @@ export async function npmFix( } } - const infoByPartialPurl = getCveInfoFromAlertsMap(alertsMap, { - limit: Math.max(limit, openPrs.length), - }) - if (!infoByPartialPurl) { - spinner?.stop() - logger.info('No fixable vulns found.') - return { ok: true, data: { fixed: false } } - } - - // Lazily access constants.packumentCache. - const { packumentCache } = constants - - const workspacePkgJsonPaths = await globWorkspace( - pkgEnvDetails.agent, - rootPath, - ) - const pkgJsonPaths = [ - ...workspacePkgJsonPaths, - // Process the workspace root last since it will add an override to package.json. - pkgEnvDetails.editablePkgJson.filename!, - ] - const sortedInfoEntries = Array.from(infoByPartialPurl.entries()).sort( - (a, b) => naturalCompare(a[0], b[0]), - ) - - const cleanupInfoEntriesLoop = () => { - logger.dedent() - spinner?.dedent() - packumentCache.clear() - } - - const handleInstallFail = (): CResult<{ fixed: boolean }> => { - cleanupInfoEntriesLoop() - return { - ok: false, - message: 'Installation failure', - cause: `Unexpected condition: ${pkgEnvDetails.agent} install failed.`, - } - } - - spinner?.stop() - - infoEntriesLoop: for ( - let i = 0, { length } = sortedInfoEntries; - i < length; - i += 1 - ) { - const isLastInfoEntry = i === length - 1 - const infoEntry = sortedInfoEntries[i]! - const partialPurlObj = getPurlObject(infoEntry[0]) - const name = resolvePackageName(partialPurlObj) - - const infos = Array.from(infoEntry[1].values()) - if (!infos.length) { - continue infoEntriesLoop - } - - logger.log(`Processing vulns for ${name}:`) - logger.indent() - spinner?.indent() - - if (getManifestData(partialPurlObj.type, name)) { - debugFn(`found: Socket Optimize variant for ${name}`) - } - // eslint-disable-next-line no-await-in-loop - const packument = await fetchPackagePackument(name) - if (!packument) { - logger.warn(`Unexpected condition: No packument found for ${name}.\n`) - cleanupInfoEntriesLoop() - continue infoEntriesLoop - } - - const activeBranches = getActiveBranchesForPackage( - ciEnv, - infoEntry[0], - openPrs, - ) - const availableVersions = Object.keys(packument.versions) - const warningsForAfter = new Set() - - // eslint-disable-next-line no-unused-labels - pkgJsonPathsLoop: for ( - let j = 0, { length: length_j } = pkgJsonPaths; - j < length_j; - j += 1 - ) { - arb = new Arborist({ - path: rootPath, - ...SAFE_ARBORIST_REIFY_OPTIONS_OVERRIDES, - }) - // eslint-disable-next-line no-await-in-loop - await arb.loadActual() - actualTree = arb.actualTree! - const isLastPkgJsonPath = j === length_j - 1 - const pkgJsonPath = pkgJsonPaths[j]! - const pkgPath = path.dirname(pkgJsonPath) - const isWorkspaceRoot = - pkgJsonPath === pkgEnvDetails.editablePkgJson.filename - const workspace = isWorkspaceRoot - ? 'root' - : path.relative(rootPath, pkgPath) - const branchWorkspace = ciEnv - ? getSocketBranchWorkspaceComponent(workspace) - : '' - - const oldVersions = arrayUnique( - findPackageNodes(actualTree, name) - .map(n => n.target?.version ?? n.version) - .filter(Boolean), - ) - - if (!oldVersions.length) { - debugFn(`skip: ${name} not found\n`) - // Skip to next package. - cleanupInfoEntriesLoop() - continue infoEntriesLoop - } - - // Always re-read the editable package.json to avoid stale mutations - // across iterations. - // eslint-disable-next-line no-await-in-loop - const editablePkgJson = await readPackageJson(pkgJsonPath, { - editable: true, - }) - const seenVersions = new Set() - - let hasAnnouncedWorkspace = false - let workspaceLogCallCount = logger.logCallCount - if (isDebug()) { - debugFn(`check: workspace ${workspace}`) - hasAnnouncedWorkspace = true - workspaceLogCallCount = logger.logCallCount - } - - oldVersionsLoop: for (const oldVersion of oldVersions) { - const oldId = `${name}@${oldVersion}` - const oldPurl = idToPurl(oldId, partialPurlObj.type) - - const node = findPackageNode(actualTree, name, oldVersion) - if (!node) { - debugFn(`skip: ${oldId} not found`) - continue oldVersionsLoop + let revertData: PackageJson | undefined + + return await agentFix( + pkgEnvDetails, + actualTree, + alertsMap, + install, + { + async beforeInstall(editablePkgJson) { + revertData = { + ...(editablePkgJson.content.dependencies && { + dependencies: { ...editablePkgJson.content.dependencies }, + }), + ...(editablePkgJson.content.optionalDependencies && { + optionalDependencies: { + ...editablePkgJson.content.optionalDependencies, + }, + }), + ...(editablePkgJson.content.peerDependencies && { + peerDependencies: { ...editablePkgJson.content.peerDependencies }, + }), + } as PackageJson + }, + async revertInstall(editablePkgJson) { + if (revertData) { + editablePkgJson.update(revertData) } - - infosLoop: for (const { - firstPatchedVersionIdentifier, - vulnerableVersionRange, - } of infos.values()) { - const newVersion = findBestPatchVersion( - node, - availableVersions, - vulnerableVersionRange, - ) - const newVersionPackument = newVersion - ? packument.versions[newVersion] - : undefined - - if (!(newVersion && newVersionPackument)) { - warningsForAfter.add( - `${oldId} not updated: requires >=${firstPatchedVersionIdentifier}`, - ) - continue infosLoop - } - if (seenVersions.has(newVersion)) { - continue infosLoop - } - if (semver.gte(oldVersion, newVersion)) { - debugFn(`skip: ${oldId} is >= ${newVersion}`) - continue infosLoop - } - if ( - activeBranches.find( - b => - b.workspace === branchWorkspace && b.newVersion === newVersion, - ) - ) { - debugFn(`skip: open PR found for ${name}@${newVersion}`) - if (++count >= limit) { - cleanupInfoEntriesLoop() - break infoEntriesLoop - } - continue infosLoop - } - - const newVersionRange = applyRange(oldVersion, newVersion, rangeStyle) - const newId = `${name}@${newVersionRange}` - - const revertData = { - ...(editablePkgJson.content.dependencies && { - dependencies: { ...editablePkgJson.content.dependencies }, - }), - ...(editablePkgJson.content.optionalDependencies && { - optionalDependencies: { - ...editablePkgJson.content.optionalDependencies, - }, - }), - ...(editablePkgJson.content.peerDependencies && { - peerDependencies: { ...editablePkgJson.content.peerDependencies }, - }), - } as PackageJson - - updateNode(node, newVersion, newVersionPackument) - updatePackageJsonFromNode( - editablePkgJson, - // eslint-disable-next-line no-await-in-loop - await arb.buildIdealTree(), - node, - newVersion, - rangeStyle, - ) - // eslint-disable-next-line no-await-in-loop - if (!(await editablePkgJson.save({ ignoreWhitespace: true }))) { - debugFn(`skip: ${workspace}/package.json unchanged`) - // Reset things just in case. - if (ciEnv) { - // eslint-disable-next-line no-await-in-loop - await gitResetAndClean(ciEnv.baseBranch, cwd) - } - continue infosLoop - } - - if (!hasAnnouncedWorkspace) { - hasAnnouncedWorkspace = true - workspaceLogCallCount = logger.logCallCount - } - - spinner?.start() - spinner?.info(`Installing ${newId} in ${workspace}.`) - - let error - let errored = false - try { - // eslint-disable-next-line no-await-in-loop - const maybeActualTree = await install(pkgEnvDetails, arb, { cwd }) - if (maybeActualTree) { - actualTree = maybeActualTree - if (test) { - spinner?.info(`Testing ${newId} in ${workspace}.`) - // eslint-disable-next-line no-await-in-loop - await runScript(testScript, [], { spinner, stdio: 'ignore' }) - } - spinner?.success(`Fixed ${name} in ${workspace}.`) - seenVersions.add(newVersion) - } else { - errored = true - } - } catch (e) { - errored = true - error = e - } - - spinner?.stop() - - // Check repoInfo to make TypeScript happy. - if (!errored && ciEnv?.repoInfo) { - try { - // eslint-disable-next-line no-await-in-loop - const result = await gitUnstagedModifiedFiles(cwd) - if (!result.ok) { - // Do we fail if this fails? If this git command - // fails then probably other git commands do too? - logger.warn( - 'Unexpected condition: Nothing to commit, skipping PR creation.', - ) - continue infosLoop - } - const moddedFilepaths = result.data.filter(p => { - const basename = path.basename(p) - return ( - basename === 'package.json' || - basename === 'package-lock.json' - ) - }) - - if (!moddedFilepaths.length) { - logger.warn( - 'Unexpected condition: Nothing to commit, skipping PR creation.', - ) - continue infosLoop - } - - const branch = getSocketBranchName(oldPurl, newVersion, workspace) - - let skipPr = false - if ( - // eslint-disable-next-line no-await-in-loop - await prExistForBranch( - ciEnv.repoInfo.owner, - ciEnv.repoInfo.repo, - branch, - ) - ) { - skipPr = true - debugFn(`skip: branch "${branch}" exists`) - } - // eslint-disable-next-line no-await-in-loop - else if (await gitRemoteBranchExists(branch, cwd)) { - skipPr = true - debugFn(`skip: remote branch "${branch}" exists`) - } else if ( - // eslint-disable-next-line no-await-in-loop - !(await gitCreateAndPushBranch( - branch, - getSocketCommitMessage(oldPurl, newVersion, workspace), - moddedFilepaths, - { - cwd, - email: ciEnv.gitEmail, - user: ciEnv.gitUser, - }, - )) - ) { - skipPr = true - logger.warn( - 'Unexpected condition: Push failed, skipping PR creation.', - ) - } - if (skipPr) { - // eslint-disable-next-line no-await-in-loop - await gitResetAndClean(ciEnv.baseBranch, cwd) - // eslint-disable-next-line no-await-in-loop - const maybeActualTree = await install(pkgEnvDetails, arb, { - cwd, - }) - if (maybeActualTree) { - actualTree = maybeActualTree - continue infosLoop - } - // Exit early if install fails. - return handleInstallFail() - } - - // eslint-disable-next-line no-await-in-loop - await Promise.allSettled([ - setGitRemoteGithubRepoUrl( - ciEnv.repoInfo.owner, - ciEnv.repoInfo.repo, - ciEnv.githubToken!, - cwd, - ), - cleanupOpenPrs(ciEnv.repoInfo.owner, ciEnv.repoInfo.repo, { - newVersion, - purl: oldPurl, - workspace, - }), - ]) - // eslint-disable-next-line no-await-in-loop - const prResponse = await openPr( - ciEnv.repoInfo.owner, - ciEnv.repoInfo.repo, - branch, - oldPurl, - newVersion, - { - baseBranch: ciEnv.baseBranch, - cwd, - workspace, - }, - ) - if (prResponse) { - const { data } = prResponse - const prRef = `PR #${data.number}` - logger.success(`Opened ${prRef}.`) - if (autoMerge) { - logger.indent() - spinner?.indent() - // eslint-disable-next-line no-await-in-loop - const { details, enabled } = await enablePrAutoMerge(data) - if (enabled) { - logger.info(`Auto-merge enabled for ${prRef}.`) - } else { - const message = `Failed to enable auto-merge for ${prRef}${ - details - ? `:\n${details.map(d => ` - ${d}`).join('\n')}` - : '.' - }` - logger.error(message) - } - logger.dedent() - spinner?.dedent() - } - } - } catch (e) { - error = e - errored = true - } - } - - if (ciEnv) { - spinner?.start() - // eslint-disable-next-line no-await-in-loop - await gitResetAndClean(ciEnv.baseBranch, cwd) - // eslint-disable-next-line no-await-in-loop - const maybeActualTree = await install(pkgEnvDetails, arb, { cwd }) - spinner?.stop() - if (maybeActualTree) { - actualTree = maybeActualTree - } else { - errored = true - } - } - if (errored) { - if (!ciEnv) { - spinner?.start() - editablePkgJson.update(revertData) - // eslint-disable-next-line no-await-in-loop - await Promise.all([ - removeNodeModules(cwd), - editablePkgJson.save({ ignoreWhitespace: true }), - ]) - // eslint-disable-next-line no-await-in-loop - const maybeActualTree = await install(pkgEnvDetails, arb, { cwd }) - spinner?.stop() - if (maybeActualTree) { - actualTree = maybeActualTree - } else { - // Exit early if install fails. - return handleInstallFail() - } - } - logger.fail(`Update failed for ${oldId} in ${workspace}.`, error) - } - if (++count >= limit) { - cleanupInfoEntriesLoop() - break infoEntriesLoop - } - } - } - if (!isLastPkgJsonPath && logger.logCallCount > workspaceLogCallCount) { - logger.logNewline() - } - } - - for (const warningText of warningsForAfter) { - logger.warn(warningText) - } - if (!isLastInfoEntry) { - logger.logNewline() - } - cleanupInfoEntriesLoop() - } - - spinner?.stop() - - // Or, did we change anything? - return { ok: true, data: { fixed: true } } + }, + }, + ciEnv, + openPrs, + options, + ) } diff --git a/src/commands/fix/pnpm-fix.mts b/src/commands/fix/pnpm-fix.mts index 20fed0b74..89f25b243 100644 --- a/src/commands/fix/pnpm-fix.mts +++ b/src/commands/fix/pnpm-fix.mts @@ -1,95 +1,35 @@ -import { existsSync, promises as fs } from 'node:fs' -import path from 'node:path' +import { promises as fs } from 'node:fs' -import semver from 'semver' - -import { getManifestData } from '@socketsecurity/registry' -import { arrayUnique } from '@socketsecurity/registry/lib/arrays' import { debugFn, isDebug } from '@socketsecurity/registry/lib/debug' -import { logger } from '@socketsecurity/registry/lib/logger' -import { runScript } from '@socketsecurity/registry/lib/npm' -import { - fetchPackagePackument, - readPackageJson, - resolvePackageName, -} from '@socketsecurity/registry/lib/packages' -import { naturalCompare } from '@socketsecurity/registry/lib/sorts' +import { hasKeys } from '@socketsecurity/registry/lib/objects' -import { getActiveBranchesForPackage } from './fix-branch-helpers.mts' +import { agentFix } from './agent-fix.mts' import { getCiEnv, getOpenPrsForEnvironment } from './fix-env-helpers.mts' -import { - getSocketBranchName, - getSocketBranchWorkspaceComponent, - getSocketCommitMessage, - gitCreateAndPushBranch, - gitRemoteBranchExists, - gitResetAndClean, - gitUnstagedModifiedFiles, -} from './git.mts' -import { - cleanupOpenPrs, - enablePrAutoMerge, - openPr, - prExistForBranch, - setGitRemoteGithubRepoUrl, -} from './open-pr.mts' +import { getActualTree } from './get-actual-tree.mts' import { getAlertsMapOptions } from './shared.mts' import constants from '../../constants.mts' -import { - Arborist, - SAFE_ARBORIST_REIFY_OPTIONS_OVERRIDES, -} from '../../shadow/npm/arborist/index.mts' -import { - findBestPatchVersion, - findPackageNode, - findPackageNodes, - updatePackageJsonFromNode, -} from '../../shadow/npm/arborist-helpers.mts' import { runAgentInstall } from '../../utils/agent.mts' import { getAlertsMapFromPnpmLockfile, getAlertsMapFromPurls, } from '../../utils/alerts-map.mts' -import { removeNodeModules } from '../../utils/fs.mts' -import { globWorkspace } from '../../utils/glob.mts' +import { readLockfile } from '../../utils/lockfile.mts' import { - extractOverridesFromPnpmLockfileContent, + extractOverridesFromPnpmLockSrc, parsePnpmLockfile, parsePnpmLockfileVersion, - readPnpmLockfile, } from '../../utils/pnpm.mts' -import { getPurlObject } from '../../utils/purl.mts' import { applyRange } from '../../utils/semver.mts' -import { getCveInfoFromAlertsMap } from '../../utils/socket-package-alert.mts' -import { idToPurl } from '../../utils/spec.mts' +import { getOverridesDataPnpm } from '../optimize/get-overrides-by-agent.mts' +import type { FixOptions, InstallOptions } from './agent-fix.mts' import type { NodeClass } from '../../shadow/npm/arborist/types.mts' import type { CResult, StringKeyValueObject } from '../../types.mts' import type { EnvDetails } from '../../utils/package-environment.mts' -import type { RangeStyle } from '../../utils/semver.mts' import type { PackageJson } from '@socketsecurity/registry/lib/packages' -import type { Spinner } from '@socketsecurity/registry/lib/spinner' const { OVERRIDES, PNPM } = constants -type InstallOptions = { - args?: string[] | undefined - cwd?: string | undefined - spinner?: Spinner | undefined -} - -async function getActualTree(cwd: string = process.cwd()): Promise { - // @npmcli/arborist DOES have partial support for pnpm structured node_modules - // folders. However, support is iffy resulting in unhappy path errors and hangs. - // So, to avoid the unhappy path, we restrict our usage to --dry-run loading - // of the node_modules folder. - const arb = new Arborist({ - path: cwd, - ...SAFE_ARBORIST_REIFY_OPTIONS_OVERRIDES, - }) - return await arb.loadActual() -} - async function install( pkgEnvDetails: EnvDetails, options: InstallOptions, @@ -119,56 +59,18 @@ async function install( export async function pnpmFix( pkgEnvDetails: EnvDetails, - { - autoMerge, - cwd, - limit, - purls, - rangeStyle, - test, - testScript, - }: { - autoMerge: boolean - cwd: string - limit: number - purls: string[] - rangeStyle: RangeStyle - test: boolean - testScript: string - }, + options: FixOptions, ): Promise> { - // Lazily access constants.spinner. - const { spinner } = constants - const { pkgPath: rootPath } = pkgEnvDetails + const { cwd, limit, purls, spinner } = options spinner?.start() - const ciEnv = await getCiEnv() - const openPrs = ciEnv ? await getOpenPrsForEnvironment(ciEnv) : [] - - let count = 0 - let actualTree: NodeClass | undefined - const lockfilePath = path.join(rootPath, 'pnpm-lock.yaml') - let lockfileContent = await readPnpmLockfile(lockfilePath) - - // If pnpm-lock.yaml does NOT exist then install with pnpm to create it. - if (!lockfileContent) { - const maybeActualTree = await install(pkgEnvDetails, { cwd, spinner }) - const maybeLockfileContent = maybeActualTree - ? await readPnpmLockfile(lockfilePath) - : null - if (maybeActualTree) { - actualTree = maybeActualTree - lockfileContent = maybeLockfileContent - } - } - - let lockfile = parsePnpmLockfile(lockfileContent) + let { lockSrc } = pkgEnvDetails + let lockfile = parsePnpmLockfile(lockSrc) // Update pnpm-lock.yaml if its version is older than what the installed pnpm // produces. if ( - lockfileContent && pkgEnvDetails.agentVersion.major >= 10 && (parsePnpmLockfileVersion(lockfile?.lockfileVersion)?.major ?? 0) <= 6 ) { @@ -177,21 +79,21 @@ export async function pnpmFix( cwd, spinner, }) - const maybeLockfileContent = maybeActualTree - ? await readPnpmLockfile(lockfilePath) + const maybeLockSrc = maybeActualTree + ? await readLockfile(pkgEnvDetails.lockPath) : null - if (maybeActualTree && maybeLockfileContent) { + if (maybeActualTree && maybeLockSrc) { actualTree = maybeActualTree - lockfileContent = maybeLockfileContent - lockfile = parsePnpmLockfile(lockfileContent) + lockSrc = maybeLockSrc + lockfile = parsePnpmLockfile(lockSrc) } else { lockfile = null } } // Exit early if pnpm-lock.yaml is not found or usable. - // Check !lockfileContent to make TypeScript happy. - if (!lockfile || !lockfileContent) { + // Check !lockSrc to make TypeScript happy. + if (!lockfile || !lockSrc) { spinner?.stop() return { ok: false, @@ -200,6 +102,9 @@ export async function pnpmFix( } } + const ciEnv = await getCiEnv() + const openPrs = ciEnv ? await getOpenPrsForEnvironment(ciEnv) : [] + let alertsMap try { alertsMap = purls.length @@ -221,569 +126,109 @@ export async function pnpmFix( } } - const infoByPartialPurl = getCveInfoFromAlertsMap(alertsMap, { - limit: Math.max(limit, openPrs.length), - }) - if (!infoByPartialPurl) { - spinner?.stop() - logger.info('No fixable vulns found.') - return { ok: true, data: { fixed: false } } - } - - if (isDebug()) { - debugFn('found: cves for', Array.from(infoByPartialPurl.keys())) - } - - // Lazily access constants.packumentCache. - const { packumentCache } = constants - - const workspacePkgJsonPaths = await globWorkspace( - pkgEnvDetails.agent, - rootPath, - ) - const pkgJsonPaths = [ - ...workspacePkgJsonPaths, - // Process the workspace root last since it will add an override to package.json. - pkgEnvDetails.editablePkgJson.filename!, - ] - const sortedInfoEntries = Array.from(infoByPartialPurl.entries()).sort( - (a, b) => naturalCompare(a[0], b[0]), - ) - - const cleanupInfoEntriesLoop = () => { - logger.dedent() - spinner?.dedent() - packumentCache.clear() - } - - const handleInstallFail = (): CResult<{ fixed: boolean }> => { - cleanupInfoEntriesLoop() - return { - ok: false, - message: 'Install failed', - cause: `Unexpected condition: ${pkgEnvDetails.agent} install failed`, - } - } - - spinner?.stop() - - infoEntriesLoop: for ( - let i = 0, { length } = sortedInfoEntries; - i < length; - i += 1 - ) { - const isLastInfoEntry = i === length - 1 - const infoEntry = sortedInfoEntries[i]! - const partialPurlObj = getPurlObject(infoEntry[0]) - const name = resolvePackageName(partialPurlObj) - - const infos = Array.from(infoEntry[1].values()) - if (!infos.length) { - continue infoEntriesLoop - } - - logger.log(`Processing vulns for ${name}:`) - logger.indent() - spinner?.indent() - - if (getManifestData(partialPurlObj.type, name)) { - debugFn(`found: Socket Optimize variant for ${name}`) - } - // eslint-disable-next-line no-await-in-loop - const packument = await fetchPackagePackument(name) - if (!packument) { - logger.warn(`Unexpected condition: No packument found for ${name}.\n`) - cleanupInfoEntriesLoop() - continue infoEntriesLoop - } - - const activeBranches = getActiveBranchesForPackage( - ciEnv, - infoEntry[0], - openPrs, - ) - const availableVersions = Object.keys(packument.versions) - const warningsForAfter = new Set() - - // eslint-disable-next-line no-unused-labels - pkgJsonPathsLoop: for ( - let j = 0, { length: length_j } = pkgJsonPaths; - j < length_j; - j += 1 - ) { - const isLastPkgJsonPath = j === length_j - 1 - const pkgJsonPath = pkgJsonPaths[j]! - const pkgPath = path.dirname(pkgJsonPath) - const isWorkspaceRoot = - pkgJsonPath === pkgEnvDetails.editablePkgJson.filename - const workspace = isWorkspaceRoot - ? 'root' - : path.relative(rootPath, pkgPath) - const branchWorkspace = ciEnv - ? getSocketBranchWorkspaceComponent(workspace) - : '' - - // actualTree may not be defined on the first iteration of pkgJsonPathsLoop. - if (!actualTree) { - if (!ciEnv) { - // eslint-disable-next-line no-await-in-loop - await removeNodeModules(cwd) - } - const maybeActualTree = - ciEnv && existsSync(path.join(rootPath, 'node_modules')) - ? // eslint-disable-next-line no-await-in-loop - await getActualTree(cwd) - : // eslint-disable-next-line no-await-in-loop - await install(pkgEnvDetails, { cwd, spinner }) - const maybeLockfileContent = maybeActualTree - ? // eslint-disable-next-line no-await-in-loop - await readPnpmLockfile(lockfilePath) - : null - if (maybeActualTree && maybeLockfileContent) { - actualTree = maybeActualTree - lockfileContent = maybeLockfileContent - } - } - if (!actualTree) { - // Exit early if install fails. - return handleInstallFail() - } - - const oldVersions = arrayUnique( - findPackageNodes(actualTree, name) - .map(n => n.version) - .filter(Boolean), - ) - - if (!oldVersions.length) { - debugFn(`skip: ${name} not found\n`) - // Skip to next package. - cleanupInfoEntriesLoop() - continue infoEntriesLoop - } - - // Always re-read the editable package.json to avoid stale mutations - // across iterations. - // eslint-disable-next-line no-await-in-loop - const editablePkgJson = await readPackageJson(pkgJsonPath, { - editable: true, - }) - const fixedVersions = new Set() - - // Get current overrides for revert logic. - const oldPnpmSection = editablePkgJson.content[PNPM] as - | StringKeyValueObject - | undefined - - const oldOverrides = oldPnpmSection?.[OVERRIDES] as - | Record - | undefined - - let hasAnnouncedWorkspace = false - let workspaceLogCallCount = logger.logCallCount - if (isDebug()) { - debugFn(`check: workspace ${workspace}`) - hasAnnouncedWorkspace = true - workspaceLogCallCount = logger.logCallCount - } - - oldVersionsLoop: for (const oldVersion of oldVersions) { - const oldId = `${name}@${oldVersion}` - const oldPurl = idToPurl(oldId, partialPurlObj.type) - - const node = findPackageNode(actualTree, name, oldVersion) - if (!node) { - debugFn(`skip: ${oldId} not found`) - continue oldVersionsLoop - } - infosLoop: for (const { - firstPatchedVersionIdentifier, - vulnerableVersionRange, - } of infos) { - const newVersion = findBestPatchVersion( - node, - availableVersions, - vulnerableVersionRange, - ) - const newVersionPackument = newVersion - ? packument.versions[newVersion] - : undefined - - if (!(newVersion && newVersionPackument)) { - warningsForAfter.add( - `${oldId} not updated: requires >=${firstPatchedVersionIdentifier}`, - ) - continue infosLoop - } - if (fixedVersions.has(newVersion)) { - continue infosLoop - } - if (semver.gte(oldVersion, newVersion)) { - debugFn(`skip: ${oldId} is >= ${newVersion}`) - continue infosLoop - } - if ( - activeBranches.find( - b => - b.workspace === branchWorkspace && b.newVersion === newVersion, - ) - ) { - debugFn(`skip: open PR found for ${name}@${newVersion}`) - if (++count >= limit) { - cleanupInfoEntriesLoop() - break infoEntriesLoop - } - continue infosLoop - } - - const overrideKey = `${name}@${vulnerableVersionRange}` - const newVersionRange = applyRange( - oldOverrides?.[overrideKey] ?? oldVersion, - newVersion, - rangeStyle, - ) - const newId = `${name}@${newVersionRange}` + let revertData: PackageJson | undefined + let revertOverrides: PackageJson | undefined + let revertOverridesSrc: string | undefined + + return await agentFix( + pkgEnvDetails, + actualTree, + alertsMap, + install, + { + async beforeInstall( + editablePkgJson, + name, + oldVersion, + newVersion, + vulnerableVersionRange, + options, + ) { + const isWorkspaceRoot = + editablePkgJson.path === pkgEnvDetails.editablePkgJson.filename + // Get current overrides for revert logic. + const { overrides: oldOverrides } = getOverridesDataPnpm( + pkgEnvDetails, + editablePkgJson.content, + ) + const oldPnpmSection = editablePkgJson.content[PNPM] as + | StringKeyValueObject + | undefined + const overrideKey = `${name}@${vulnerableVersionRange}` - const updateOverrides = isWorkspaceRoot - ? ({ - [PNPM]: { - ...oldPnpmSection, - [OVERRIDES]: { - ...oldOverrides, - [overrideKey]: newVersionRange, - }, - }, - } as PackageJson) - : undefined + revertOverrides = undefined + revertOverridesSrc = extractOverridesFromPnpmLockSrc(lockSrc) - const revertOverrides = ( - isWorkspaceRoot + if (isWorkspaceRoot) { + revertOverrides = { + [PNPM]: oldPnpmSection ? { - [PNPM]: oldPnpmSection + ...oldPnpmSection, + [OVERRIDES]: hasKeys(oldOverrides) ? { - ...oldPnpmSection, - [OVERRIDES]: - oldOverrides && Object.keys(oldOverrides).length > 1 - ? { - ...oldOverrides, - [overrideKey]: undefined, - } - : undefined, + ...oldOverrides, + [overrideKey]: undefined, } : undefined, } - : {} - ) as PackageJson - - const revertData = { - ...revertOverrides, - ...(editablePkgJson.content.dependencies && { - dependencies: { ...editablePkgJson.content.dependencies }, - }), - ...(editablePkgJson.content.optionalDependencies && { - optionalDependencies: { - ...editablePkgJson.content.optionalDependencies, - }, - }), - ...(editablePkgJson.content.peerDependencies && { - peerDependencies: { ...editablePkgJson.content.peerDependencies }, - }), + : undefined, } as PackageJson - - if (updateOverrides) { - // Update overrides in the root package.json so that when `pnpm install` - // generates pnpm-lock.yaml it updates transitive dependencies too. - editablePkgJson.update(updateOverrides) - } - updatePackageJsonFromNode( - editablePkgJson, - actualTree, - node, - newVersion, - rangeStyle, - ) - // eslint-disable-next-line no-await-in-loop - if (!(await editablePkgJson.save({ ignoreWhitespace: true }))) { - debugFn(`skip: ${workspace}/package.json unchanged`) - // Reset things just in case. - if (ciEnv) { - // eslint-disable-next-line no-await-in-loop - await gitResetAndClean(ciEnv.baseBranch, cwd) - } - continue infosLoop - } - - if (!hasAnnouncedWorkspace) { - hasAnnouncedWorkspace = true - workspaceLogCallCount = logger.logCallCount - } - - spinner?.start() - spinner?.info(`Installing ${newId} in ${workspace}.`) - - let error - let errored = false - try { - const revertOverridesContent = - extractOverridesFromPnpmLockfileContent(lockfileContent) - // eslint-disable-next-line no-await-in-loop - const maybeActualTree = await install(pkgEnvDetails, { - cwd, - spinner, - }) - const maybeLockfileContent = maybeActualTree - ? // eslint-disable-next-line no-await-in-loop - await readPnpmLockfile(lockfilePath) - : null - if (maybeActualTree && maybeLockfileContent) { - actualTree = maybeActualTree - lockfileContent = maybeLockfileContent - // Revert overrides metadata in package.json now that pnpm-lock.yaml - // has been updated. - editablePkgJson.update(revertOverrides) - // eslint-disable-next-line no-await-in-loop - await editablePkgJson.save({ ignoreWhitespace: true }) - const updatedOverridesContent = - extractOverridesFromPnpmLockfileContent(lockfileContent) - if (updatedOverridesContent) { - lockfileContent = lockfileContent!.replace( - updatedOverridesContent, - revertOverridesContent, - ) - // eslint-disable-next-line no-await-in-loop - await fs.writeFile(lockfilePath, lockfileContent, 'utf8') - } - if (test) { - spinner?.info(`Testing ${newId} in ${workspace}.`) - // eslint-disable-next-line no-await-in-loop - await runScript(testScript, [], { spinner, stdio: 'ignore' }) - } - spinner?.success(`Fixed ${name} in ${workspace}.`) - fixedVersions.add(newVersion) - } else { - errored = true - } - } catch (e) { - error = e - errored = true - } - - spinner?.stop() - - // Check repoInfo to make TypeScript happy. - if (!errored && ciEnv?.repoInfo) { - try { - // eslint-disable-next-line no-await-in-loop - const result = await gitUnstagedModifiedFiles(cwd) - if (!result.ok) { - logger.warn( - 'Unexpected condition: Nothing to commit, skipping PR creation.', - ) - continue - } - const moddedFilepaths = result.data.filter(p => { - const basename = path.basename(p) - return ( - basename === 'package.json' || basename === 'pnpm-lock.yaml' - ) - }) - if (!moddedFilepaths.length) { - logger.warn( - 'Unexpected condition: Nothing to commit, skipping PR creation.', - ) - continue infosLoop - } - - const branch = getSocketBranchName(oldPurl, newVersion, workspace) - let skipPr = false - if ( - // eslint-disable-next-line no-await-in-loop - await prExistForBranch( - ciEnv.repoInfo.owner, - ciEnv.repoInfo.repo, - branch, - ) - ) { - skipPr = true - debugFn(`skip: branch "${branch}" exists`) - } - // eslint-disable-next-line no-await-in-loop - else if (await gitRemoteBranchExists(branch, cwd)) { - skipPr = true - debugFn(`skip: remote branch "${branch}" exists`) - } else if ( - // eslint-disable-next-line no-await-in-loop - !(await gitCreateAndPushBranch( - branch, - getSocketCommitMessage(oldPurl, newVersion, workspace), - moddedFilepaths, - { - cwd, - email: ciEnv.gitEmail, - user: ciEnv.gitUser, - }, - )) - ) { - skipPr = true - logger.warn( - 'Unexpected condition: Push failed, skipping PR creation.', - ) - } - if (skipPr) { - // eslint-disable-next-line no-await-in-loop - await gitResetAndClean(ciEnv.baseBranch, cwd) - // eslint-disable-next-line no-await-in-loop - const maybeActualTree = await install(pkgEnvDetails, { - cwd, - spinner, - }) - const maybeLockfileContent = maybeActualTree - ? // eslint-disable-next-line no-await-in-loop - await readPnpmLockfile(lockfilePath) - : null - if (maybeActualTree && maybeLockfileContent) { - actualTree = maybeActualTree - lockfileContent = maybeLockfileContent - continue infosLoop - } - // Exit early if install fails. - return handleInstallFail() - } - - // eslint-disable-next-line no-await-in-loop - await Promise.allSettled([ - setGitRemoteGithubRepoUrl( - ciEnv.repoInfo.owner, - ciEnv.repoInfo.repo, - ciEnv.githubToken!, - cwd, - ), - cleanupOpenPrs(ciEnv.repoInfo.owner, ciEnv.repoInfo.repo, { + // Update overrides in the root package.json so that when `pnpm install` + // generates pnpm-lock.yaml it updates transitive dependencies too. + editablePkgJson.update({ + [PNPM]: { + ...oldPnpmSection, + [OVERRIDES]: { + ...oldOverrides, + [overrideKey]: applyRange( + oldOverrides?.[overrideKey] ?? oldVersion, newVersion, - purl: oldPurl, - workspace, - }), - ]) - // eslint-disable-next-line no-await-in-loop - const prResponse = await openPr( - ciEnv.repoInfo.owner, - ciEnv.repoInfo.repo, - branch, - oldPurl, - newVersion, - { - baseBranch: ciEnv.baseBranch, - cwd, - workspace, - }, - ) - if (prResponse) { - const { data } = prResponse - const prRef = `PR #${data.number}` - logger.success(`Opened ${prRef}.`) - if (autoMerge) { - logger.indent() - spinner?.indent() - // eslint-disable-next-line no-await-in-loop - const { details, enabled } = await enablePrAutoMerge(data) - if (enabled) { - logger.info(`Auto-merge enabled for ${prRef}.`) - } else { - const message = `Failed to enable auto-merge for ${prRef}${ - details - ? `:\n${details.map(d => ` - ${d}`).join('\n')}` - : '.' - }` - logger.error(message) - } - logger.dedent() - spinner?.dedent() - } - } - } catch (e) { - error = e - errored = true - } - } - - if (ciEnv) { - spinner?.start() - // eslint-disable-next-line no-await-in-loop - await gitResetAndClean(ciEnv.baseBranch, cwd) - // eslint-disable-next-line no-await-in-loop - const maybeActualTree = await install(pkgEnvDetails, { - cwd, - spinner, - }) - const maybeLockfileContent = maybeActualTree - ? // eslint-disable-next-line no-await-in-loop - await readPnpmLockfile(lockfilePath) - : null - spinner?.stop() - if (maybeActualTree) { - actualTree = maybeActualTree - lockfileContent = maybeLockfileContent - } else { - errored = true - } - } - if (errored) { - if (!ciEnv) { - spinner?.start() - editablePkgJson.update(revertData) - // eslint-disable-next-line no-await-in-loop - await Promise.all([ - removeNodeModules(cwd), - editablePkgJson.save({ ignoreWhitespace: true }), - ]) - // eslint-disable-next-line no-await-in-loop - const maybeActualTree = await install(pkgEnvDetails, { - cwd, - spinner, - }) - const maybeLockfileContent = maybeActualTree - ? // eslint-disable-next-line no-await-in-loop - await readPnpmLockfile(lockfilePath) - : null - spinner?.stop() - if (maybeActualTree) { - actualTree = maybeActualTree - lockfileContent = maybeLockfileContent - } else { - // Exit early if install fails. - return handleInstallFail() - } - } - return { - ok: false, - message: 'Update failed', - cause: `Update failed for ${oldId} in ${workspace}${error ? '; ' + error : ''}`, - } - } - debugFn('name:', name) - debugFn('increment: count', count + 1) - if (++count >= limit) { - cleanupInfoEntriesLoop() - break infoEntriesLoop - } + options.rangeStyle, + ), + }, + }, + }) } - } - if (!isLastPkgJsonPath && logger.logCallCount > workspaceLogCallCount) { - logger.logNewline() - } - } - - for (const warningText of warningsForAfter) { - logger.warn(warningText) - } - if (!isLastInfoEntry) { - logger.logNewline() - } - cleanupInfoEntriesLoop() - } - spinner?.stop() - - // Or, did we change anything? - return { ok: true, data: { fixed: true } } + revertData = { + ...revertOverrides, + ...(editablePkgJson.content.dependencies && { + dependencies: { ...editablePkgJson.content.dependencies }, + }), + ...(editablePkgJson.content.optionalDependencies && { + optionalDependencies: { + ...editablePkgJson.content.optionalDependencies, + }, + }), + ...(editablePkgJson.content.peerDependencies && { + peerDependencies: { ...editablePkgJson.content.peerDependencies }, + }), + } as PackageJson + }, + async afterInstall(editablePkgJson) { + if (revertOverrides) { + // Revert overrides metadata in package.json now that pnpm-lock.yaml + // has been updated. + editablePkgJson.update(revertOverrides) + } + await editablePkgJson.save({ ignoreWhitespace: true }) + const updatedOverridesContent = extractOverridesFromPnpmLockSrc(lockSrc) + if (updatedOverridesContent && revertOverridesSrc) { + lockSrc = lockSrc!.replace( + updatedOverridesContent, + revertOverridesSrc, + ) + await fs.writeFile(pkgEnvDetails.lockPath, lockSrc, 'utf8') + } + }, + async revertInstall(editablePkgJson) { + if (revertData) { + editablePkgJson.update(revertData) + } + }, + }, + ciEnv, + openPrs, + options, + ) } diff --git a/src/commands/fix/run-fix.mts b/src/commands/fix/run-fix.mts deleted file mode 100644 index fab64dc45..000000000 --- a/src/commands/fix/run-fix.mts +++ /dev/null @@ -1,81 +0,0 @@ -import { logger } from '@socketsecurity/registry/lib/logger' - -import { npmFix } from './npm-fix.mts' -import { pnpmFix } from './pnpm-fix.mts' -import { CMD_NAME } from './shared.mts' -import constants from '../../constants.mts' -import { detectAndValidatePackageEnvironment } from '../../utils/package-environment.mts' - -import type { CResult } from '../../types.mts' -import type { RangeStyle } from '../../utils/semver.mts' - -const { NPM, PNPM } = constants - -export async function runFix({ - autoMerge, - cwd, - limit, - purls, - rangeStyle, - test, - testScript, -}: { - autoMerge: boolean - cwd: string - limit: number - purls: string[] - rangeStyle: RangeStyle - test: boolean - testScript: string -}): Promise> { - const result = await detectAndValidatePackageEnvironment(cwd, { - cmdName: CMD_NAME, - logger, - }) - - if (!result.ok) { - return result - } - const pkgEnvDetails = result.data - if (!pkgEnvDetails) { - return { - ok: false, - message: 'No package found', - cause: `No valid package environment was found in given cwd (${cwd})`, - } - } - - logger.info( - `Fixing packages for ${pkgEnvDetails.agent} v${pkgEnvDetails.agentVersion}.\n`, - ) - - const { agent } = pkgEnvDetails - - if (agent === NPM) { - return await npmFix(pkgEnvDetails, { - autoMerge, - cwd, - limit, - purls, - rangeStyle, - test, - testScript, - }) - } else if (agent === PNPM) { - return await pnpmFix(pkgEnvDetails, { - autoMerge, - cwd, - limit, - purls, - rangeStyle, - test, - testScript, - }) - } else { - return { - ok: false, - message: 'Not supported', - cause: `${agent} is not supported by this command at the moment.`, - } - } -} diff --git a/src/commands/optimize/get-overrides-by-agent.mts b/src/commands/optimize/get-overrides-by-agent.mts index 1df16501a..e4851e054 100644 --- a/src/commands/optimize/get-overrides-by-agent.mts +++ b/src/commands/optimize/get-overrides-by-agent.mts @@ -15,60 +15,60 @@ const { YARN_CLASSIC, } = constants -function getOverridesDataBun( +export function getOverridesDataBun( pkgEnvDetails: EnvDetails, pkgJson = pkgEnvDetails.editablePkgJson.content, ) { - const overrides = pkgJson?.[RESOLUTIONS] ?? ({} as PnpmOrYarnOverrides) + const overrides = (pkgJson?.[RESOLUTIONS] ?? {}) as PnpmOrYarnOverrides return { type: YARN_BERRY, overrides } } // npm overrides documentation: // https://docs.npmjs.com/cli/v10/configuring-npm/package-json#overrides -function getOverridesDataNpm( +export function getOverridesDataNpm( pkgEnvDetails: EnvDetails, pkgJson = pkgEnvDetails.editablePkgJson.content, ) { - const overrides = pkgJson?.[OVERRIDES] ?? ({} as NpmOverrides) + const overrides = (pkgJson?.[OVERRIDES] ?? {}) as NpmOverrides return { type: NPM, overrides } } // pnpm overrides documentation: // https://pnpm.io/package_json#pnpmoverrides -function getOverridesDataPnpm( +export function getOverridesDataPnpm( pkgEnvDetails: EnvDetails, pkgJson = pkgEnvDetails.editablePkgJson.content, ) { - const overrides = - (pkgJson as any)?.[PNPM]?.[OVERRIDES] ?? ({} as PnpmOrYarnOverrides) + const overrides = ((pkgJson as any)?.[PNPM]?.[OVERRIDES] ?? + {}) as PnpmOrYarnOverrides return { type: PNPM, overrides } } -function getOverridesDataVlt( +export function getOverridesDataVlt( pkgEnvDetails: EnvDetails, pkgJson = pkgEnvDetails.editablePkgJson.content, ) { - const overrides = pkgJson?.[OVERRIDES] ?? ({} as NpmOverrides) + const overrides = (pkgJson?.[OVERRIDES] ?? {}) as NpmOverrides return { type: VLT, overrides } } // Yarn resolutions documentation: // https://yarnpkg.com/configuration/manifest#resolutions -function getOverridesDataYarn( +export function getOverridesDataYarn( pkgEnvDetails: EnvDetails, pkgJson = pkgEnvDetails.editablePkgJson.content, ) { - const overrides = pkgJson?.[RESOLUTIONS] ?? ({} as PnpmOrYarnOverrides) + const overrides = (pkgJson?.[RESOLUTIONS] ?? {}) as PnpmOrYarnOverrides return { type: YARN_BERRY, overrides } } // Yarn resolutions documentation: // https://classic.yarnpkg.com/en/docs/selective-version-resolutions -function getOverridesDataYarnClassic( +export function getOverridesDataYarnClassic( pkgEnvDetails: EnvDetails, pkgJson = pkgEnvDetails.editablePkgJson.content, ) { - const overrides = pkgJson?.[RESOLUTIONS] ?? ({} as PnpmOrYarnOverrides) + const overrides = (pkgJson?.[RESOLUTIONS] ?? {}) as PnpmOrYarnOverrides return { type: YARN_CLASSIC, overrides } } diff --git a/src/utils/lockfile.mts b/src/utils/lockfile.mts new file mode 100644 index 000000000..4c78e9d5c --- /dev/null +++ b/src/utils/lockfile.mts @@ -0,0 +1,9 @@ +import { existsSync } from 'node:fs' + +import { readFileUtf8 } from './fs.mts' + +export async function readLockfile( + lockfilePath: string, +): Promise { + return existsSync(lockfilePath) ? await readFileUtf8(lockfilePath) : null +} diff --git a/src/utils/pnpm.mts b/src/utils/pnpm.mts index 74217bf14..db3386e80 100644 --- a/src/utils/pnpm.mts +++ b/src/utils/pnpm.mts @@ -12,9 +12,7 @@ import { idToNpmPurl } from './spec.mts' import type { LockfileObject, PackageSnapshot } from '@pnpm/lockfile.fs' import type { SemVer } from 'semver' -export function extractOverridesFromPnpmLockfileContent( - lockfileContent: any, -): string { +export function extractOverridesFromPnpmLockSrc(lockfileContent: any): string { return typeof lockfileContent === 'string' ? (/^overrides:(\r?\n {2}.+)+(?:\r?\n)*/m.exec(lockfileContent)?.[0] ?? '') : ''