diff --git a/packages/cli/contrib/build_appland_archive b/packages/cli/contrib/build_appland_archive new file mode 100755 index 0000000000..bfef9e7e8d --- /dev/null +++ b/packages/cli/contrib/build_appland_archive @@ -0,0 +1,54 @@ +#!/usr/bin/env ruby + +require 'fileutils' + +revision = ARGV[0] +no_test = true if ARGV[1..-1].member?('--no-test') + +cli_dir = Dir.pwd + +sys_command = ->(cmd, env = nil) do + Array(cmd).each do |c| + args = [ env, c ].compact + p args + system *args or raise "Command failed: #{c}" + end +end + +node_version = `node -v` +raise "Node version should be 16, got #{node_version}" unless node_version.index('v16.') + +Dir.chdir File.expand_path('~/source/appland/appland') +appland_dir = Dir.pwd + +required_ruby_version = File.read('.ruby-version').strip +ruby_version = `ruby -v` +raise "Ruby version must be #{required_ruby_version}, is #{ruby_version}" unless ruby_version.index(required_ruby_version) + +sys_command.call [ + "git checkout #{revision}", +] + +unless no_test + sys_command.call 'git lfs track "*.tar"' \ + unless File.read('.gitattributes').index('*.tar filter=lfs diff=lfs merge=lfs -text') + + env = `bundle exec rake dev:db`.split("\n").select { |line| line.index('=') }.map { |line| k, v = line.split('='); [ k['export '.length..-1], v] }.to_h + + FileUtils.rm_rf 'tmp/appmap' + FileUtils.mkdir_p 'tmp/appmap/rspec' + FileUtils.mkdir_p '.appmap/archive/full' + + sys_command.call [ + 'npm install', + './bin/webpack', + 'psql -U postgres -h localhost -c "create database appland_test" || true', + 'rails db:migrate', + 'rails db:test:prepare', + 'bundle exec rspec' + ], env.merge('RAILS_ENV' => 'test', 'DISABLE_SPRING' => 'true') +end + +Dir.chdir cli_dir + +system "node ./built/src/cli.js archive" diff --git a/packages/cli/contrib/build_rails_sample_app_6th_ed_archive b/packages/cli/contrib/build_rails_sample_app_6th_ed_archive new file mode 100755 index 0000000000..3d46ccef1b --- /dev/null +++ b/packages/cli/contrib/build_rails_sample_app_6th_ed_archive @@ -0,0 +1,61 @@ +#!/usr/bin/env ruby + +require 'fileutils' + +revision = ARGV[0] +no_test = true if ARGV[1..-1]&.member?('--no-test') + +cli_dir = Dir.pwd + +sys_command = ->(cmd, env = nil) do + Array(cmd).each do |c| + args = [ env, c ].compact + p args + system *args or raise "Command failed: #{c}" + end +end + +node_version = `node -v` +raise "Node version should be 14, got #{node_version}" unless node_version.index('v14.') + +Dir.chdir File.expand_path('~/source/land-of-apps/sample_app_6th_ed') +project_dir = Dir.pwd + +required_ruby_version = '2.7.3' +ruby_version = `ruby -v` +raise "Ruby version must be #{required_ruby_version}, is #{ruby_version}" unless ruby_version.index(required_ruby_version) + +sys_command.call [ + "git checkout #{revision}", +] + +unless no_test + sys_command.call 'git lfs track "*.tar"' \ + unless File.read('.gitattributes').index('*.tar filter=lfs diff=lfs merge=lfs -text') + + File.write 'appmap.yml', <<~APPMAP_CONFIG +name: sample_app_6th_ed +language: ruby +appmap_dir: tmp/appmap +packages: + - path: app + - path: lib +APPMAP_CONFIG + + FileUtils.rm_rf 'tmp/appmap' + FileUtils.mkdir_p 'tmp/appmap/minitest' + FileUtils.mkdir_p '.appmap/archive/full' + + sys_command.call [ + 'bundle update mimemagic puma', + 'bundle info appmap || bundle add appmap', + 'bundle update appmap', + 'yarn install', + 'rails db:test:prepare', + 'rails test' + ], { 'RAILS_ENV' => 'test', 'DISABLE_SPRING' => 'true' } +end + +Dir.chdir cli_dir + +system "node ./built/src/cli.js archive -d #{project_dir}" diff --git a/packages/cli/package.json b/packages/cli/package.json index 8d48178aaa..64df351d3f 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -79,6 +79,7 @@ "@appland/sequence-diagram": "workspace:^1", "@sidvind/better-ajv-errors": "^0.9.1", "@types/moo": "^0.5.5", + "@zip.js/zip.js": "^2.6.75", "JSONStream": "^1.3.5", "ajv": "^8.6.3", "applicationinsights": "^2.1.4", diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index b2d4dc4edc..36e5f2d0eb 100755 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -33,6 +33,10 @@ const SequenceDiagramCommand = require('./cmds/sequenceDiagram'); const SequenceDiagramDiffCommand = require('./cmds/sequenceDiagramDiff'); const StatsCommand = require('./cmds/stats/stats'); import UploadCommand from './cmds/upload'; +const BuildArchive = require('./cmds/archive/archive'); +const RestoreArchive = require('./cmds/archive/restore'); +const UpdateAppMaps = require('./cmds/update'); +const DescribeChange = require('./cmds/describeChange'); class DiffCommand { public appMapNames: any; @@ -455,6 +459,10 @@ yargs(process.argv.slice(2)) .command(SequenceDiagramDiffCommand) .command(PruneCommand) .command(UploadCommand) + .command(DescribeChange) + .command(BuildArchive) + .command(RestoreArchive) + .command(UpdateAppMaps) .strict() .demandCommand() .help().argv; diff --git a/packages/cli/src/cmds/archive/Metadata.ts b/packages/cli/src/cmds/archive/Metadata.ts new file mode 100644 index 0000000000..56894ef095 --- /dev/null +++ b/packages/cli/src/cmds/archive/Metadata.ts @@ -0,0 +1,14 @@ +import { AppMapConfig } from '../../lib/loadAppMapConfig'; + +export type Metadata = { + versions: Record; + workingDirectory: string; + appMapDir: string; + commandArguments: Record; + baseRevision?: string; + revision: string; + timestamp: string; + oversizedAppMaps: string[]; + deletedAppMaps?: string[]; + config: AppMapConfig; +}; diff --git a/packages/cli/src/cmds/archive/archive.ts b/packages/cli/src/cmds/archive/archive.ts new file mode 100644 index 0000000000..44c5a44697 --- /dev/null +++ b/packages/cli/src/cmds/archive/archive.ts @@ -0,0 +1,275 @@ +import { exec } from 'child_process'; +import { basename, dirname, join, resolve } from 'path'; +import yargs from 'yargs'; +import { handleWorkingDirectory } from '../../lib/handleWorkingDirectory'; +import { locateAppMapDir } from '../../lib/locateAppMapDir'; +import { exists, verbose } from '../../utils'; +import PackageConfig from '../../../package.json'; +import { readFile, stat, unlink, writeFile } from 'fs/promises'; +import { glob } from 'glob'; +import { promisify } from 'util'; +import FingerprintDirectoryCommand from '../../fingerprint/fingerprintDirectoryCommand'; +import { + buildDiagram, + format, + FormatType, + SequenceDiagramOptions, + Specification, +} from '@appland/sequence-diagram'; +import { queue } from 'async'; +import { AppMapFilter, buildAppMap, Filter } from '@appland/models'; +import { DefaultMaxAppMapSizeInMB } from '../../lib/fileSizeFilter'; +import loadAppMapConfig from '../../lib/loadAppMapConfig'; +import { VERSION as IndexVersion } from '../../fingerprint/fingerprinter'; +import chalk from 'chalk'; +import gitRevision from './gitRevision'; +import { Metadata } from './Metadata'; + +const ArchiveVersion = '1.0'; +const { name: ApplandAppMapPackageName, version: ApplandAppMapPackageVersion } = PackageConfig; + +/** + * Default filters for each language - plus a set of default filters that apply to all languages. + */ +export const DefaultFilters = { + default: ['label:unstable', 'label:serialize', 'label:deserialize.safe', 'label:log'], + ruby: ['label:mvc.template.resolver', 'package:ruby', 'package:activesupport'], + python: [], + java: [], + javascript: [], +}; + +export const command = 'archive'; +export const describe = 'Build an AppMap archive from a directory containing AppMaps'; + +export const builder = (args: yargs.Argv) => { + args.option('directory', { + describe: 'program working directory', + type: 'string', + alias: 'd', + }); + + args.option('type', { + describe: 'archive type', + choices: ['full', 'incremental', 'auto'], + default: 'auto', + alias: 't', + }); + + args.option('revision', { + describe: `revision identifier. + +If not explicitly specified, the current git revision will be used. +When this command is used in an CI server, it's best to explicitly the provide the revision +from an environment variable provided by the CI system, such as GITHUB_HEAD_SHA, because the +commit of the current git revision may not be the one that triggered the build.`, + type: 'string', + alias: 'r', + }); + + args.option('output-dir', { + describe: `directory in which to save the output file. By default, it's .appmap/archive/.`, + type: 'string', + }); + + args.option('output-file', { + describe: 'output file name. Default output name is .tar', + type: 'string', + alias: 'f', + }); + + args.option('concurrency', { + describe: 'number of AppMap which will be processed in parallel', + type: 'number', + default: 5, + }); + + args.option('max-size', { + describe: 'maximum AppMap size that will be processed, in filesystem-reported MB', + default: DefaultMaxAppMapSizeInMB, + }); + + return args.strict(); +}; + +export const handler = async (argv: any) => { + verbose(argv.verbose); + + handleWorkingDirectory(argv.directory); + const workingDirectory = process.cwd(); + + const appmapConfig = await loadAppMapConfig(); + if (!appmapConfig) throw new Error(`Unable to load appmap.yml config file`); + + const appMapDir = await locateAppMapDir(); + + const { + concurrency, + maxSize, + type: typeArg, + revision: defaultRevision, + outputFile: outputFileNameArg, + } = argv; + const { outputDirArg } = argv; + + const maxAppMapSizeInBytes = Math.round(parseFloat(maxSize) * 1024 * 1024); + + console.log(`Building '${typeArg}' archive from ${appMapDir}`); + + const revision = await gitRevision(defaultRevision); + + console.log(`Building archive of revision ${revision}`); + const versions = { archive: ArchiveVersion, index: IndexVersion }; + versions[ApplandAppMapPackageName] = ApplandAppMapPackageVersion; + + process.chdir(appMapDir); + + process.stdout.write(`Indexing AppMaps...`); + const numIndexed = await new FingerprintDirectoryCommand('.').execute(); + process.stdout.write(`done (${numIndexed})\n`); + + console.log('Generating sequence diagrams'); + + const specOptions = { + loops: true, + } as SequenceDiagramOptions; + + const filter = new AppMapFilter(); + filter.declutter.hideMediaRequests.on = true; + filter.declutter.limitRootEvents.on = true; + + const preflightConfig = appmapConfig.preflight; + + const defaultFilterGroups = (): string[] => { + const result = ['default']; + if (appmapConfig.language) result.push(appmapConfig.language); + return result; + }; + + const configuredFilterGroups = (): string[] | undefined => { + if (!preflightConfig?.filter) return; + + return preflightConfig?.filter?.groups; + }; + + const filterGroups = configuredFilterGroups() || defaultFilterGroups(); + const filterGroupNames = filterGroups.map((group) => DefaultFilters[group]).flat(); + const filterNames = preflightConfig?.filter?.names || []; + + if (filterGroupNames.length > 0 || filterNames.length > 0) { + filter.declutter.hideName.on = true; + filter.declutter.hideName.names = [...new Set([...filterGroupNames, ...filterNames])].sort(); + } + + const oversizedAppMaps = new Array(); + const sequenceDiagramQueue = queue(async (appmapFileName: string) => { + // Determine size of file appmapFileName in bytes + const stats = await stat(appmapFileName); + if (stats.size > maxAppMapSizeInBytes) { + console.log( + `Skipping, and removing, ${appmapFileName} because its size of ${stats.size} exceeds the maximum size of ${maxSize} MB` + ); + oversizedAppMaps.push(appmapFileName); + await unlink(appmapFileName); + return; + } + + const fullAppMap = buildAppMap() + .source(await readFile(appmapFileName, 'utf8')) + .build(); + const filteredAppMap = filter.filter(fullAppMap, []); + const specification = Specification.build(filteredAppMap, specOptions); + const diagram = buildDiagram(appmapFileName, filteredAppMap, specification); + const diagramOutput = format(FormatType.JSON, diagram, appmapFileName); + const indexDir = join(dirname(appmapFileName), basename(appmapFileName, '.appmap.json')); + const diagramFileName = join(indexDir, 'sequence.json'); + await writeFile(diagramFileName, diagramOutput.diagram); + }, concurrency); + + (await promisify(glob)('**/*.appmap.json')).forEach((appmapFileName) => + sequenceDiagramQueue.push(appmapFileName) + ); + await sequenceDiagramQueue.drain(); + + const metadata: Metadata = { + versions, + workingDirectory, + appMapDir, + commandArguments: argv, + revision, + timestamp: Date.now().toString(), + oversizedAppMaps, + config: appmapConfig, + }; + + let type: string; + if (await exists('appmap_archive.json')) { + const existingMetadata = JSON.parse(await readFile('appmap_archive.json', 'utf8')); + const { revision: baseRevision } = existingMetadata; + if (typeArg === 'auto') { + console.log( + `The AppMap directory contains appmap_archive.json, so the archive type will be 'incremental'. +The base revision is ${baseRevision}.` + ); + } + if (typeArg === 'full') { + console.warn( + chalk.yellow( + `\nThe AppMap directory contains appmap_archive.json, so it looks like the directory contains incremental AppMaps +that build on revision ${baseRevision}. However, you've specified --type=full. You should +remove appmap_archive.json if your intention is to build a full archive. Otherwise, use --type=auto or --type=incremental.\n` + ) + ); + } else { + metadata.baseRevision = baseRevision; + } + type = 'incremental'; + } else { + if (typeArg === 'auto') { + console.log( + `The AppMap directory does not contain appmap_archive.json, so the archive type will be 'full'.` + ); + } else if (typeArg === 'incremental') { + throw new Error( + `AppMap directory does not contain appmap_archive.json, but you've specified --type=incremental. +The base revision cannot be determined, so either use --type=auto or --type=full.` + ); + } + type = 'full'; + } + + await new Promise((resolveCB, rejectCB) => { + exec( + `tar czf appmaps.tar.gz --exclude appmaps.tar.gz --exclude appmap_archive.json *`, + (error) => { + if (error) return rejectCB(error); + + resolveCB(); + } + ); + }); + + process.on('exit', () => unlink('appmaps.tar.gz')); + + await writeFile('appmap_archive.json', JSON.stringify(metadata, null, 2)); + + const outputFileName = outputFileNameArg || `${revision}.tar`; + + const defaultOutputDir = () => join('.appmap', 'archive', type); + const outputDir = outputDirArg || defaultOutputDir(); + + await new Promise((resolveCB, rejectCB) => { + exec( + `tar cf ${join( + resolve(workingDirectory, outputDir), + outputFileName + )} appmap_archive.json appmaps.tar.gz`, + (error) => { + if (error) return rejectCB(error); + + resolveCB(); + } + ); + }); + console.log(`Created AppMap archive ${join(outputDir, outputFileName)}`); +}; diff --git a/packages/cli/src/cmds/archive/gitAncestors.ts b/packages/cli/src/cmds/archive/gitAncestors.ts new file mode 100644 index 0000000000..8a29c5ee53 --- /dev/null +++ b/packages/cli/src/cmds/archive/gitAncestors.ts @@ -0,0 +1,11 @@ +import { exec } from 'child_process'; + +export default async function gitAncestors(): Promise { + return new Promise((resolve, reject) => { + exec('git rev-list HEAD', (error, stdout) => { + if (error) reject(error); + + resolve(stdout.trim().split('\n')); + }); + }); +} diff --git a/packages/cli/src/cmds/archive/gitDeletedFiles.ts b/packages/cli/src/cmds/archive/gitDeletedFiles.ts new file mode 100644 index 0000000000..857dbb6316 --- /dev/null +++ b/packages/cli/src/cmds/archive/gitDeletedFiles.ts @@ -0,0 +1,17 @@ +import { exec } from 'child_process'; + +export default async function gitDeletedFiles( + revision: string, + folders?: string[] +): Promise { + const command = [`git diff --name-only --no-renames --diff-filter=D ${revision}`]; + if (folders) command.push(...folders); + + return new Promise((resolve, reject) => { + exec(command.join(' '), (error, stdout) => { + if (error) reject(error); + + resolve(stdout.trim().split('\n').filter(Boolean)); + }); + }); +} diff --git a/packages/cli/src/cmds/archive/gitModifiedFiles.ts b/packages/cli/src/cmds/archive/gitModifiedFiles.ts new file mode 100644 index 0000000000..e1ffb48b78 --- /dev/null +++ b/packages/cli/src/cmds/archive/gitModifiedFiles.ts @@ -0,0 +1,20 @@ +import { exec } from 'child_process'; + +export default async function gitModifiedFiles( + revision: string, + diffFilters: string[], + folders?: string[] +): Promise { + const command = [`git diff --name-only --no-renames `]; + if (diffFilters.length > 0) command.push(`--diff-filter=${diffFilters.join('')}`); + command.push(revision); + if (folders) command.push(...folders); + + return new Promise((resolve, reject) => { + exec(command.join(' '), (error, stdout) => { + if (error) reject(error); + + resolve(stdout.trim().split('\n').filter(Boolean)); + }); + }); +} diff --git a/packages/cli/src/cmds/archive/gitNewFiles.ts b/packages/cli/src/cmds/archive/gitNewFiles.ts new file mode 100644 index 0000000000..8bb19a692e --- /dev/null +++ b/packages/cli/src/cmds/archive/gitNewFiles.ts @@ -0,0 +1,14 @@ +import { exec } from 'child_process'; + +export default async function gitNewFiles(folders?: string[]): Promise { + const command = ['git ls-files -o --exclude-standard']; + if (folders) command.push(...folders); + + return new Promise((resolve, reject) => { + exec(command.join(' '), (error, stdout) => { + if (error) reject(error); + + resolve(stdout.trim().split('\n').filter(Boolean)); + }); + }); +} diff --git a/packages/cli/src/cmds/archive/gitRevision.ts b/packages/cli/src/cmds/archive/gitRevision.ts new file mode 100644 index 0000000000..648b32a3b1 --- /dev/null +++ b/packages/cli/src/cmds/archive/gitRevision.ts @@ -0,0 +1,20 @@ +import { exec } from 'child_process'; + +export default async function gitRevision(defaultRevision?: string): Promise { + if (defaultRevision) return Promise.resolve(defaultRevision); + + const revision = await new Promise((resolve) => { + exec('git rev-parse HEAD', (error, stdout) => { + if (error) resolve(undefined); + + resolve(stdout.trim()); + }); + }); + + if (!revision) + throw new Error( + `Unable to determine revision. Use --revision to specify it, or run this command in a Git repo.` + ); + + return revision; +} diff --git a/packages/cli/src/cmds/archive/restore.ts b/packages/cli/src/cmds/archive/restore.ts new file mode 100644 index 0000000000..39e6620d19 --- /dev/null +++ b/packages/cli/src/cmds/archive/restore.ts @@ -0,0 +1,96 @@ +import assert from 'assert'; +import { rmdir, unlink } from 'fs/promises'; +import { glob } from 'glob'; +import { basename, join } from 'path'; +import { promisify } from 'util'; +import yargs from 'yargs'; +import FingerprintDirectoryCommand from '../../fingerprint/fingerprintDirectoryCommand'; +import { handleWorkingDirectory } from '../../lib/handleWorkingDirectory'; +import { verbose } from '../../utils'; +import gitAncestors from './gitAncestors'; +import gitRevision from './gitRevision'; +import unpackArchive from './unpackArchive'; + +export const command = 'restore'; +export const describe = 'Restore the most current available AppMap data from available archives'; + +export const builder = (args: yargs.Argv) => { + args.option('directory', { + describe: 'program working directory', + type: 'string', + alias: 'd', + }); + + args.option('revision', { + describe: `revision to restore`, + }); + + args.option('output-dir', { + describe: 'directory in which to restore the data', + type: 'string', + demandOption: true, + }); + + args.option('archive-dir', { + describe: 'directory in which the archives are stored', + type: 'string', + default: '.appmap/archive', + }); + + return args.strict(); +}; + +export const handler = async (argv: any) => { + verbose(argv.verbose); + + handleWorkingDirectory(argv.directory); + + const { revision: defaultRevision, outputDir, archiveDir } = argv; + + const revision = await gitRevision(defaultRevision); + + console.log(`Restoring AppMaps of revision ${revision} to ${outputDir}`); + + const ancestors = await gitAncestors(); + + // Find the AppMap tarball that's closest in the git ancestry. + const fullArchivesAvailable = await promisify(glob)(join(archiveDir, 'full', '*.tar')); + const mostRecentFullArchive = fullArchivesAvailable.find((archive) => { + const revision = basename(archive, '.tar'); + return ancestors.includes(revision); + }); + if (!mostRecentFullArchive) throw new Error(`No AppMap tarball found in the ancestry of HEAD`); + + const baseRevision = basename(mostRecentFullArchive, '.tar'); + + console.log(`Using revision ${baseRevision} as the baseline`); + + await rmdir(outputDir, { recursive: true }); + + console.log(`Restoring full archive ${mostRecentFullArchive}`); + await unpackArchive(outputDir, mostRecentFullArchive); + + const baseRevisionIndex = ancestors.indexOf(baseRevision); + assert(baseRevisionIndex !== -1); + const ancestorsAfterBaseRevision = new Set(ancestors.slice(0, baseRevisionIndex)); + const incrementalArchivesAvailable = ( + await promisify(glob)(join(archiveDir, 'incremental', '*.tar')) + ).filter((archive) => { + const revision = basename(archive, '.tar'); + return ancestorsAfterBaseRevision.has(revision); + }); + + console.log(`Applying incremental archives ${incrementalArchivesAvailable.join(', ')}`); + + for (const archive of incrementalArchivesAvailable) { + await unpackArchive(outputDir, archive); + } + + console.log(`Updating indexes`); + + process.stdout.write(`Indexing AppMaps...`); + const numIndexed = await new FingerprintDirectoryCommand(outputDir).execute(); + process.stdout.write(`done (${numIndexed})\n`); + + console.log(`Restore complete`); +}; diff --git a/packages/cli/src/cmds/archive/runTests.ts b/packages/cli/src/cmds/archive/runTests.ts new file mode 100644 index 0000000000..caab643da6 --- /dev/null +++ b/packages/cli/src/cmds/archive/runTests.ts @@ -0,0 +1,41 @@ +import chalk from 'chalk'; +import { spawn } from 'child_process'; +import { mkdtemp, rmdir, writeFile } from 'fs/promises'; +import { tmpdir } from 'os'; +import { join } from 'path'; + +export default async function runTests(baseCommand: string, testFiles: string[]) { + const injectTestFiles = (): string | undefined => { + if (!baseCommand.includes('{testFiles}')) return; + + const [prefix, suffix] = baseCommand.split('{testFiles}'); + return [prefix, testFiles.join(' '), suffix].join(''); + }; + + const argumentTestFiles = async (): Promise => { + const tempDir = await mkdtemp(join(tmpdir(), 'appmap_')); + + process.on('exit', () => rmdir(tempDir, { recursive: true })); + + await writeFile(join(tempDir, 'added'), testFiles.join('\n')); + + return [baseCommand, join(tempDir, 'added')].join(' '); + }; + + const command = injectTestFiles() || (await argumentTestFiles()); + + console.log(chalk.magenta(`Running tests: ${command}`)); + + return new Promise((resolve) => { + const cmd = spawn(command.split(' ')[0], command.split(' ').slice(1)); + cmd.stderr.pipe(process.stderr); + cmd.stdout.pipe(process.stdout); + cmd.on('close', (code) => { + if (code) { + console.error(chalk.red(`Test command failed (${code})`)); + } + + resolve(); + }); + }); +} diff --git a/packages/cli/src/cmds/archive/unpackArchive.ts b/packages/cli/src/cmds/archive/unpackArchive.ts new file mode 100644 index 0000000000..0110ddd6e1 --- /dev/null +++ b/packages/cli/src/cmds/archive/unpackArchive.ts @@ -0,0 +1,40 @@ +import { exec } from 'child_process'; +import { mkdir, readFile, rename, rmdir, unlink } from 'fs/promises'; +import { basename, join, resolve } from 'path'; +import { Metadata } from './Metadata'; + +export default async function unpackArchive(outputDir: any, archivePath: string) { + await mkdir(outputDir, { recursive: true }); + + const dir = process.cwd(); + try { + process.chdir(outputDir); + + await new Promise((resolveCB, rejectCB) => { + exec(`tar xf ${resolve(dir, archivePath)}`, (error) => { + if (error) rejectCB(error); + + resolveCB(); + }); + }); + await new Promise((resolveCB, rejectCB) => { + exec(`tar xf appmaps.tar.gz`, (error) => { + if (error) rejectCB(error); + + resolveCB(); + }); + }); + await unlink('appmaps.tar.gz'); + + const metadata: Metadata = JSON.parse(await readFile('appmap_archive.json', 'utf8')); + await rename('appmap_archive.json', `appmap_archive.${metadata.revision}.json`); + + const deletedAppMaps = metadata.deletedAppMaps || []; + for (const deletedAppMap of deletedAppMaps) { + await unlink(deletedAppMap); + await rmdir(basename(deletedAppMap, '.appmap.json'), { recursive: true }); + } + } finally { + process.chdir(dir); + } +} diff --git a/packages/cli/src/cmds/describeChange.ts b/packages/cli/src/cmds/describeChange.ts new file mode 100644 index 0000000000..be0605ddaf --- /dev/null +++ b/packages/cli/src/cmds/describeChange.ts @@ -0,0 +1,688 @@ +import { mkdir, rm, writeFile } from 'fs/promises'; +import yargs from 'yargs'; +import readline from 'readline'; +import { OpenAPIV3 } from 'openapi-types'; + +import { handleWorkingDirectory } from '../lib/handleWorkingDirectory'; +import { locateAppMapDir } from '../lib/locateAppMapDir'; +import { exists, verbose } from '../utils'; +import { isAbsolute, join, relative } from 'path'; +import { Diagram, format, FormatType, ServerRPC } from '@appland/sequence-diagram'; +import { glob } from 'glob'; +import { promisify } from 'util'; +import { DiffDiagrams } from '../sequenceDiagramDiff/DiffDiagrams'; +import assert from 'assert'; +import chalk from 'chalk'; +import { AppMapReference } from '../describeChange/AppMapReference'; +import { RevisionName } from '../describeChange/RevisionName'; +import { OpenAPICommand } from '../openapi/OpenAPICommand'; +import { DefaultMaxAppMapSizeInMB, fileSizeFilter } from '../openapi/fileSizeFilter'; +import buildChangeReport from '../describeChange/buildChangeReport'; +import { executeCommand } from '../describeChange/executeCommand'; +import { OperationReference } from '../describeChange/OperationReference'; +import { Operation } from '../describeChange/types'; +import puppeteer from 'puppeteer'; +import { renderSequenceDiagramPNG } from '../sequenceDiagram/renderSequenceDiagramPNG'; +import BrowserRenderer from './sequenceDiagram/browserRenderer'; + +export class ValidationError extends Error {} + +export const command = 'describe-change'; +export const describe = 'Compare base and head revisions'; + +export const builder = (args: yargs.Argv) => { + args.option('directory', { + describe: 'program working directory', + type: 'string', + alias: 'd', + }); + + args.option('base-revision', { + describe: 'base revision name or commit SHA (default is the previous commit)', + alias: 'base', + }); + args.option('head-revision', { + describe: 'head revision name or commit SHA (default is the current commit)', + alias: 'head', + }); + + args.option('output-dir', { + describe: 'directory in which to save the report files', + }); + args.option('clobber-output-dir', { + describe: 'remove the output directory if it exists', + type: 'boolean', + default: false, + }); + + args.option('base-command', { + describe: 'command to use to run the base tests', + }); + args.option('head-command', { + describe: 'command to use to run the head tests', + }); + + args.option('include', { + describe: 'code objects to include in the comparison (inclusive of descendants)', + }); + args.option('exclude', { + describe: 'code objects to exclude from the comparison', + }); + + return args.strict(); +}; + +export const handler = async (argv: any) => { + verbose(argv.verbose); + + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, + }); + rl.on('close', function () { + yargs.exit(0, new Error()); + }); + + handleWorkingDirectory(argv.directory); + const appmapDir = await locateAppMapDir(); + + const { baseCommand, headCommand } = argv; + let { outputDir } = argv; + + let currentBranch = (await executeCommand(`git branch --show-current`)).trim(); + if (currentBranch === '') { + currentBranch = (await executeCommand(`git show --format=oneline --abbrev-commit`)).split( + /\s/ + )[0]; + } + + console.log(prominentStyle(`Current revision is: ${currentBranch}`)); + console.log( + mutedStyle( + `Note: The repository will be returned to this revision if this command succeeds, but not if it fails.` + ) + ); + + const baseRevision = argv.baseRevision || 'HEAD~1'; + const headRevision = argv.headRevision || (currentBranch || 'HEAD').trim(); + + if (baseRevision === headRevision) { + throw new ValidationError(`Base and head revisions are the same: ${baseRevision}`); + } + + if (!outputDir) { + outputDir = `revision-report/${sanitizeRevision(baseRevision)}-${sanitizeRevision( + headRevision + )}`; + } + + if (await exists(outputDir)) { + if ( + argv.clobberOutputDir || + !(await confirm(`Use existing data directory ${outputDir}?`, rl)) + ) { + if ( + !argv.clobberOutputDir && + !(await confirm(`Delete existing data directory ${outputDir}?`, rl)) + ) { + const msg = `The data directory ${outputDir} exists but you don't want to use it or delete it. Aborting...`; + console.warn(msg); + yargs.exit(1, new Error(msg)); + } + await rm(outputDir, { recursive: true, force: true }); + // Rapid rm and then mkdir will silently fail in practice. + await new Promise((resolve) => setTimeout(resolve, 100)); + } + } + + // stashAll() + + await mkdir(outputDir, { recursive: true }); + await mkdir(join(outputDir, RevisionName.Diff), { recursive: true }); + + const operationReference = new OperationReference(outputDir); + const appmapReferences = new Map(); + let baseAppMapFileNames: Set; + let headAppMapFileNames: Set; + + const processAppMaps = async ( + revisionName: RevisionName, + revision: string, + command?: string + ): Promise> => { + await mkdir(join(outputDir, revisionName), { recursive: true }); + await mkdir(join(outputDir, revisionName, 'operations'), { + recursive: true, + }); + + await checkout(revisionName, revision); + await createBaselineAppMaps(rl, command); + process.stdout.write(`Processing AppMaps in ${appmapDir}...`); + const result = new Set([ + ...(await processAppMapDir( + appmapDir, + outputDir, + revisionName, + operationReference, + appmapReferences + )), + ]); + process.stdout.write(`done (${result.size})\n`); + return result; + }; + + const restoreAppMaps = async (revisionName: RevisionName): Promise> => { + process.stdout.write( + `Loading existing AppMap and diagram data from ${join(outputDir, revisionName)}...` + ); + const result = new Set([ + ...(await restoreAppMapDir(outputDir, revisionName, operationReference, appmapReferences)), + ]); + process.stdout.write(`done (${result.size})\n`); + return result; + }; + + const processOrRestoreAppMaps = async ( + revisionName: RevisionName, + revision: string, + command?: string + ): Promise> => { + if (await exists(join(outputDir, revisionName))) { + // TODO: This is temporary; operation reference indexing is not needed here. + operationReference.startIndexing(); + const result = await restoreAppMaps(revisionName); + await operationReference.finishIndexing(); + return result; + } else { + operationReference.startIndexing(); + const result = await processAppMaps(revisionName, revision, command); + await operationReference.finishIndexing(); + return result; + } + }; + + const buildOpenAPI = async (revisionName: RevisionName): Promise => { + const cmd = new OpenAPICommand(AppMapReference.outputPath(outputDir, revisionName)); + cmd.filter = fileSizeFilter(DefaultMaxAppMapSizeInMB * 1024 * 1024); + const [openapi] = await cmd.execute(); + return openapi.paths; + }; + + baseAppMapFileNames = await processOrRestoreAppMaps(RevisionName.Base, baseRevision, baseCommand); + headAppMapFileNames = await processOrRestoreAppMaps(RevisionName.Head, headRevision, headCommand); + + const basePaths = await buildOpenAPI(RevisionName.Base); + const headPaths = await buildOpenAPI(RevisionName.Head); + + // unstashAll() + + const diffDiagrams = new DiffDiagrams(); + + if (argv.include) + (Array.isArray(argv.include) ? argv.include : [argv.include]).forEach((expr: string) => + diffDiagrams.include(expr) + ); + if (argv.exclude) + (Array.isArray(argv.exclude) ? argv.exclude : [argv.exclude]).forEach((expr: string) => + diffDiagrams.exclude(expr) + ); + + const changeReport = await buildChangeReport( + diffDiagrams, + baseRevision, + basePaths, + headPaths, + operationReference + ); + + const operationUrl = (operation: Operation): string => + [operation.method.toUpperCase(), operation.path, `(${operation.status})`].join(' '); + + if (verbose()) console.warn(`Preparing browser for PNG rendering`); + const browserRender = new BrowserRenderer(argv.showBrowser); + + async function saveSequenceDiagram(subdir: string, diagram: Diagram, name?: string) { + if (!name) { + const { subtreeDigest } = diagram.rootActions[0]; + name = subtreeDigest; + } + const serverRPC = diagram.rootActions[0] as ServerRPC; + + let { route, status } = serverRPC; + const [method, path] = route.split(' '); + + const operationDir = join( + outputDir, + RevisionName.Diff, + subdir, + method.toUpperCase(), + path, + status.toString() + ); + await mkdir(operationDir, { recursive: true }); + await writeFile( + join(operationDir, [name, 'sequence.json'].join('.')), + format(FormatType.JSON, diagram, name).diagram + ); + + const diagramAsText = format(FormatType.Text, diagram, ''); + if (diagramAsText.diagram.trim().length > 0) + await writeFile(join(operationDir, [name, 'sequence.txt'].join('.')), diagramAsText.diagram); + + try { + await renderSequenceDiagramPNG( + join(operationDir, [name, 'sequence.png'].join('.')), + join(operationDir, [name, 'sequence.json'].join('.')), + browserRender + ); + } catch (e) { + console.warn(`Failed to render sequence diagram for ${operationDir}: ${e}`); + } + } + + await Promise.all( + changeReport.routeChanges.added.map(async (change) => { + console.log(`Added route: ${operationUrl(change.operation)}`); + if (change.sourceDiff) console.log(`Source diff: ${change.sourceDiff}`); + console.log(`${change.sequenceDiagrams?.length} sequence diagrams`); + for (const diagram of change.sequenceDiagrams || []) { + assert(diagram.rootActions.length === 1, 'Expecting a single root action'); + + await saveSequenceDiagram('added', diagram); + } + }) + ); + + await browserRender.close(); + + changeReport.routeChanges.removed.forEach((change) => { + console.log(`Removed route: ${operationUrl(change.operation)}`); + if (change.sourceDiff) console.log(`Source diff: ${change.sourceDiff}`); + }); + await Promise.all( + changeReport.routeChanges.changed.map(async (change) => { + if (!change.sourceDiff && (!change.sequenceDiagrams || change.sequenceDiagrams.length === 0)) + return; + + assert(change.sequenceDiagrams); + console.log(`Changed route: ${operationUrl(change.operation)}`); + if (change.sourceDiff) console.log(`Source diff: ${change.sourceDiff}`); + console.log(`${change.sequenceDiagrams?.length} sequence diagrams`); + for (let index = 0; index < change.sequenceDiagrams.length; index++) { + const diagram = change.sequenceDiagrams[index]; + assert(diagram.rootActions.length === 1, 'Expecting a single root action'); + await saveSequenceDiagram('changed', diagram, `diagram_${index + 1}`); + } + }) + ); + + if (changeReport.failedTests) { + console.log(`${changeReport.failedTests.length} tests failed`); + for (const test of changeReport.failedTests) { + console.log(`${test.testLocation} (${test.name})`); + console.log(`${test.appmapFile}`); + console.log(`Log messages:`); + for (const logEntry of test.logEntries) { + process.stdout.write(`${logEntry.message}`); + if (verbose()) console.log(` ${logEntry.stack.join('\n ')}`); + } + } + } else { + console.log(`All tests passed`); + } + + /* + + const headAppMapFileNameArray = [...headAppMapFileNames].sort(); + const changedAppMaps = new Array(); + const diffSnippets = new Set(); + const reportLines = new Array(); + for (let i = 0; i < headAppMapFileNameArray.length; i++) { + const appmapFileName = headAppMapFileNameArray[i]; + const appmapReference = appmapReferences.get(appmapFileName); + assert(appmapReference); + + if (!baseAppMapFileNames.has(appmapFileName)) { + console.log(`${appmapFileName} is new in the head revision`); + await printSourceDiff(baseRevision, appmapReference, rl); + await reportDiagram( + RevisionName.Head, + outputDir, + baseRevision, + headRevision, + appmapReference, + reportLines + ); + continue; + } + + const diffDiagrams = new DiffDiagrams(); + + if (argv.include) + (Array.isArray(argv.include) ? argv.include : [argv.include]).forEach((expr: string) => + diffDiagrams.include(expr) + ); + if (argv.exclude) + (Array.isArray(argv.exclude) ? argv.exclude : [argv.exclude]).forEach((expr: string) => + diffDiagrams.exclude(expr) + ); + + let diffDiagram: Diagram | undefined; + const baseDiagram = await appmapReference.loadSequenceDiagramJSON(RevisionName.Base); + const headDiagram = await appmapReference.loadSequenceDiagramJSON(RevisionName.Head); + try { + diffDiagram = diffDiagrams.diff(baseDiagram, headDiagram); + if (!diffDiagram) continue; + } catch (e) { + console.warn(`Error comparing ${appmapFileName} ${baseRevision}..${headRevision}: ${e}`); + continue; + } + + const diffActions = new Set(); + const markDiffActions = (action: Action): void => { + if (action.diffMode) { + let ancestor: Action | undefined = action; + while (ancestor) { + diffActions.add(ancestor); + ancestor = ancestor.parent; + } + } + action.children.forEach(markDiffActions); + }; + + diffDiagram.rootActions.forEach(markDiffActions); + + const diffActors = new Set(); + const filterDiffActions = (actions: Action[]): Action[] => { + const result = actions.filter((action) => diffActions.has(action)); + result.forEach((action) => { + actionActors(action).forEach((actor) => (actor ? diffActors.add(actor.name) : undefined)); + action.children = filterDiffActions(action.children); + }); + return result; + }; + diffDiagram.rootActions = filterDiffActions(diffDiagram.rootActions); + diffDiagram.actors = diffDiagram.actors.filter((actor) => diffActors.has(actor.name)); + + await writeFile( + appmapReference.sequenceDiagramFilePath(RevisionName.Diff, FormatType.JSON, true), + format(FormatType.JSON, diffDiagram, `Compare ${appmapFileName}`).diagram + ); + + const diffText = format( + FormatType.Text, + diffDiagram, + `Compare ${appmapFileName}` + ).diagram.trim(); + + await writeFile( + appmapReference.sequenceDiagramFilePath(RevisionName.Diff, FormatType.Text, true), + diffText + ); + + if (diffSnippets.has(diffText)) continue; + + diffSnippets.add(diffText); + + changedAppMaps.push(appmapReference); + } + + console.log( + prominentStyle(`${changedAppMaps.length} AppMaps have changed between these two code versions.`) + ); + console.log(); + + for (let i = 0; i < changedAppMaps.length; i++) { + const appmapReference = changedAppMaps[i]; + const diagramText = await appmapReference.loadSequenceDiagramText(RevisionName.Diff); + + const diagramTextLines = diagramText + .split('\n') + .filter((line) => !/^\d+ times:$/.test(line.trim())); + if (diagramTextLines.length === 0) { + continue; + } + + await reportDiagram( + RevisionName.Diff, + outputDir, + baseRevision, + headRevision, + appmapReference, + reportLines + ); + } + + if (outputDir) { + const report = ` + AppMap code change report + + + ${reportLines.join('\n')} +`; + const diagramHTML = ` + Sequence diagram + + +
+ +`; + const appmapHTML = ` + AppMap + + +
+ +`; + await writeFile(join(outputDir, 'report.html'), report); + await writeFile(join(outputDir, 'diagram.html'), diagramHTML); + await writeFile(join(outputDir, 'appmap.html'), appmapHTML); + } + + */ + rl.close(); +}; + +async function ask(rl: readline.Interface, q: string): Promise { + return new Promise((resolve) => { + rl.question(q, resolve); + }); +} + +async function createBaselineAppMaps(rl: readline.Interface, testCommand?: string): Promise { + if (testCommand) { + await executeCommand(testCommand); + return; + } + + console.log( + prominentStyle( + `Generate the AppMaps for this revision in a separate terminal - for example, by running the tests.` + ) + ); + await waitForEnter(rl, `Press Enter when the AppMaps are ready`); +} + +async function checkout(revisionName: string, revision: string): Promise { + console.log(); + console.log(actionStyle(`Switching to ${revisionName} revision: ${revision}`)); + await executeCommand(`git checkout ${revision}`, true, false, false); + console.log(); +} + +async function processAppMapDir( + appmapDir: string, + outputDir: string, + revisionName: RevisionName, + operationReference: OperationReference, + appmapReferences: Map +): Promise { + const appmapFileNames = await promisify(glob)(`${appmapDir}/**/*.appmap.json`); + for (let i = 0; i < appmapFileNames.length; i++) { + const appmapFileName = appmapFileNames[i]; + const appmapReference = new AppMapReference(operationReference, outputDir, appmapFileName); + await appmapReference.processAppMap(revisionName); + if (!appmapReferences.get(appmapFileName)) { + appmapReferences.set(appmapFileName, appmapReference); + } + } + return appmapFileNames; +} + +async function restoreAppMapDir( + outputDir: string, + revisionName: RevisionName, + operationReference: OperationReference, + appmapReferences: Map +): Promise { + const baseDir = join(outputDir, revisionName); + const appmapFileNames = await promisify(glob)(`${baseDir}/*.appmap.json`); + for (let i = 0; i < appmapFileNames.length; i++) { + const appmapFileName = appmapFileNames[i]; + const appmapReference = new AppMapReference( + operationReference, + outputDir, + relative(baseDir, appmapFileName) + ); + await appmapReference.restoreMetadata(revisionName); + if (!appmapReferences.get(appmapReference.appmapFileName)) + appmapReferences.set(appmapReference.appmapFileName, appmapReference); + } + return appmapFileNames.map((fileName) => relative(baseDir, fileName)); +} + +async function confirm(prompt: string, rl: readline.Interface): Promise { + let response = ''; + while (!['y', 'n'].includes(response)) { + response = await ask(rl, `${prompt} (y/n) `); + } + return response === 'y'; +} + +function actionStyle(message: string): string { + return chalk.bold(chalk.green(message)); +} +function prominentStyle(message: string): string { + return chalk.bold(message); +} +function mutedStyle(message: string): string { + return chalk.dim(message); +} +export function commandStyle(message: string): string { + return chalk.gray(`$ ${message}`); +} +async function waitForEnter( + rl: readline.Interface, + prompt = 'Press Enter to continue' +): Promise { + console.log(prominentStyle(`${prompt}...`)); + await new Promise((resolve) => { + const listener = () => { + rl.removeListener('line', listener); + resolve(); + }; + rl.addListener('line', listener); + }); +} + +async function reportDiagram( + revisionName: RevisionName, + outputDir: string, + baseRevision: string, + headRevision: string, + appmapReference: AppMapReference, + reportLines: string[] +): Promise { + if (true /* await confirm(`Include in the change report?`, rl)) */) { + const diagramText = await appmapReference.loadSequenceDiagramText(revisionName); + + const appmapName = appmapReference.appmapName || appmapReference.appmapFileName; + + if (revisionName === RevisionName.Head) { + reportLines.push( + `

+ ${appmapName} is new in ${headRevision} +

` + ); + } else { + assert( + revisionName === RevisionName.Diff, + `Expecting revisionName to be '${RevisionName.Diff}', got '${revisionName}'` + ); + reportLines.push( + `

+ ${appmapName} has changed +

` + ); + } + + if (appmapReference.sourceLocation) { + const sourcePath = appmapReference.sourceLocation.split(':')[0]; + const fileURL = isAbsolute(sourcePath) + ? relative(outputDir, sourcePath) + : relative(outputDir, join(process.cwd(), sourcePath)); + reportLines.push(''); + reportLines.push( + `${relative(process.cwd(), appmapReference.sourceLocation)}` + ); + reportLines.push(''); + } + + const sourceDiff = await appmapReference.sourceDiff(baseRevision); + if (sourceDiff) { + reportLines.push('
'); + reportLines.push(''); + reportLines.push('
');
+      reportLines.push(sourceDiff);
+      reportLines.push('
'); + reportLines.push(''); + reportLines.push('
'); + } + + reportLines.push('
'); + reportLines.push('
'); + reportLines.push(''); + reportLines.push('
');
+    reportLines.push(diagramText);
+    reportLines.push('
'); + reportLines.push(''); + // reportLines.push(''); + // reportLines.push( + // `![${appmapName}](${appmapReference.sequenceDiagramFilePath(revisionName, 'svg', false)})` + // ); + // reportLines.push(''); + } +} + +function sanitizeRevision(revision: string): string { + return revision.replace(/[^a-zA-Z0-9_]/g, '_'); +} + +async function printSourceDiff( + baseRevision: any, + appmapReference: AppMapReference, + rl: readline.Interface +): Promise { + const sourceDiff = await appmapReference.sourceDiff(baseRevision); + if (sourceDiff) { + // if (await confirm(`View ${sourceDiff.split('\n').length} line source diff?`, rl)) { + console.log(); + console.log(sourceDiff); + console.log(); + // } + } +} + +function printDiagramText(diagramTextLines: string[]): void { + console.log(`${diagramTextLines.length} line sequence diagram diff:`); + console.log(); + diagramTextLines.forEach((line) => console.log(line)); + console.log(); +} diff --git a/packages/cli/src/cmds/openapi.ts b/packages/cli/src/cmds/openapi.ts index ee2d2150e0..7f9c96fe41 100644 --- a/packages/cli/src/cmds/openapi.ts +++ b/packages/cli/src/cmds/openapi.ts @@ -1,128 +1,20 @@ import { join } from 'path'; -import { existsSync, promises as fsp, Stats } from 'fs'; -import { readFile, stat } from 'fs/promises'; -import { queue } from 'async'; -import { glob } from 'glob'; +import { promises as fsp } from 'fs'; +import { readFile } from 'fs/promises'; import yaml, { load } from 'js-yaml'; import { OpenAPIV3 } from 'openapi-types'; -import { - Model, - parseHTTPServerRequests, - rpcRequestForEvent, - SecuritySchemes, - verbose, -} from '@appland/openapi'; -import { Event } from '@appland/models'; +import { verbose } from '@appland/openapi'; import { Arguments, Argv, number } from 'yargs'; -import { inspect } from 'util'; import { locateAppMapDir } from '../lib/locateAppMapDir'; import { handleWorkingDirectory } from '../lib/handleWorkingDirectory'; import { locateAppMapConfigFile } from '../lib/locateAppMapConfigFile'; import Telemetry, { Git, GitState } from '../telemetry'; import { findRepository } from '../lib/git'; +import { DefaultMaxAppMapSizeInMB, fileSizeFilter } from '../lib/fileSizeFilter'; -type FilterFunction = (file: string) => Promise<{ enable: boolean; message?: string }>; - -// Skip files that are larger than a specified max size. -export function fileSizeFilter(maxFileSize: number): FilterFunction { - return async (file: string): Promise<{ enable: boolean; message?: string }> => { - let fileStat: Stats; - try { - fileStat = await stat(file); - } catch { - return { enable: false, message: `File ${file} not found` }; - } - - if (fileStat.size <= maxFileSize) return { enable: true }; - else - return { - enable: false, - message: `Skipping ${file} as its file size of ${fileStat.size} bytes is larger than the maximum configured file size of ${maxFileSize} bytes`, - }; - }; -} - -class OpenAPICommand { - private readonly model = new Model(); - private readonly securitySchemes = new SecuritySchemes(); - - public errors: string[] = []; - public filter: FilterFunction = async (file: string) => ({ enable: true }); - - constructor(private readonly appmapDir: string) {} - - async execute(): Promise< - [ - { - paths: OpenAPIV3.PathsObject; - securitySchemes: Record; - }, - number - ] - > { - const q = queue(this.collectAppMap.bind(this), 5); - q.pause(); - - // Make sure the directory exists -- if it doesn't, the glob below just returns nothing. - if (!existsSync(this.appmapDir)) { - throw new Error(`AppMap directory ${this.appmapDir} does not exist`); - } - - const files = glob.sync(join(this.appmapDir, '**', '*.appmap.json')); - for (let index = 0; index < files.length; index++) { - const file = files[index]; - const filterResult = await this.filter(file); - if (!filterResult.enable) { - if (filterResult.message) console.warn(filterResult.message); - continue; - } - - q.push(file); - } - - if (q.length() > 0) { - await new Promise((resolve, reject) => { - q.drain(resolve); - q.error(reject); - q.resume(); - }); - } - - return [ - { - paths: this.model.openapi(), - securitySchemes: this.securitySchemes.openapi(), - }, - files.length, - ]; - } - - async collectAppMap(file: string): Promise { - try { - const data = await fsp.readFile(file, 'utf-8'); - parseHTTPServerRequests(JSON.parse(data), (e: Event) => { - const request = rpcRequestForEvent(e); - if (request) { - this.model.addRpcRequest(request); - this.securitySchemes.addRpcRequest(request); - } - }); - } catch (e) { - // Re-throwing this error crashes the whole process. - // So if there is a malformed AppMap, indicate it here but don't blow everything up. - // Do not write to stdout! - let errorString: string; - try { - errorString = inspect(e); - } catch (x) { - errorString = ((e as any) || '').toString(); - } - this.errors.push(errorString); - } - } -} +export type FilterFunction = (file: string) => Promise<{ enable: boolean; message?: string }>; async function loadTemplate(fileName: string): Promise { if (!fileName) { @@ -148,7 +40,7 @@ export default { }); args.option('max-size', { describe: 'maximum AppMap size that will be processed, in filesystem-reported MB', - default: '50', + default: `${DefaultMaxAppMapSizeInMB}mb`, }); args.option('output-file', { alias: ['o'], diff --git a/packages/cli/src/cmds/sequenceDiagram.ts b/packages/cli/src/cmds/sequenceDiagram.ts index b36925820a..28739b33d4 100644 --- a/packages/cli/src/cmds/sequenceDiagram.ts +++ b/packages/cli/src/cmds/sequenceDiagram.ts @@ -12,7 +12,7 @@ import { Formatters, FormatType, } from '@appland/sequence-diagram'; -import { serveAndOpenSequenceDiagram } from '../lib/serveAndOpen'; +import { renderSequenceDiagramPNG } from '../sequenceDiagram/renderSequenceDiagramPNG'; import assert from 'assert'; import BrowserRenderer from './sequenceDiagram/browserRenderer'; @@ -112,22 +112,14 @@ export const handler = async (argv: any) => { if (argv.format === 'png') { // PNG rendering is performed by loading the sequence // diagram in a browser and taking a screenshot. + assert(browserRender); const diagramPath = await printDiagram(FormatType.JSON); - const outputPath = await new Promise((resolve) => - serveAndOpenSequenceDiagram(diagramPath, false, async (url) => { - if (verbose()) console.warn(`Rendering PNG`); - assert(browserRender, 'Browser not initialized'); - - const outputPath = join( - dirname(diagramPath), - [basename(diagramPath, '.json'), '.png'].join('') - ); - - await browserRender.screenshot(url, outputPath); - - resolve(outputPath); - }) + const outputPath = join( + dirname(diagramPath), + [basename(diagramPath, '.json'), '.png'].join('') ); + await renderSequenceDiagramPNG(outputPath, diagramPath, browserRender); + console.warn(`Printed diagram ${outputPath}`); } else { // Other forms of output are produced directly by the diff --git a/packages/cli/src/cmds/update.ts b/packages/cli/src/cmds/update.ts new file mode 100644 index 0000000000..d300a6c4d1 --- /dev/null +++ b/packages/cli/src/cmds/update.ts @@ -0,0 +1,210 @@ +import chalk from 'chalk'; +import { mkdir, readFile, writeFile } from 'fs/promises'; +import { glob } from 'glob'; +import { dirname, join, relative, resolve } from 'path'; +import { promisify } from 'util'; +import yargs from 'yargs'; +import { Metadata as AppMapMetadata } from '@appland/models'; +import Depends from '../depends'; +import { handleWorkingDirectory } from '../lib/handleWorkingDirectory'; +import loadAppMapConfig from '../lib/loadAppMapConfig'; +import { locateAppMapDir } from '../lib/locateAppMapDir'; +import { exists, verbose } from '../utils'; +import gitDeletedFiles from './archive/gitDeletedFiles'; +import gitModifiedFiles from './archive/gitModifiedFiles'; +import gitNewFiles from './archive/gitNewFiles'; +import { Metadata } from './archive/Metadata'; +import runTests from './archive/runTests'; +import FingerprintDirectoryCommand from '../fingerprint/fingerprintDirectoryCommand'; +import gitAncestors from './archive/gitAncestors'; +import assert from 'assert'; + +export const command = 'update'; +export const describe = 'Update AppMaps by running new and out-of-date tests'; + +export const builder = (args: yargs.Argv) => { + args.option('directory', { + describe: 'program working directory', + type: 'string', + alias: 'd', + }); + args.option('appmap-dir', { + describe: 'directory to recursively inspect for AppMaps', + }); + + args.option('base-appmap-dir', { + describe: 'directory in which base AppMaps are available', + type: 'string', + demandOption: true, + }); + + args.option('test-folder', { + describe: 'folder in which tests are located', + array: true, + }); + + args.option('test-command', { + describe: 'command to invoke with the updated (added + modified) tests', + array: true, + }); + + return args.strict(); +}; + +export const handler = async (argv: any) => { + verbose(argv.verbose); + + const { baseAppmapDir } = argv; + + handleWorkingDirectory(argv.directory); + const appmapDir = await locateAppMapDir(); + const appmapConfig = await loadAppMapConfig(); + if (!appmapConfig) throw new Error(`Unable to load appmap.yml config file`); + + const testFoldersArgument = (): string[] | undefined => argv.testFolder; + const testFoldersConfig = (): string[] | undefined => appmapConfig.preflight?.test_folders; + + const testFolders = testFoldersArgument() || testFoldersConfig(); + if (!testFolders) + console.warn( + chalk.yellow(`No test folders specified. Use --test-folder arguments, or configure +test_folders in appmap.yml. In the absence of this information, all added and changed files will +be included in the file arguments passed to the test runner.`) + ); + + const testCommandsArgument = (): Record | undefined => { + const testCommands = argv.testCommand as string[] | undefined; + if (!testCommands) return; + + return testCommands.reduce((memo, line) => { + const match = /\[([^\]]+)\]\s+(.*)/.exec(line); + if (!match) { + console.warn(chalk.red(`Invalid test command: ${line}`)); + } else { + const folder = match[1]; + const command = match[2]; + memo[folder] = command; + } + return memo; + }, {}); + }; + const testCommandsConfig = (): Record | undefined => + appmapConfig.preflight?.test_commands; + const testCommands = testCommandsArgument() || testCommandsConfig(); + + const archiveMetadata = await Promise.all( + ( + await promisify(glob)(join(baseAppmapDir, 'appmap_archive.*.json')) + ).map(async (metadataFile) => JSON.parse(await readFile(metadataFile, 'utf8'))) + ); + + const archiveRevisions = archiveMetadata.map((metadata) => metadata.revision); + const repoRevisions = await gitAncestors(); + const baseRevision = repoRevisions.find((revision) => archiveRevisions.includes(revision)); + + if (!baseRevision) throw new Error(`No AppMap archive found for any ancestor revision.`); + + console.log(`Comparing code changes with revision ${baseRevision}`); + + const baseArchive = archiveMetadata.find((metadata) => metadata.revision === baseRevision); + assert(baseArchive); + + const addedTestFiles = await gitNewFiles(testFolders); + const modifiedFiles = await gitModifiedFiles(baseRevision, []); + const modifiedTestFiles = await gitModifiedFiles(baseRevision, ['d'], testFolders); + const deletedTestFiles = await gitDeletedFiles(baseRevision, testFolders); + const outOfDateTestFiles = new Set([...addedTestFiles, ...modifiedTestFiles]); + + console.log(`${outOfDateTestFiles.size} test case files are added or modified.`); + + if (verbose()) { + if (addedTestFiles.length > 0) console.log(`Added tests: ${addedTestFiles.join(' ')}`); + if (modifiedTestFiles.length > 0) console.log(`Modified tests: ${modifiedTestFiles.join(' ')}`); + if (deletedTestFiles.length > 0) console.log(`Deleted tests: ${deletedTestFiles.join(' ')}`); + } + + const indexAppMaps = async (dir: string) => { + process.stdout.write(`Indexing AppMaps...`); + const numIndexed = await new FingerprintDirectoryCommand(dir).execute(); + process.stdout.write(`done (${numIndexed})\n`); + }; + + await indexAppMaps(baseAppmapDir); + + const dirtyTests = new Set([...modifiedTestFiles, ...deletedTestFiles]); + const deletedAppMaps = new Array(); + // Remove AppMaps for deleted and modified tests + await Promise.all( + ( + await promisify(glob)(join(appmapDir, '**/metadata.json')) + ).map(async (metadataFile) => { + const metadata = JSON.parse(await readFile(metadataFile, 'utf-8')); + const { source_location: sourceLocation } = metadata; + if (!sourceLocation) return; + + const [path] = sourceLocation.split(':'); + + if (dirtyTests.has(path)) { + const appmapName = dirname(metadataFile); + console.warn( + `AppMap ${appmapName} is out of date because ${path} has been ${ + modifiedTestFiles.includes(path) ? 'modified' : 'deleted' + }` + ); + deletedAppMaps.push(appmapName); + } + }) + ); + + const depends = new Depends(baseAppmapDir); + depends.files = modifiedFiles; + const outOfDateAppMaps = await depends.depends(); + if (outOfDateAppMaps.length > 0) { + console.log(`${outOfDateAppMaps.length} AppMaps are out of date.`); + if (verbose()) console.log(`Out-of-date AppMaps: ${outOfDateAppMaps.join(' ')}`); + for (const appmap of outOfDateAppMaps) { + const metadata = JSON.parse( + await readFile(join(appmap, 'metadata.json'), 'utf-8') + ) as AppMapMetadata; + if (metadata.source_location) outOfDateTestFiles.add(metadata.source_location.split(':')[0]); + } + } + + if (verbose()) { + if (outOfDateTestFiles.size > 0) + console.log(`Updated (added+modified) tests: ${[...outOfDateTestFiles].join(' ')}`); + } + + if (outOfDateTestFiles.size === 0) { + console.log('No changes detected to the code or tests.'); + return; + } + + console.log(`${outOfDateTestFiles.size} tests will be re-run.`); + + await mkdir(appmapDir, { recursive: true }); + + const testFilesByFolder = new Map(); + for (const testFile of outOfDateTestFiles) { + const localPath = relative(process.cwd(), testFile); + if (!(await exists(testFile))) { + console.warn(chalk.yellow(`Test file ${testFile} does not exist`)); + continue; + } + testFolders?.forEach((folder) => { + if (localPath.startsWith(folder)) + testFilesByFolder.set(folder, [...(testFilesByFolder.get(folder) || []), localPath]); + }); + } + + for (const [folder, testFiles] of testFilesByFolder.entries()) { + const command = testCommands?.[folder]; + if (!command) { + console.warn(chalk.yellow(`No test command configured for folder ${folder}`)); + continue; + } + await runTests(command, testFiles); + } + + await writeFile(join(appmapDir, 'appmap_archive.json'), JSON.stringify(baseArchive, null, 2)); +}; diff --git a/packages/cli/src/depends.js b/packages/cli/src/depends.js index 2845b1df7f..c08e26fe81 100644 --- a/packages/cli/src/depends.js +++ b/packages/cli/src/depends.js @@ -66,7 +66,7 @@ class Depends { * @param {function} callback * @returns string[] */ - async depends(callback) { + async depends(callback = undefined) { const outOfDateNames = new Set(); const checkClassMap = async (fileName) => { diff --git a/packages/cli/src/describeChange/AppMapReference.ts b/packages/cli/src/describeChange/AppMapReference.ts new file mode 100644 index 0000000000..5df9941093 --- /dev/null +++ b/packages/cli/src/describeChange/AppMapReference.ts @@ -0,0 +1,235 @@ +import { copyFile, readFile, writeFile } from 'fs/promises'; +import { basename, dirname, isAbsolute, join } from 'path'; +import { existsSync } from 'fs'; +import { + buildDiagram, + Diagram, + format, + FormatType, + SequenceDiagramOptions, + Specification, +} from '@appland/sequence-diagram'; +import { AppMap, CodeObject, buildAppMap, Event } from '@appland/models'; +import { readDiagramFile } from '../cmds/sequenceDiagram/readDiagramFile'; +import { executeCommand } from './executeCommand'; +import { RevisionName } from './RevisionName'; +import { OperationReference } from './OperationReference'; +import { promisify } from 'util'; +import { glob } from 'glob'; + +type Metadata = { + sourceLocation: string | undefined; + appmapName: string | undefined; + sourcePaths: string[]; + testStatus?: string; +}; + +export class AppMapReference { + // Set of all source files that were used by the app. + public sourcePaths = new Set(); + // Name of the test case that produced these AppMaps. + public sourceLocation: string | undefined; + public appmapName: string | undefined; + + constructor( + private operationReference: OperationReference, + public outputDir: string, + public appmapFileName: string + ) {} + + static outputPath(outputDir: string, revisionName: RevisionName): string { + return join(outputDir, revisionName); + } + + async sourceDiff(baseRevision: string): Promise { + if (this.sourcePaths.size === 0) return; + + const existingSourcePaths = [...this.sourcePaths].filter(existsSync).sort(); + return ( + await executeCommand( + `git diff ${baseRevision} -- ${existingSourcePaths.join(' ')}`, + false, + false, + false + ) + ).trim(); + } + + sequenceDiagramFileName(format: string): string { + return [basename(this.appmapFileName, '.appmap.json'), `sequence.${format}`].join('.'); + } + + metadataFileName(): string { + return [basename(this.appmapFileName, '.appmap.json'), `metadata.json`].join('.'); + } + + sequenceDiagramFilePath( + revisionName: RevisionName, + format: FormatType | string, + includeOutputDir: boolean + ): string { + const tokens = [revisionName, this.sequenceDiagramFileName(format)]; + if (includeOutputDir) tokens.unshift(this.outputDir); + return join(...tokens); + } + + archivedAppMapFilePath(revisionName: RevisionName, includeOutputDir: boolean): string { + const tokens = [revisionName, basename(this.appmapFileName)]; + if (includeOutputDir) tokens.unshift(this.outputDir); + return join(...tokens); + } + + /** + * Gets the AppMap file names of failed test cases. + */ + static async listFailedTests(outputDir: string, revisionName: RevisionName): Promise { + const metadataFiles = await promisify(glob)(`${outputDir}/${revisionName}/*.metadata.json`); + const result = new Array(); + for (const metadataFile of metadataFiles) { + const metadata = JSON.parse(await readFile(metadataFile, 'utf-8')) as Metadata; + if (metadata.testStatus === 'failed') { + result.push( + join( + dirname(metadataFile), + [basename(metadataFile, '.metadata.json'), 'appmap.json'].join('.') + ) + ); + } + } + return result; + } + + async loadSequenceDiagramJSON(revisionName: RevisionName): Promise { + return readDiagramFile(this.sequenceDiagramFilePath(revisionName, FormatType.JSON, true)); + } + + async loadSequenceDiagramText(revisionName: RevisionName): Promise { + return await readFile( + this.sequenceDiagramFilePath(revisionName, FormatType.Text, true), + 'utf-8' + ); + } + + async processAppMap(revisionName: RevisionName) { + const appmap = await this.loadAppMap(); + + this.collectAppMapOperationData(revisionName, appmap); + const metadata = AppMapReference.collectMetadata(appmap); + const diagram = await this.buildSequenceDiagram(appmap); + + await copyFile(this.appmapFileName, this.archivedAppMapFilePath(revisionName, true)); + await writeFile( + this.archivedMetadataFilePath(revisionName, true), + JSON.stringify(metadata, null, 2) + ); + await writeFile( + this.sequenceDiagramFilePath(revisionName, FormatType.JSON, true), + format(FormatType.JSON, diagram, this.appmapFileName).diagram + ); + await writeFile( + this.sequenceDiagramFilePath(revisionName, FormatType.JSON, true), + format(FormatType.JSON, diagram, this.appmapFileName).diagram + ); + await writeFile( + this.sequenceDiagramFilePath(revisionName, FormatType.Text, true), + format(FormatType.Text, diagram, this.appmapFileName).diagram + ); + } + + async restoreMetadata(revisionName: RevisionName) { + const appmapData = JSON.parse( + await readFile(this.archivedAppMapFilePath(revisionName, true), 'utf-8') + ); + const appmap = buildAppMap().source(appmapData).build(); + + this.collectAppMapOperationData(revisionName, appmap); + const metadata = JSON.parse( + await readFile(this.archivedMetadataFilePath(revisionName, true), 'utf-8') + ); + + this.sourceLocation = metadata.sourceLocation; + this.appmapName = metadata.appmapName; + this.sourcePaths = new Set(metadata.sourcePaths); + } + + private async buildSequenceDiagram(appmap: AppMap): Promise { + const specOptions = { loops: false } as SequenceDiagramOptions; + const specification = Specification.build(appmap, specOptions); + return buildDiagram(this.appmapFileName, appmap, specification); + } + + private async loadAppMap(): Promise { + const appmapData = JSON.parse(await readFile(this.appmapFileName, 'utf-8')); + return buildAppMap().source(appmapData).build(); + } + + private archivedMetadataFilePath(revisionName: RevisionName, includeOutputDir: boolean): string { + const tokens = [revisionName, this.metadataFileName()]; + if (includeOutputDir) tokens.unshift(this.outputDir); + return join(...tokens); + } + + private collectAppMapOperationData(revisionName: RevisionName, appmap: AppMap) { + for (let eventId = 0; eventId < appmap.events.length; eventId++) { + const event = appmap.events[eventId]; + if (event.httpServerRequest && event.httpServerResponse) { + const requestAppMap = AppMapReference.buildServerRPCAppMap(appmap, event); + this.operationReference.addServerRPC(revisionName, requestAppMap); + eventId = event.returnEvent.id; + } + } + } + + private static buildServerRPCAppMap(appmap: AppMap, event: Event) { + const startId = event.id; + const endId = event.returnEvent.id; + const events = appmap.events.filter((e) => { + if (e.id < startId || e.id > endId) return false; + + if (e.codeObject.type !== 'function') return true; + + const { isLocal } = AppMapReference.isLocalPath(e.codeObject.location); + return isLocal; + }); + + return buildAppMap({ + events, + classMap: appmap.classMap.roots.map((c) => ({ ...c.data })), + metadata: appmap.metadata, + }).build(); + } + + static isLocalPath(location?: string): { isLocal: boolean; path?: string } { + if (!location) return { isLocal: false }; + + if (!location.includes(':')) return { isLocal: false }; + + const path = location.split(':')[0]; + if (path.match(/\.\w+$/) && !isAbsolute(path)) return { isLocal: true, path }; + + return { isLocal: false }; + } + + static collectPath(codeObject: CodeObject, fn: (path: string) => void) { + const { isLocal, path } = AppMapReference.isLocalPath(codeObject.location); + + if (isLocal && path) fn(path); + } + + private static collectMetadata(appmap: AppMap): Metadata { + const sourcePaths = new Set(); + + const collectSourcePath = (codeObject: CodeObject) => { + AppMapReference.collectPath(codeObject, (path) => sourcePaths.add(path)); + }; + + appmap.classMap.visit(collectSourcePath); + + return { + sourceLocation: (appmap.metadata as any).source_location, + appmapName: appmap.metadata.name, + testStatus: appmap.metadata.test_status, + sourcePaths: [...sourcePaths].sort(), + }; + } +} diff --git a/packages/cli/src/describeChange/OperationReference.ts b/packages/cli/src/describeChange/OperationReference.ts new file mode 100644 index 0000000000..46194d0a8a --- /dev/null +++ b/packages/cli/src/describeChange/OperationReference.ts @@ -0,0 +1,194 @@ +import { AppMap, CodeObject } from '@appland/models'; +import { + buildDiagram, + Diagram, + Diagram as SequenceDiagram, + format, + FormatType, + SequenceDiagramOptions, + Specification, +} from '@appland/sequence-diagram'; +import assert from 'assert'; +import * as async from 'async'; +import { link, mkdir, unlink, writeFile } from 'fs/promises'; +import { glob } from 'glob'; +import { basename, join } from 'path'; +import { promisify } from 'util'; +import { readDiagramFile } from '../cmds/sequenceDiagram/readDiagramFile'; +import { exists } from '../utils'; +import { AppMapReference } from './AppMapReference'; +import { RevisionName } from './RevisionName'; + +type Route = { + method: string; + path: string; + status: number; +}; + +type QueueJob = { revisionName: RevisionName; appmap: AppMap; diagram: Diagram }; + +export class OperationReference { + // Set of all source files, indexed by route ([ ()]). + public sourcePathsByOperation = new Map>(); + + // The queue has concurrency of 1, so it's used to ensure that only one route + // is processed / updated at a time. It's essential to begin indexing with startIndexing, + // and to wait for finishIndexing() to complete before continuing. + private queue: async.QueueObject | undefined; + + constructor(public outputDir: string) {} + + static operationKey(method: string, path: string, status: number): string { + return `${method.toUpperCase()} ${path} (${status})`; + } + + startIndexing() { + this.queue = async.queue(async (job: QueueJob) => { + try { + const { revisionName, appmap, diagram } = job; + assert(appmap.events[0].httpServerRequest, 'Expecting an HTTP server request (only)'); + const rootEvent = appmap.events[0]; + assert(rootEvent.httpServerRequest); + assert(rootEvent.httpServerResponse); + assert(diagram.rootActions.length === 1, 'Expecting a single root action'); + const { subtreeDigest } = diagram.rootActions[0]; + + const method = rootEvent.httpServerRequest?.request_method.toUpperCase(); + const path = + rootEvent.httpServerRequest.normalized_path_info || rootEvent.httpServerRequest.path_info; + const status = rootEvent.httpServerResponse.status; + + const key = OperationReference.operationKey(method, path, status); + appmap.classMap.visit((codeObject) => { + AppMapReference.collectPath(codeObject, (path) => + this.addSourcePath(key, path, codeObject) + ); + }); + + await this.findOrCreateAppMap(revisionName, appmap, diagram); + await this.saveLinks(revisionName, { method, path, status: status }, subtreeDigest); + } catch (err) { + console.warn(err); + } + }, 1); + } + + // Returns true if the appmap was created, false if it already existed. + async findOrCreateAppMap( + revisionName: RevisionName, + appmap: AppMap, + diagram: Diagram + ): Promise { + assert(diagram.rootActions.length === 1, 'Expecting a single root action'); + const { subtreeDigest } = diagram.rootActions[0]; + + if ( + (await exists(this.diagramPath(revisionName, subtreeDigest))) && + (await exists(this.appmapPath(revisionName, subtreeDigest))) + ) { + return false; + } + + await writeFile(this.appmapPath(revisionName, subtreeDigest), JSON.stringify(appmap, null, 2)); + await writeFile( + this.diagramPath(revisionName, subtreeDigest), + format(FormatType.JSON, diagram, subtreeDigest).diagram + ); + return true; + } + + async listSequenceDiagrams(revisionName: RevisionName, route: Route): Promise { + const files = await promisify(glob)( + `${this.operationPath(revisionName, route)}/*.sequence.json` + ); + return files.map((file) => basename(file, '.sequence.json')); + } + + async loadSequenceDiagram( + revisionName: RevisionName, + subtreeDigest: string + ): Promise { + const diagramPath = this.diagramPath(revisionName, subtreeDigest); + return readDiagramFile(diagramPath); + } + + async finishIndexing() { + if (!this.queue) return; + + this.queue.length() === 0 || (await this.queue.drain()); + + this.queue = undefined; + } + + addServerRPC(revisionName: RevisionName, appmap: AppMap) { + assert(this.queue, 'OperationReference is not in processing mode'); + + const specOptions = { loops: false } as SequenceDiagramOptions; + const specification = Specification.build(appmap, specOptions); + const diagram = buildDiagram('', appmap, specification); + + this.queue.push({ revisionName, appmap, diagram }); + } + + private addSourcePath(key: string, path: string, codeObject: CodeObject): void { + if (!this.sourcePathsByOperation.get(key)) + this.sourcePathsByOperation.set(key, new Set()); + this.sourcePathsByOperation.get(key)!.add(path); + } + + private async saveLinks(revisionName: RevisionName, route: Route, subtreeDigest: string) { + await mkdir(this.operationPath(revisionName, route), { + recursive: true, + }); + + const paths = [ + [ + this.diagramPath(revisionName, subtreeDigest), + join(this.operationPath(revisionName, route), [subtreeDigest, 'sequence.json'].join('.')), + ], + [ + this.appmapPath(revisionName, subtreeDigest), + join(this.operationPath(revisionName, route), [subtreeDigest, 'appmap.json'].join('.')), + ], + ]; + for (const [src, dest] of paths) { + if (await exists(dest)) continue; + + await link(src, dest); + } + } + + private operationPath(revisionName: RevisionName, route: Route): string { + let { path } = route; + // To avoid a conflict with a real route, use a character that's invalid in a URL. + // The mneumonic for '^' is that it's used in regexp to start a line. + if (path === '/') path = '^'; + + return join( + this.outputDir, + revisionName, + 'operations', + route.method.toUpperCase(), + path, + route.status.toString() + ); + } + + private appmapPath(revisionName: RevisionName, subtreeDigest: string): string { + return join( + this.outputDir, + revisionName, + 'operations', + [subtreeDigest, 'appmap.json'].join('.') + ); + } + + private diagramPath(revisionName: RevisionName, subtreeDigest: string): string { + return join( + this.outputDir, + revisionName, + 'operations', + [subtreeDigest, 'sequence.json'].join('.') + ); + } +} diff --git a/packages/cli/src/describeChange/RevisionName.ts b/packages/cli/src/describeChange/RevisionName.ts new file mode 100644 index 0000000000..9d3005a251 --- /dev/null +++ b/packages/cli/src/describeChange/RevisionName.ts @@ -0,0 +1,5 @@ +export enum RevisionName { + Base = 'base', + Head = 'head', + Diff = 'diff', +} diff --git a/packages/cli/src/describeChange/buildChangeReport.ts b/packages/cli/src/describeChange/buildChangeReport.ts new file mode 100644 index 0000000000..312e23452c --- /dev/null +++ b/packages/cli/src/describeChange/buildChangeReport.ts @@ -0,0 +1,231 @@ +import { AppMapReference } from './AppMapReference'; +import { OpenAPIV3 } from 'openapi-types'; +import { Changes, LogEntry, Operation, OperationChange, RouteChanges, TestFailure } from './types'; +import assert from 'assert'; +import { executeCommand } from './executeCommand'; +import { exists, shuffleArray } from '../utils'; +import { OperationReference } from './OperationReference'; +import { RevisionName } from './RevisionName'; +import { Action, Diagram as SequenceDiagram } from '@appland/sequence-diagram'; +import { DiffDiagrams } from '../sequenceDiagramDiff/DiffDiagrams'; +import { readFile } from 'fs/promises'; +import { buildAppMap } from '@appland/models'; + +export default async function buildChangeReport( + diffDiagrams: DiffDiagrams, + baseRevision: string, + basePaths: OpenAPIV3.PathsObject, + headPaths: OpenAPIV3.PathsObject, + operationReference: OperationReference +): Promise { + const buildOperationAdded = async (operation: Operation): Promise => { + const diagramIds = await operationReference.listSequenceDiagrams(RevisionName.Head, { + method: operation.method, + path: operation.path, + status: operation.status, + }); + const sequenceDiagrams = new Array(); + for (const diagramId of diagramIds) + sequenceDiagrams.push( + await operationReference.loadSequenceDiagram(RevisionName.Head, diagramId) + ); + + return { operation, sequenceDiagrams }; + }; + const buildOperationChanged = async ( + operation: Operation + ): Promise => { + console.log( + OperationReference.operationKey(operation.method, operation.path, operation.status) + ); + + const baseDiagrams = new Set( + await operationReference.listSequenceDiagrams(RevisionName.Base, { + method: operation.method, + path: operation.path, + status: operation.status, + }) + ); + const headDiagrams = new Set( + await operationReference.listSequenceDiagrams(RevisionName.Head, { + method: operation.method, + path: operation.path, + status: operation.status, + }) + ); + const baseOnlyDiagrams = new Array(); + const headOnlyDiagrams = new Array(); + baseDiagrams.forEach((diagram) => + headDiagrams.has(diagram) ? undefined : baseOnlyDiagrams.push(diagram) + ); + headDiagrams.forEach((diagram) => + baseDiagrams.has(diagram) ? undefined : headOnlyDiagrams.push(diagram) + ); + + if (baseOnlyDiagrams.length === 0 && headOnlyDiagrams.length === 0) return; + + function countDiffActions(diagram: SequenceDiagram | undefined): number { + if (!diagram) return 0; + + let count = 0; + const countAction = (action: Action) => { + if (action.diffMode) count += 1; + action.children.forEach(countAction); + }; + diagram.rootActions.forEach(countAction); + return count; + } + + const minimalDiff = async ( + headDiagram: SequenceDiagram, + baseDiagrams: string[] + ): Promise => { + return ( + await Promise.all( + baseDiagrams.map(async (baseDiagramId) => { + const baseDiagram = await operationReference.loadSequenceDiagram( + RevisionName.Base, + baseDiagramId + ); + return diffDiagrams.diff(baseDiagram, headDiagram); + }) + ) + ) + .filter(Boolean) + .sort((a, b) => countDiffActions(a) - countDiffActions(b))[0]; + }; + + const sequenceDiagrams = ( + await Promise.all( + headOnlyDiagrams.map(async (headDiagramId) => { + const headDiagram = await operationReference.loadSequenceDiagram( + RevisionName.Head, + headDiagramId + ); + // Choose three random base diagrams. Compute the diff between the head diagram and each of + // the randomly selected base diagrams. Report the diff with the smallest number of changes. + const baseDiagramIds = shuffleArray([...baseOnlyDiagrams]).slice(0, 3); + return await minimalDiff(headDiagram, baseDiagramIds); + }) + ) + ).filter(Boolean) as SequenceDiagram[]; + + let sourcePaths: string[] | undefined; + let sourceDiff: string | undefined; + + const candidateSourcePaths = operationReference.sourcePathsByOperation.get( + OperationReference.operationKey(operation.method, operation.path, operation.status) + ); + if (candidateSourcePaths) { + const existingSourcePaths = new Set(); + await Promise.all( + [...candidateSourcePaths].map( + async (path) => (await exists(path)) && existingSourcePaths.add(path) + ) + ); + sourcePaths = [...candidateSourcePaths].filter((path) => existingSourcePaths.has(path)); + } + + if (sourcePaths && sourcePaths.length > 0) { + sourceDiff = await executeCommand( + `git diff ${baseRevision} -- ${[...sourcePaths].sort().join(' ')}`, + true, + true, + true + ); + } + + return { operation, sourceDiff, sequenceDiagrams }; + }; + const buildOperationRemoved = (operation: Operation): OperationChange => { + return { operation }; + }; + + const routeChanges = { + added: [], + removed: [], + changed: [], + } as RouteChanges; + + for (const pattern of Object.keys(basePaths)) { + const pathInfo = basePaths[pattern]; + assert(pathInfo); + + for (const method of Object.keys(pathInfo)) { + const operation = pathInfo[method]; + assert(operation); + + for (const status of Object.keys(operation.responses)) { + const operation: Operation = { method, path: pattern, status: parseInt(status, 10) }; + + const headResponse = headPaths[pattern]?.[method]?.responses[status]; + if (!headResponse) { + routeChanges.removed.push(buildOperationRemoved(operation)); + break; + } + + const changed = await buildOperationChanged(operation); + if (changed) routeChanges.changed.push(changed); + } + } + } + + for (const pattern of Object.keys(headPaths)) { + const pathInfo = headPaths[pattern]; + assert(pathInfo); + + for (const method of Object.keys(pathInfo)) { + const operation = pathInfo[method]; + assert(operation); + + for (const status of Object.keys(operation.responses)) { + const operation: Operation = { method, path: pattern, status: parseInt(status, 10) }; + + const baseResponse = basePaths[pattern]?.[method]?.responses[status]; + if (!baseResponse) { + routeChanges.added.push(await buildOperationAdded(operation)); + } + } + } + } + + const appmapFileNamesOfFailedTests = await AppMapReference.listFailedTests( + operationReference.outputDir, + RevisionName.Head + ); + + const buildFailedTest = async (appmapFileName: string): Promise => { + const appmapData = await readFile(appmapFileName, 'utf-8'); + const appmap = buildAppMap().source(appmapData).build(); + + const logEntries = appmap.events + .filter((event) => event.isCall() && event.labels.has('log')) + .map((log) => { + const message = (log.parameters || []).map((param) => param.value).join(''); + if (message) { + return { + message, + stack: log + .callStack() + .map((event) => event.codeObject.location || event.codeObject.fqid), + }; + } + }) + .filter(Boolean) as LogEntry[]; + + return { + appmapFile: appmapFileName, + name: appmap.metadata.name, + testLocation: appmap.metadata.source_location, + logEntries, + }; + }; + + const failedTests = await Promise.all(appmapFileNamesOfFailedTests.map(buildFailedTest)); + + return { + routeChanges, + findings: [], + failedTests, + }; +} diff --git a/packages/cli/src/describeChange/executeCommand.ts b/packages/cli/src/describeChange/executeCommand.ts new file mode 100644 index 0000000000..e846649e03 --- /dev/null +++ b/packages/cli/src/describeChange/executeCommand.ts @@ -0,0 +1,29 @@ +import { exec } from 'child_process'; +import { commandStyle } from '../cmds/describeChange'; + +export function executeCommand( + cmd: string, + printCommand = true, + printStdout = true, + printStderr = true +): Promise { + if (printCommand) console.log(commandStyle(cmd)); + const command = exec(cmd); + const result: string[] = []; + if (command.stdout) { + command.stdout.addListener('data', (data) => { + if (printStdout) process.stdout.write(data); + result.push(data); + }); + } + if (printStderr && command.stderr) command.stderr.pipe(process.stdout); + return new Promise((resolve, reject) => { + command.addListener('exit', (code) => { + if (code === 0) { + resolve(result.join('')); + } else { + reject(new Error(`Command failed with code ${code}`)); + } + }); + }); +} diff --git a/packages/cli/src/describeChange/types.ts b/packages/cli/src/describeChange/types.ts new file mode 100644 index 0000000000..d8af015517 --- /dev/null +++ b/packages/cli/src/describeChange/types.ts @@ -0,0 +1,41 @@ +import { Diagram as SequenceDiagram } from '@appland/sequence-diagram'; + +export type Operation = { + method: string; + path: string; + status: number; +}; + +export type OperationChange = { + operation: Operation; + sourceDiff?: string | undefined; + sequenceDiagrams?: SequenceDiagram[]; +}; + +export type RouteChanges = { + added: OperationChange[]; + removed: OperationChange[]; + changed: OperationChange[]; +}; + +export type Finding = { + ruleId: string; +}; + +export type LogEntry = { + stack: string[]; + message: string; +}; + +export type TestFailure = { + appmapFile: string; + name: string; + testLocation?: string; + logEntries: LogEntry[]; +}; + +export type Changes = { + routeChanges: RouteChanges; + findings: Finding[]; + failedTests: TestFailure[]; +}; diff --git a/packages/cli/src/fingerprint/fingerprintDirectoryCommand.js b/packages/cli/src/fingerprint/fingerprintDirectoryCommand.js index 14d6774715..d331b424c2 100644 --- a/packages/cli/src/fingerprint/fingerprintDirectoryCommand.js +++ b/packages/cli/src/fingerprint/fingerprintDirectoryCommand.js @@ -18,7 +18,7 @@ class FingerprintDirectoryCommand { count += 1; return fpQueue.push(file); }); - await fpQueue.process(); + if (count > 0) await fpQueue.process(); return count; } diff --git a/packages/cli/src/fingerprint/fingerprinter.ts b/packages/cli/src/fingerprint/fingerprinter.ts index a55a11d7f3..38e501ab82 100644 --- a/packages/cli/src/fingerprint/fingerprinter.ts +++ b/packages/cli/src/fingerprint/fingerprinter.ts @@ -40,7 +40,7 @@ const renameFile = promisify(gracefulFs.rename); * * Fix handling of parent assignment in normalization. * * sql can contain the analysis (action, tables, columns), and/or the normalized query string. */ -const VERSION = '1.1.4'; +export const VERSION = '1.1.4'; const MAX_APPMAP_SIZE = 50 * 1000 * 1000; diff --git a/packages/cli/src/html/appmap.js b/packages/cli/src/html/appmap.js index 58add8f659..c8fcdfd1a4 100644 --- a/packages/cli/src/html/appmap.js +++ b/packages/cli/src/html/appmap.js @@ -20,7 +20,8 @@ async function initializeApp() { async mounted() { const params = new URL(document.location).searchParams; const appmap = params.get('appmap'); - const res = await fetch(`/resource?${encodeURIComponent(appmap)}`); + const res = await fetch(appmap); + const { ui } = this.$refs; ui.loadData((await res.json()) || {}); diff --git a/packages/cli/src/html/sequenceDiagram.js b/packages/cli/src/html/sequenceDiagram.js index 39585b6d6a..acd99a1582 100644 --- a/packages/cli/src/html/sequenceDiagram.js +++ b/packages/cli/src/html/sequenceDiagram.js @@ -1,5 +1,6 @@ import Vue from 'vue'; import Vuex from 'vuex'; +import { BlobReader, ZipReader } from '@zip.js/zip.js'; import plugin, { VDiagramSequence } from '@appland/components'; import '@appland/diagrams/dist/style.css'; @@ -7,6 +8,20 @@ import '@appland/diagrams/dist/style.css'; Vue.use(Vuex); Vue.use(plugin); +// https://stackoverflow.com/a/18650249/953770 +async function blobToBase64(blob) { + return new Promise((resolve, _) => { + const reader = new FileReader(); + reader.onloadend = () => resolve(reader.result); + reader.readAsDataURL(blob); + }); +} + +// https://stackoverflow.com/a/36183085/953770 +async function base64UrltoBlob(base64Url) { + return fetch(base64Url).then((res) => res.blob()); +} + async function initializeApp() { return new Vue({ el: '#app', @@ -19,11 +34,52 @@ async function initializeApp() { }), async mounted() { const params = new URL(document.location).searchParams; - const diagram = params.get('diagram'); - const res = await fetch(`/resource?${encodeURIComponent(diagram)}`); + const diagramUrl = params.get('diagram'); + let diagramData; + if (diagramUrl.endsWith('.zip')) { + const resourceId = params.get('resourceId'); + if (!resourceId) throw new Error(`resourceId is required with diagram resource ZIP file`); + + const localStorageKey = ['appmap', 'diagram', diagramUrl].join('-'); + let diagramDataEncoded; + if (!['false', 'no'].includes(params.get('cache'))) + diagramDataEncoded = localStorage.getItem(localStorageKey); + if (diagramDataEncoded) { + diagramData = await base64UrltoBlob(diagramDataEncoded); + } else { + diagramData = await (await fetch(diagramUrl)).blob(); + diagramDataEncoded = await blobToBase64(diagramData); + localStorage.setItem(localStorageKey, diagramDataEncoded); + } + + // Creates a BlobReader object used to read `zipFileBlob`. + const zipFileReader = new BlobReader(diagramData); + // Creates a TransformStream object, the content of the first entry in the zip + // will be written in the `writable` property. + const dataStream = new TransformStream(); + // Creates a Promise object resolved to the content of the first entry returned + // as text from `helloWorldStream.readable`. + const dataStreamPromise = new Response(dataStream.readable).text(); + + // Creates a ZipReader object reading the zip content via `zipFileReader`, + // retrieves metadata (name, dates, etc.) of the first entry, retrieves its + // content into `helloWorldStream.writable`, and closes the reader. + const zipReader = new ZipReader(zipFileReader); + const entries = await zipReader.getEntries(); + const requestedEntry = entries.find((entry) => entry.filename === resourceId); + if (!requestedEntry) throw new Error(`Resource ${resourceId} not found in ${diagramUrl}`); + + await requestedEntry.getData(dataStream.writable); + await zipReader.close(); + const diagramDataRaw = await dataStreamPromise; + diagramData = JSON.parse(diagramDataRaw); + } else { + diagramData = (await fetch(diagramUrl)).json(); + } + const { ui } = this.$refs; - ui.loadData((await res.json()) || {}); + ui.loadData((await diagramData) || {}); }, }); } diff --git a/packages/cli/src/lib/fileSizeFilter.ts b/packages/cli/src/lib/fileSizeFilter.ts new file mode 100644 index 0000000000..62f21d344f --- /dev/null +++ b/packages/cli/src/lib/fileSizeFilter.ts @@ -0,0 +1,24 @@ +import { Stats } from 'fs'; +import { stat } from 'fs/promises'; +import { FilterFunction } from '../cmds/openapi'; + +export const DefaultMaxAppMapSizeInMB = 50; + +// Skip files that are larger than a specified max size. +export function fileSizeFilter(maxFileSize: number): FilterFunction { + return async (file: string): Promise<{ enable: boolean; message?: string }> => { + let fileStat: Stats; + try { + fileStat = await stat(file); + } catch { + return { enable: false, message: `File ${file} not found` }; + } + + if (fileStat.size <= maxFileSize) return { enable: true }; + else + return { + enable: false, + message: `Skipping ${file} as its file size of ${fileStat.size} bytes is larger than the maximum configured file size of ${maxFileSize} bytes`, + }; + }; +} diff --git a/packages/cli/src/lib/loadAppMapConfig.ts b/packages/cli/src/lib/loadAppMapConfig.ts index 8981a50684..e3dbeca730 100644 --- a/packages/cli/src/lib/loadAppMapConfig.ts +++ b/packages/cli/src/lib/loadAppMapConfig.ts @@ -1,19 +1,35 @@ -import { exists } from 'fs'; import { readFile } from 'fs/promises'; import { load } from 'js-yaml'; -import { promisify } from 'util'; + +interface PreflightFilterConfig { + groups?: string[]; + names?: string[]; +} + +interface PreflightConfig { + base_branch?: string; + test_folders?: string[]; + test_commands?: Record; + filter?: PreflightFilterConfig; +} export interface AppMapConfig { name: string; + language?: string; appmap_dir?: string; + preflight?: PreflightConfig; } export default async function loadAppMapConfig(): Promise { - const appMapConfigExists = await promisify(exists)('appmap.yml'); - if (appMapConfigExists) { - const appMapConfigData = load((await readFile('appmap.yml')).toString()); - if (appMapConfigData && typeof appMapConfigData === 'object') { - return appMapConfigData as AppMapConfig; - } + let appMapConfigData: string | undefined; + try { + appMapConfigData = await readFile('appmap.yml', 'utf-8'); + } catch (e) { + return; + } + + const appMapConfig = load(appMapConfigData) as any; + if (appMapConfig && typeof appMapConfig === 'object') { + return appMapConfig as AppMapConfig; } } diff --git a/packages/cli/src/lib/serveAndOpen.ts b/packages/cli/src/lib/serveAndOpen.ts index df86885f82..dc83db15a4 100644 --- a/packages/cli/src/lib/serveAndOpen.ts +++ b/packages/cli/src/lib/serveAndOpen.ts @@ -75,7 +75,6 @@ export default async function serveAndOpen( } else { serveStaticFile(baseDir, (pathname || '/').slice(1)); } - } catch (e: any) { console.log(e.stack); res.writeHead(500); @@ -108,7 +107,7 @@ export async function serveAndOpenSequenceDiagram( serveAndOpen( 'sequenceDiagram.html', { - diagram: diagramFile, + diagram: `/resource?${encodeURIComponent(diagramFile)}`, }, verifyInSubdir, async (url) => { @@ -127,7 +126,7 @@ export async function serveAndOpenAppMap( serveAndOpen( 'appmap.html', { - appmap: appMapFile, + appmap: `/resource?${encodeURIComponent(appMapFile)}`, }, verifyInSubdir, async (url) => { diff --git a/packages/cli/src/openapi/OpenAPICommand.ts b/packages/cli/src/openapi/OpenAPICommand.ts new file mode 100644 index 0000000000..db684d36f3 --- /dev/null +++ b/packages/cli/src/openapi/OpenAPICommand.ts @@ -0,0 +1,94 @@ +import { join } from 'path'; +import { existsSync, promises as fsp } from 'fs'; +import { queue } from 'async'; +import { glob } from 'glob'; +import { OpenAPIV3 } from 'openapi-types'; +import { + Model, + parseHTTPServerRequests, + rpcRequestForEvent, + SecuritySchemes, +} from '@appland/openapi'; +import { Event } from '@appland/models'; +import { inspect } from 'util'; +import { FilterFunction } from '../cmds/openapi'; + +export class OpenAPICommand { + private readonly model = new Model(); + private readonly securitySchemes = new SecuritySchemes(); + + public errors: string[] = []; + public filter: FilterFunction = async (file: string) => ({ enable: true }); + + constructor(private readonly appmapDir: string) {} + + async execute(): Promise< + [ + { + paths: OpenAPIV3.PathsObject; + securitySchemes: Record; + }, + number + ] + > { + const q = queue(this.collectAppMap.bind(this), 5); + q.pause(); + + // Make sure the directory exists -- if it doesn't, the glob below just returns nothing. + if (!existsSync(this.appmapDir)) { + throw new Error(`AppMap directory ${this.appmapDir} does not exist`); + } + + const files = glob.sync(join(this.appmapDir, '**', '*.appmap.json')); + for (let index = 0; index < files.length; index++) { + const file = files[index]; + const filterResult = await this.filter(file); + if (!filterResult.enable) { + if (filterResult.message) console.warn(filterResult.message); + continue; + } + + q.push(file); + } + + if (q.length() > 0) { + await new Promise((resolve, reject) => { + q.drain(resolve); + q.error(reject); + q.resume(); + }); + } + + return [ + { + paths: this.model.openapi(), + securitySchemes: this.securitySchemes.openapi(), + }, + files.length, + ]; + } + + async collectAppMap(file: string): Promise { + try { + const data = await fsp.readFile(file, 'utf-8'); + parseHTTPServerRequests(JSON.parse(data), (e: Event) => { + const request = rpcRequestForEvent(e); + if (request) { + this.model.addRpcRequest(request); + this.securitySchemes.addRpcRequest(request); + } + }); + } catch (e) { + // Re-throwing this error crashes the whole process. + // So if there is a malformed AppMap, indicate it here but don't blow everything up. + // Do not write to stdout! + let errorString: string; + try { + errorString = inspect(e); + } catch (x) { + errorString = ((e as any) || '').toString(); + } + this.errors.push(errorString); + } + } +} diff --git a/packages/cli/src/openapi/fileSizeFilter.ts b/packages/cli/src/openapi/fileSizeFilter.ts new file mode 100644 index 0000000000..b2a2e33508 --- /dev/null +++ b/packages/cli/src/openapi/fileSizeFilter.ts @@ -0,0 +1,25 @@ +import { Stats } from 'fs'; +import { stat } from 'fs/promises'; +import { FilterFunction } from '../cmds/openapi'; + +// Skip files that are larger than a specified max size. + +export const DefaultMaxAppMapSizeInMB = 50; + +export function fileSizeFilter(maxFileSize: number): FilterFunction { + return async (file: string): Promise<{ enable: boolean; message?: string }> => { + let fileStat: Stats; + try { + fileStat = await stat(file); + } catch { + return { enable: false, message: `File ${file} not found` }; + } + + if (fileStat.size <= maxFileSize) return { enable: true }; + else + return { + enable: false, + message: `Skipping ${file} as its file size of ${fileStat.size} bytes is larger than the maximum configured file size of ${maxFileSize} bytes`, + }; + }; +} diff --git a/packages/cli/src/sequenceDiagram/renderSequenceDiagramPNG.ts b/packages/cli/src/sequenceDiagram/renderSequenceDiagramPNG.ts new file mode 100644 index 0000000000..276d7bbab8 --- /dev/null +++ b/packages/cli/src/sequenceDiagram/renderSequenceDiagramPNG.ts @@ -0,0 +1,26 @@ +import { Browser } from 'puppeteer'; +import { verbose } from '../utils'; +import { serveAndOpenSequenceDiagram } from '../lib/serveAndOpen'; +import assert from 'assert'; +import BrowserRenderer from '../cmds/sequenceDiagram/browserRenderer'; + +export async function renderSequenceDiagramPNG( + outputPath: string, + diagramPath: string, + browser: BrowserRenderer +): Promise { + return new Promise((resolve, reject) => + serveAndOpenSequenceDiagram(diagramPath, false, async (url) => { + try { + if (verbose()) console.warn(`Rendering PNG`); + assert(browser, 'Browser not initialized'); + + await browser.screenshot(url, outputPath); + + resolve(); + } catch (e) { + reject(e); + } + }) + ); +} diff --git a/packages/cli/src/touchFile.ts b/packages/cli/src/touchFile.ts new file mode 100644 index 0000000000..6c7c2c77b1 --- /dev/null +++ b/packages/cli/src/touchFile.ts @@ -0,0 +1,16 @@ +import { close, open, utimes } from 'fs'; + +export async function touch(path: string): Promise { + return new Promise((resolve, reject) => { + const time = new Date(); + utimes(path, time, time, (err) => { + if (err) { + return open(path, 'w', (err, fd) => { + if (err) return reject(err); + close(fd, (err) => (err ? reject(err) : resolve())); + }); + } + resolve(); + }); + }); +} diff --git a/packages/cli/src/utils.ts b/packages/cli/src/utils.ts index 93145ad447..ffef829fb4 100644 --- a/packages/cli/src/utils.ts +++ b/packages/cli/src/utils.ts @@ -136,6 +136,16 @@ export function exists(path: PathLike): Promise { }); } +// Shuffles an array in place, returning it for convenience. +// https://stackoverflow.com/a/12646864/953770 +export function shuffleArray(array: Array): Array { + for (let i = array.length - 1; i > 0; i--) { + const j = Math.floor(Math.random() * (i + 1)); + [array[i], array[j]] = [array[j], array[i]]; + } + return array; +} + /** * Append a prefix to each line in a string * @param str the string to be prefixed diff --git a/packages/cli/tests/unit/openapi.spec.ts b/packages/cli/tests/unit/openapi.spec.ts index 0c54ac1fcc..df8499ecca 100644 --- a/packages/cli/tests/unit/openapi.spec.ts +++ b/packages/cli/tests/unit/openapi.spec.ts @@ -1,7 +1,8 @@ import { verbose } from '../../src/utils'; -import { default as openapi, fileSizeFilter } from '../../src/cmds/openapi'; +import { default as openapi } from '../../src/cmds/openapi'; import assert from 'assert'; import path from 'path'; +import { fileSizeFilter } from '../../src/lib/fileSizeFilter'; describe('OpenAPI', () => { const malformedHTTPServerRequestDir = 'tests/unit/fixtures/malformedHTTPServerRequest'; diff --git a/yarn.lock b/yarn.lock index edc8c5ef1e..d2f49f28e8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -108,6 +108,7 @@ __metadata: "@types/tmp": ^0.2.3 "@types/validator": ^13.7.10 "@types/w3c-xmlserializer": ^2.0.2 + "@zip.js/zip.js": ^2.6.75 JSONStream: ^1.3.5 ajv: ^8.6.3 applicationinsights: ^2.1.4 @@ -8432,6 +8433,13 @@ __metadata: languageName: node linkType: hard +"@zip.js/zip.js@npm:^2.6.75": + version: 2.6.75 + resolution: "@zip.js/zip.js@npm:2.6.75" + checksum: bf152ca599350be3cc87bcfa06467aa4209226c7772243ba57f41ef0eb71e2db2848e9df1c37d7b1c4a717158db33e76b0485ccae1a9cd3b20d03f5a772627cb + languageName: node + linkType: hard + "JSONStream@npm:^1.0.4, JSONStream@npm:^1.3.5": version: 1.3.5 resolution: "JSONStream@npm:1.3.5" @@ -11086,17 +11094,6 @@ __metadata: languageName: node linkType: hard -"chromium-bidi@npm:0.4.4": - version: 0.4.4 - resolution: "chromium-bidi@npm:0.4.4" - dependencies: - mitt: 3.0.0 - peerDependencies: - devtools-protocol: "*" - checksum: 153a276fcce8eb934c9e6a6f8bde5bc37c454d76e82994ce3eca16ab32415a44c553cba3e2ab7ab48ef65f7423f4be35e0ff5b57c7eb378f65fd9b2042df7d03 - languageName: node - linkType: hard - "chromium-bidi@npm:0.4.5": version: 0.4.5 resolution: "chromium-bidi@npm:0.4.5" @@ -12083,18 +12080,6 @@ __metadata: languageName: node linkType: hard -"cosmiconfig@npm:8.0.0": - version: 8.0.0 - resolution: "cosmiconfig@npm:8.0.0" - dependencies: - import-fresh: ^3.2.1 - js-yaml: ^4.1.0 - parse-json: ^5.0.0 - path-type: ^4.0.0 - checksum: ff4cdf89ac1ae52e7520816622c21a9e04380d04b82d653f5139ec581aa4f7f29e096d46770bc76c4a63c225367e88a1dfa233ea791669a35101f5f9b972c7d1 - languageName: node - linkType: hard - "cosmiconfig@npm:8.1.0": version: 8.1.0 resolution: "cosmiconfig@npm:8.1.0" @@ -26519,30 +26504,6 @@ __metadata: languageName: node linkType: hard -"puppeteer-core@npm:19.7.2": - version: 19.7.2 - resolution: "puppeteer-core@npm:19.7.2" - dependencies: - chromium-bidi: 0.4.4 - cross-fetch: 3.1.5 - debug: 4.3.4 - devtools-protocol: 0.0.1094867 - extract-zip: 2.0.1 - https-proxy-agent: 5.0.1 - proxy-from-env: 1.1.0 - rimraf: 3.0.2 - tar-fs: 2.1.1 - unbzip2-stream: 1.4.3 - ws: 8.11.0 - peerDependencies: - typescript: ">= 4.7.4" - peerDependenciesMeta: - typescript: - optional: true - checksum: dafb2f63b74e3270426011ddcd1f4a06e74ca44322137d4240bfaf6090d8450fbf3607336109286861aa96e1f4f2ab9c13cef66c2fc4ec1a690d92d48e8fe773 - languageName: node - linkType: hard - "puppeteer-core@npm:19.7.5": version: 19.7.5 resolution: "puppeteer-core@npm:19.7.5" @@ -26567,20 +26528,7 @@ __metadata: languageName: node linkType: hard -"puppeteer@npm:^19.7.2": - version: 19.7.2 - resolution: "puppeteer@npm:19.7.2" - dependencies: - cosmiconfig: 8.0.0 - https-proxy-agent: 5.0.1 - progress: 2.0.3 - proxy-from-env: 1.1.0 - puppeteer-core: 19.7.2 - checksum: 10a77ee40173e56ddce605c07296b7e538a7e18b0a517e55d570fb2944c7d983d1b19581e31e0f3626e21544c63d20bbac1583307a7a3afbfc80f83fa695f1ae - languageName: node - linkType: hard - -"puppeteer@npm:^19.7.5": +"puppeteer@npm:^19.7.2, puppeteer@npm:^19.7.5": version: 19.7.5 resolution: "puppeteer@npm:19.7.5" dependencies: @@ -27660,7 +27608,7 @@ resolve@1.1.7: languageName: node linkType: hard -"rimraf@npm:*, rimraf@npm:3.0.2, rimraf@npm:^3.0.0, rimraf@npm:^3.0.2": +"rimraf@npm:*, rimraf@npm:^3.0.0, rimraf@npm:^3.0.2": version: 3.0.2 resolution: "rimraf@npm:3.0.2" dependencies: @@ -32704,21 +32652,6 @@ typescript@~4.4.3: languageName: node linkType: hard -"ws@npm:8.11.0, ws@npm:^8.0.0": - version: 8.11.0 - resolution: "ws@npm:8.11.0" - peerDependencies: - bufferutil: ^4.0.1 - utf-8-validate: ^5.0.2 - peerDependenciesMeta: - bufferutil: - optional: true - utf-8-validate: - optional: true - checksum: 316b33aba32f317cd217df66dbfc5b281a2f09ff36815de222bc859e3424d83766d9eb2bd4d667de658b6ab7be151f258318fb1da812416b30be13103e5b5c67 - languageName: node - linkType: hard - "ws@npm:8.12.1": version: 8.12.1 resolution: "ws@npm:8.12.1" @@ -32767,6 +32700,21 @@ typescript@~4.4.3: languageName: node linkType: hard +"ws@npm:^8.0.0": + version: 8.13.0 + resolution: "ws@npm:8.13.0" + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: ">=5.0.2" + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + checksum: 53e991bbf928faf5dc6efac9b8eb9ab6497c69feeb94f963d648b7a3530a720b19ec2e0ec037344257e05a4f35bd9ad04d9de6f289615ffb133282031b18c61c + languageName: node + linkType: hard + "ws@npm:^8.2.3": version: 8.5.0 resolution: "ws@npm:8.5.0"