From 4ea1a9418bca5d9aa3483368050f03f6363816ed Mon Sep 17 00:00:00 2001 From: Kevin Gilpin Date: Wed, 15 Mar 2023 17:05:28 -0400 Subject: [PATCH 01/24] feat: Build AppMap archive --- packages/cli/src/cli.ts | 2 + packages/cli/src/cmds/archive/archive.ts | 292 ++++++++++++++++++ packages/cli/src/cmds/archive/restore.ts | 0 packages/cli/src/cmds/openapi.ts | 2 + packages/cli/src/fingerprint/fingerprinter.ts | 2 +- packages/cli/src/lib/loadAppMapConfig.ts | 31 +- 6 files changed, 320 insertions(+), 9 deletions(-) create mode 100644 packages/cli/src/cmds/archive/archive.ts create mode 100644 packages/cli/src/cmds/archive/restore.ts diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index b2d4dc4edc..396433e538 100755 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -33,6 +33,7 @@ const SequenceDiagramCommand = require('./cmds/sequenceDiagram'); const SequenceDiagramDiffCommand = require('./cmds/sequenceDiagramDiff'); const StatsCommand = require('./cmds/stats/stats'); import UploadCommand from './cmds/upload'; +const BuildArchive = require('./cmds/archive/archive'); class DiffCommand { public appMapNames: any; @@ -455,6 +456,7 @@ yargs(process.argv.slice(2)) .command(SequenceDiagramDiffCommand) .command(PruneCommand) .command(UploadCommand) + .command(BuildArchive) .strict() .demandCommand() .help().argv; diff --git a/packages/cli/src/cmds/archive/archive.ts b/packages/cli/src/cmds/archive/archive.ts new file mode 100644 index 0000000000..36724eba4e --- /dev/null +++ b/packages/cli/src/cmds/archive/archive.ts @@ -0,0 +1,292 @@ +import { exec } from 'child_process'; +import { basename, dirname, join, resolve } from 'path'; +import yargs from 'yargs'; +import { handleWorkingDirectory } from '../../lib/handleWorkingDirectory'; +import { locateAppMapDir } from '../../lib/locateAppMapDir'; +import { exists, verbose } from '../../utils'; +import PackageConfig from '../../../package.json'; +import { readFile, stat, unlink, writeFile } from 'fs/promises'; +import { glob } from 'glob'; +import { promisify } from 'util'; +import FingerprintDirectoryCommand from '../../fingerprint/fingerprintDirectoryCommand'; +import { + buildDiagram, + format, + FormatType, + SequenceDiagramOptions, + Specification, +} from '@appland/sequence-diagram'; +import { queue } from 'async'; +import { AppMapFilter, buildAppMap, Filter } from '@appland/models'; +import { DefaultMaxAppMapSizeInMB } from '../../openapi/fileSizeFilter'; +import loadAppMapConfig, { AppMapConfig } from '../../lib/loadAppMapConfig'; +import { VERSION as IndexVersion } from '../../fingerprint/fingerprinter'; +import chalk from 'chalk'; + +const ArchiveVersion = '1.0'; +const { name: ApplandAppMapPackageName, version: ApplandAppMapPackageVersion } = PackageConfig; + +/** + * Default filters for each language - plus a set of default filters that apply to all languages. + */ +export const DefaultFilters = { + default: ['label:unstable', 'label:serialize', 'label:deserialize.safe', 'label:log'], + ruby: ['label:mvc.template.resolver', 'package:ruby', 'package:activesupport'], + python: [], + java: [], + javascript: [], +}; + +export type Metadata = { + versions: Record; + workingDirectory: string; + appMapDir: string; + commandArguments: Record; + baseRevision?: string; + revision: string; + timestamp: string; + oversizedAppMaps: string[]; + config: AppMapConfig; +}; + +export const command = 'archive'; +export const describe = 'Build an AppMap archive from a directory containing AppMaps'; + +export const builder = (args: yargs.Argv) => { + args.option('directory', { + describe: 'program working directory', + type: 'string', + alias: 'd', + }); + + args.option('type', { + describe: 'archive type', + choices: ['full', 'incremental', 'auto'], + default: 'auto', + alias: 't', + }); + + args.option('revision', { + describe: `revision identifier. + +If not explicitly specified, the current git revision will be used. +When this command is used in an CI server, it's best to explicitly the provide the revision +from an environment variable provided by the CI system, such as GITHUB_HEAD_SHA, because the +commit of the current git revision may not be the one that triggered the build.`, + type: 'string', + alias: 'r', + }); + + args.option('output-dir', { + describe: 'directory in which to save the output file', + type: 'string', + }); + + args.option('output-file', { + describe: 'output file name', + type: 'string', + default: 'appmap_archive.tar', + alias: 'f', + }); + + args.option('concurrency', { + describe: 'number of AppMap which will be processed in parallel', + type: 'number', + default: 5, + }); + + args.option('max-size', { + describe: 'maximum AppMap size that will be processed, in filesystem-reported MB', + default: DefaultMaxAppMapSizeInMB, + }); + + return args.strict(); +}; + +async function gitRevision(): Promise { + return new Promise((resolve) => { + exec('git rev-parse HEAD', (error, stdout) => { + if (error) resolve(undefined); + + resolve(stdout.trim()); + }); + }); +} + +export const handler = async (argv: any) => { + verbose(argv.verbose); + + handleWorkingDirectory(argv.directory); + const workingDirectory = process.cwd(); + + const appmapConfig = await loadAppMapConfig(); + if (!appmapConfig) throw new Error(`Unable to load appmap.yml config file`); + + const appMapDir = await locateAppMapDir(); + + const { + concurrency, + maxSize, + type, + revision: defaultRevision, + outputFile: outputFileName, + } = argv; + const { outputDir } = argv || '.'; + + const maxAppMapSizeInBytes = Math.round(parseFloat(maxSize) * 1024 * 1024); + + console.log(`Building '${type}' archive from ${appMapDir}`); + + const revision = defaultRevision || (await gitRevision()); + if (!revision) + throw new Error( + `Unable to determine revision. Use --revision to specify it, or run this command in a Git repo.` + ); + + console.log(`Building archive of revision ${revision}`); + const versions = { archive: ArchiveVersion, index: IndexVersion }; + versions[ApplandAppMapPackageName] = ApplandAppMapPackageVersion; + + process.chdir(appMapDir); + + process.stdout.write(`Indexing AppMaps...`); + const numIndexed = await new FingerprintDirectoryCommand('.').execute(); + process.stdout.write(`done (${numIndexed})\n`); + + console.log('Generating sequence diagrams'); + + const specOptions = { + loops: true, + } as SequenceDiagramOptions; + + const filter = new AppMapFilter(); + filter.declutter.hideMediaRequests.on = true; + filter.declutter.limitRootEvents.on = true; + + const preflightConfig = appmapConfig.preflight; + + const defaultFilterGroups = (): string[] => { + const result = ['default']; + if (appmapConfig.language) result.push(appmapConfig.language); + return result; + }; + + const configuredFilterGroups = (): string[] | undefined => { + if (!preflightConfig?.filter) return; + + return preflightConfig?.filter?.groups; + }; + + const filterGroups = configuredFilterGroups() || defaultFilterGroups(); + const filterGroupNames = filterGroups.map((group) => DefaultFilters[group]).flat(); + const filterNames = preflightConfig?.filter?.names || []; + + if (filterGroupNames.length > 0 || filterNames.length > 0) { + filter.declutter.hideName.on = true; + filter.declutter.hideName.names = [...new Set([...filterGroupNames, ...filterNames])].sort(); + } + + const oversizedAppMaps = new Array(); + const sequenceDiagramQueue = queue(async (appmapFileName: string) => { + // Determine size of file appmapFileName in bytes + const stats = await stat(appmapFileName); + if (stats.size > maxAppMapSizeInBytes) { + console.log( + `Skipping, and removing, ${appmapFileName} because its size of ${stats.size} exceeds the maximum size of ${maxSize} MB` + ); + oversizedAppMaps.push(appmapFileName); + await unlink(appmapFileName); + return; + } + + const fullAppMap = buildAppMap() + .source(await readFile(appmapFileName, 'utf8')) + .build(); + const filteredAppMap = filter.filter(fullAppMap, []); + const specification = Specification.build(filteredAppMap, specOptions); + const diagram = buildDiagram(appmapFileName, filteredAppMap, specification); + const diagramOutput = format(FormatType.JSON, diagram, appmapFileName); + const indexDir = join(dirname(appmapFileName), basename(appmapFileName, '.appmap.json')); + const diagramFileName = join(indexDir, 'sequence.json'); + await writeFile(diagramFileName, diagramOutput.diagram); + }, concurrency); + + (await promisify(glob)('**/*.appmap.json')).forEach((appmapFileName) => + sequenceDiagramQueue.push(appmapFileName) + ); + await sequenceDiagramQueue.drain(); + + const metadata: Metadata = { + versions, + workingDirectory, + appMapDir, + commandArguments: argv, + revision, + timestamp: Date.now().toString(), + oversizedAppMaps, + config: appmapConfig, + }; + + if (await exists('appmap_archive.json')) { + const existingMetadata = JSON.parse(await readFile('appmap_archive.json', 'utf8')); + const { revision: baseRevision } = existingMetadata; + if (type === 'auto') { + console.log( + `The AppMap directory contains appmap_archive.json, so the archive type will be 'incremental'. +The base revision is ${baseRevision}.` + ); + } + if (type === 'full') { + console.warn( + chalk.yellow( + `\nThe AppMap directory contains appmap_archive.json, so it looks like the directory contains incremental AppMaps +that build on revision ${baseRevision}. However, you've specified --type=full. You should +remove appmap_archive.json if your intention is to build a full archive. Otherwise, use --type=auto or --type=incremental.\n` + ) + ); + } else { + metadata.baseRevision = baseRevision; + } + } else { + if (type === 'auto') { + console.log( + `The AppMap directory does not contain appmap_archive.json, so the archive type will be 'full'.` + ); + } else if (type === 'incremental') { + throw new Error( + `AppMap directory does not contain appmap_archive.json, but you've specified --type=incremental. +The base revision cannot be determined, so either use --type=auto or --type=full.` + ); + } + } + + await new Promise((resolveCB, rejectCB) => { + exec( + `tar czf appmaps.tar.gz --exclude appmaps.tar.gz --exclude appmap_archive.json *`, + (error) => { + if (error) return rejectCB(error); + + resolveCB(); + } + ); + }); + + process.on('exit', () => unlink('appmaps.tar.gz')); + + await writeFile('appmap_archive.json', JSON.stringify(metadata, null, 2)); + + await new Promise((resolveCB, rejectCB) => { + exec( + `tar cf ${join( + resolve(workingDirectory, outputDir), + outputFileName + )} appmap_archive.json appmaps.tar.gz`, + (error) => { + if (error) return rejectCB(error); + + resolveCB(); + } + ); + }); + console.log(`Created AppMap archive ${join(outputDir, outputFileName)}`); +}; diff --git a/packages/cli/src/cmds/archive/restore.ts b/packages/cli/src/cmds/archive/restore.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/cli/src/cmds/openapi.ts b/packages/cli/src/cmds/openapi.ts index ee2d2150e0..47b741ab72 100644 --- a/packages/cli/src/cmds/openapi.ts +++ b/packages/cli/src/cmds/openapi.ts @@ -22,6 +22,8 @@ import { handleWorkingDirectory } from '../lib/handleWorkingDirectory'; import { locateAppMapConfigFile } from '../lib/locateAppMapConfigFile'; import Telemetry, { Git, GitState } from '../telemetry'; import { findRepository } from '../lib/git'; +import { DefaultMaxAppMapSizeInMB, fileSizeFilter } from '../openapi/fileSizeFilter'; +import { OpenAPICommand } from '../openapi/OpenAPICommand'; type FilterFunction = (file: string) => Promise<{ enable: boolean; message?: string }>; diff --git a/packages/cli/src/fingerprint/fingerprinter.ts b/packages/cli/src/fingerprint/fingerprinter.ts index a55a11d7f3..38e501ab82 100644 --- a/packages/cli/src/fingerprint/fingerprinter.ts +++ b/packages/cli/src/fingerprint/fingerprinter.ts @@ -40,7 +40,7 @@ const renameFile = promisify(gracefulFs.rename); * * Fix handling of parent assignment in normalization. * * sql can contain the analysis (action, tables, columns), and/or the normalized query string. */ -const VERSION = '1.1.4'; +export const VERSION = '1.1.4'; const MAX_APPMAP_SIZE = 50 * 1000 * 1000; diff --git a/packages/cli/src/lib/loadAppMapConfig.ts b/packages/cli/src/lib/loadAppMapConfig.ts index 8981a50684..c98ec46dd3 100644 --- a/packages/cli/src/lib/loadAppMapConfig.ts +++ b/packages/cli/src/lib/loadAppMapConfig.ts @@ -1,19 +1,34 @@ -import { exists } from 'fs'; import { readFile } from 'fs/promises'; import { load } from 'js-yaml'; -import { promisify } from 'util'; + +interface PreflightFilterConfig { + groups?: string[]; + names?: string[]; +} + +interface PreflightConfig { + base_branch?: string; + test_command?: string; + filter?: PreflightFilterConfig; +} export interface AppMapConfig { name: string; + language?: string; appmap_dir?: string; + preflight?: PreflightConfig; } export default async function loadAppMapConfig(): Promise { - const appMapConfigExists = await promisify(exists)('appmap.yml'); - if (appMapConfigExists) { - const appMapConfigData = load((await readFile('appmap.yml')).toString()); - if (appMapConfigData && typeof appMapConfigData === 'object') { - return appMapConfigData as AppMapConfig; - } + let appMapConfigData: string | undefined; + try { + appMapConfigData = await readFile('appmap.yml', 'utf-8'); + } catch (e) { + return; + } + + const appMapConfig = load(appMapConfigData) as any; + if (appMapConfig && typeof appMapConfig === 'object') { + return appMapConfig as AppMapConfig; } } From 0f36e2ec216557cd9ba091f47bf1e61aa13b38c9 Mon Sep 17 00:00:00 2001 From: Kevin Gilpin Date: Thu, 16 Mar 2023 12:11:18 -0400 Subject: [PATCH 02/24] refactor: Move fileSizeFilter to its own file --- packages/cli/src/cmds/openapi.ts | 30 +++++--------------------- packages/cli/src/lib/fileSizeFilter.ts | 24 +++++++++++++++++++++ 2 files changed, 29 insertions(+), 25 deletions(-) create mode 100644 packages/cli/src/lib/fileSizeFilter.ts diff --git a/packages/cli/src/cmds/openapi.ts b/packages/cli/src/cmds/openapi.ts index 47b741ab72..14d3549190 100644 --- a/packages/cli/src/cmds/openapi.ts +++ b/packages/cli/src/cmds/openapi.ts @@ -1,7 +1,7 @@ import { join } from 'path'; -import { existsSync, promises as fsp, Stats } from 'fs'; -import { readFile, stat } from 'fs/promises'; +import { existsSync, promises as fsp } from 'fs'; +import { readFile } from 'fs/promises'; import { queue } from 'async'; import { glob } from 'glob'; import yaml, { load } from 'js-yaml'; @@ -22,29 +22,9 @@ import { handleWorkingDirectory } from '../lib/handleWorkingDirectory'; import { locateAppMapConfigFile } from '../lib/locateAppMapConfigFile'; import Telemetry, { Git, GitState } from '../telemetry'; import { findRepository } from '../lib/git'; -import { DefaultMaxAppMapSizeInMB, fileSizeFilter } from '../openapi/fileSizeFilter'; -import { OpenAPICommand } from '../openapi/OpenAPICommand'; +import { DefaultMaxAppMapSizeInMB, fileSizeFilter } from '../lib/fileSizeFilter'; -type FilterFunction = (file: string) => Promise<{ enable: boolean; message?: string }>; - -// Skip files that are larger than a specified max size. -export function fileSizeFilter(maxFileSize: number): FilterFunction { - return async (file: string): Promise<{ enable: boolean; message?: string }> => { - let fileStat: Stats; - try { - fileStat = await stat(file); - } catch { - return { enable: false, message: `File ${file} not found` }; - } - - if (fileStat.size <= maxFileSize) return { enable: true }; - else - return { - enable: false, - message: `Skipping ${file} as its file size of ${fileStat.size} bytes is larger than the maximum configured file size of ${maxFileSize} bytes`, - }; - }; -} +export type FilterFunction = (file: string) => Promise<{ enable: boolean; message?: string }>; class OpenAPICommand { private readonly model = new Model(); @@ -150,7 +130,7 @@ export default { }); args.option('max-size', { describe: 'maximum AppMap size that will be processed, in filesystem-reported MB', - default: '50', + default: `${DefaultMaxAppMapSizeInMB}mb`, }); args.option('output-file', { alias: ['o'], diff --git a/packages/cli/src/lib/fileSizeFilter.ts b/packages/cli/src/lib/fileSizeFilter.ts new file mode 100644 index 0000000000..62f21d344f --- /dev/null +++ b/packages/cli/src/lib/fileSizeFilter.ts @@ -0,0 +1,24 @@ +import { Stats } from 'fs'; +import { stat } from 'fs/promises'; +import { FilterFunction } from '../cmds/openapi'; + +export const DefaultMaxAppMapSizeInMB = 50; + +// Skip files that are larger than a specified max size. +export function fileSizeFilter(maxFileSize: number): FilterFunction { + return async (file: string): Promise<{ enable: boolean; message?: string }> => { + let fileStat: Stats; + try { + fileStat = await stat(file); + } catch { + return { enable: false, message: `File ${file} not found` }; + } + + if (fileStat.size <= maxFileSize) return { enable: true }; + else + return { + enable: false, + message: `Skipping ${file} as its file size of ${fileStat.size} bytes is larger than the maximum configured file size of ${maxFileSize} bytes`, + }; + }; +} From 494a48aa25a64b1f56436331f949d63578136407 Mon Sep 17 00:00:00 2001 From: Kevin Gilpin Date: Thu, 16 Mar 2023 12:11:45 -0400 Subject: [PATCH 03/24] feat: Restore AppMaps from available archives --- packages/cli/src/cli.ts | 2 + packages/cli/src/cmds/archive/Metadata.ts | 14 +++ packages/cli/src/cmds/archive/archive.ts | 34 +------ packages/cli/src/cmds/archive/gitAncestors.ts | 11 +++ packages/cli/src/cmds/archive/gitRevision.ts | 20 ++++ packages/cli/src/cmds/archive/restore.ts | 96 +++++++++++++++++++ .../cli/src/cmds/archive/unpackArchive.ts | 40 ++++++++ 7 files changed, 188 insertions(+), 29 deletions(-) create mode 100644 packages/cli/src/cmds/archive/Metadata.ts create mode 100644 packages/cli/src/cmds/archive/gitAncestors.ts create mode 100644 packages/cli/src/cmds/archive/gitRevision.ts create mode 100644 packages/cli/src/cmds/archive/unpackArchive.ts diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 396433e538..741c0744ad 100755 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -34,6 +34,7 @@ const SequenceDiagramDiffCommand = require('./cmds/sequenceDiagramDiff'); const StatsCommand = require('./cmds/stats/stats'); import UploadCommand from './cmds/upload'; const BuildArchive = require('./cmds/archive/archive'); +const RestoreArchive = require('./cmds/archive/restore'); class DiffCommand { public appMapNames: any; @@ -457,6 +458,7 @@ yargs(process.argv.slice(2)) .command(PruneCommand) .command(UploadCommand) .command(BuildArchive) + .command(RestoreArchive) .strict() .demandCommand() .help().argv; diff --git a/packages/cli/src/cmds/archive/Metadata.ts b/packages/cli/src/cmds/archive/Metadata.ts new file mode 100644 index 0000000000..56894ef095 --- /dev/null +++ b/packages/cli/src/cmds/archive/Metadata.ts @@ -0,0 +1,14 @@ +import { AppMapConfig } from '../../lib/loadAppMapConfig'; + +export type Metadata = { + versions: Record; + workingDirectory: string; + appMapDir: string; + commandArguments: Record; + baseRevision?: string; + revision: string; + timestamp: string; + oversizedAppMaps: string[]; + deletedAppMaps?: string[]; + config: AppMapConfig; +}; diff --git a/packages/cli/src/cmds/archive/archive.ts b/packages/cli/src/cmds/archive/archive.ts index 36724eba4e..12cac0dd01 100644 --- a/packages/cli/src/cmds/archive/archive.ts +++ b/packages/cli/src/cmds/archive/archive.ts @@ -18,10 +18,12 @@ import { } from '@appland/sequence-diagram'; import { queue } from 'async'; import { AppMapFilter, buildAppMap, Filter } from '@appland/models'; -import { DefaultMaxAppMapSizeInMB } from '../../openapi/fileSizeFilter'; -import loadAppMapConfig, { AppMapConfig } from '../../lib/loadAppMapConfig'; +import { DefaultMaxAppMapSizeInMB } from '../../lib/fileSizeFilter'; +import loadAppMapConfig from '../../lib/loadAppMapConfig'; import { VERSION as IndexVersion } from '../../fingerprint/fingerprinter'; import chalk from 'chalk'; +import gitRevision from './gitRevision'; +import { Metadata } from './Metadata'; const ArchiveVersion = '1.0'; const { name: ApplandAppMapPackageName, version: ApplandAppMapPackageVersion } = PackageConfig; @@ -37,18 +39,6 @@ export const DefaultFilters = { javascript: [], }; -export type Metadata = { - versions: Record; - workingDirectory: string; - appMapDir: string; - commandArguments: Record; - baseRevision?: string; - revision: string; - timestamp: string; - oversizedAppMaps: string[]; - config: AppMapConfig; -}; - export const command = 'archive'; export const describe = 'Build an AppMap archive from a directory containing AppMaps'; @@ -103,16 +93,6 @@ commit of the current git revision may not be the one that triggered the build.` return args.strict(); }; -async function gitRevision(): Promise { - return new Promise((resolve) => { - exec('git rev-parse HEAD', (error, stdout) => { - if (error) resolve(undefined); - - resolve(stdout.trim()); - }); - }); -} - export const handler = async (argv: any) => { verbose(argv.verbose); @@ -137,11 +117,7 @@ export const handler = async (argv: any) => { console.log(`Building '${type}' archive from ${appMapDir}`); - const revision = defaultRevision || (await gitRevision()); - if (!revision) - throw new Error( - `Unable to determine revision. Use --revision to specify it, or run this command in a Git repo.` - ); + const revision = await gitRevision(defaultRevision); console.log(`Building archive of revision ${revision}`); const versions = { archive: ArchiveVersion, index: IndexVersion }; diff --git a/packages/cli/src/cmds/archive/gitAncestors.ts b/packages/cli/src/cmds/archive/gitAncestors.ts new file mode 100644 index 0000000000..1e2b7b1fb0 --- /dev/null +++ b/packages/cli/src/cmds/archive/gitAncestors.ts @@ -0,0 +1,11 @@ +import { exec } from 'child_process'; + +export default async function gitAncestors(defaultRevision?: string): Promise { + return new Promise((resolve, reject) => { + exec('git rev-list HEAD', (error, stdout) => { + if (error) reject(error); + + resolve(stdout.trim().split('\n')); + }); + }); +} diff --git a/packages/cli/src/cmds/archive/gitRevision.ts b/packages/cli/src/cmds/archive/gitRevision.ts new file mode 100644 index 0000000000..648b32a3b1 --- /dev/null +++ b/packages/cli/src/cmds/archive/gitRevision.ts @@ -0,0 +1,20 @@ +import { exec } from 'child_process'; + +export default async function gitRevision(defaultRevision?: string): Promise { + if (defaultRevision) return Promise.resolve(defaultRevision); + + const revision = await new Promise((resolve) => { + exec('git rev-parse HEAD', (error, stdout) => { + if (error) resolve(undefined); + + resolve(stdout.trim()); + }); + }); + + if (!revision) + throw new Error( + `Unable to determine revision. Use --revision to specify it, or run this command in a Git repo.` + ); + + return revision; +} diff --git a/packages/cli/src/cmds/archive/restore.ts b/packages/cli/src/cmds/archive/restore.ts index e69de29bb2..44984695bc 100644 --- a/packages/cli/src/cmds/archive/restore.ts +++ b/packages/cli/src/cmds/archive/restore.ts @@ -0,0 +1,96 @@ +import assert from 'assert'; +import { rmdir, unlink } from 'fs/promises'; +import { glob } from 'glob'; +import { basename, join } from 'path'; +import { promisify } from 'util'; +import yargs from 'yargs'; +import FingerprintDirectoryCommand from '../../fingerprint/fingerprintDirectoryCommand'; +import { handleWorkingDirectory } from '../../lib/handleWorkingDirectory'; +import { verbose } from '../../utils'; +import gitAncestors from './gitAncestors'; +import gitRevision from './gitRevision'; +import unpackArchive from './unpackArchive'; + +export const command = 'restore'; +export const describe = 'Restore the most current available AppMap data from available archives'; + +export const builder = (args: yargs.Argv) => { + args.option('directory', { + describe: 'program working directory', + type: 'string', + alias: 'd', + }); + + args.option('revision', { + describe: `revision to restore`, + }); + + args.option('output-dir', { + describe: 'directory in which to restore the data', + type: 'string', + demandOption: true, + }); + + args.option('archive-dir', { + describe: 'directory in which the archives are stored', + type: 'string', + default: '.appmap/archive', + }); + + return args.strict(); +}; + +export const handler = async (argv: any) => { + verbose(argv.verbose); + + handleWorkingDirectory(argv.directory); + + const { revision: defaultRevision, outputDir, archiveDir } = argv; + + const revision = await gitRevision(defaultRevision); + + console.log(`Restoring AppMaps of revision ${revision} to ${outputDir}`); + + const ancestors = await gitAncestors(defaultRevision); + + // Find the AppMap tarball that's closest in the git ancestry. + const fullArchivesAvailable = await promisify(glob)(join(archiveDir, 'full', '*.tar')); + const mostRecentFullArchive = fullArchivesAvailable.find((archive) => { + const revision = basename(archive, '.tar'); + return ancestors.includes(revision); + }); + if (!mostRecentFullArchive) throw new Error(`No AppMap tarball found in the ancestry of HEAD`); + + const baseRevision = basename(mostRecentFullArchive, '.tar'); + + console.log(`Using revision ${baseRevision} as the baseline`); + + await rmdir(outputDir, { recursive: true }); + + console.log(`Restoring full archive ${mostRecentFullArchive}`); + await unpackArchive(outputDir, mostRecentFullArchive); + + const baseRevisionIndex = ancestors.indexOf(baseRevision); + assert(baseRevisionIndex !== -1); + const ancestorsAfterBaseRevision = new Set(ancestors.slice(0, baseRevisionIndex)); + const incrementalArchivesAvailable = ( + await promisify(glob)(join(archiveDir, 'incremental', '*.tar')) + ).filter((archive) => { + const revision = basename(archive, '.tar'); + return ancestorsAfterBaseRevision.has(revision); + }); + + console.log(`Applying incremental archives ${incrementalArchivesAvailable.join(', ')}`); + + for (const archive of incrementalArchivesAvailable) { + await unpackArchive(outputDir, archive); + } + + console.log(`Updating indexes`); + + process.stdout.write(`Indexing AppMaps...`); + const numIndexed = await new FingerprintDirectoryCommand(outputDir).execute(); + process.stdout.write(`done (${numIndexed})\n`); + + console.log(`Restore complete`); +}; diff --git a/packages/cli/src/cmds/archive/unpackArchive.ts b/packages/cli/src/cmds/archive/unpackArchive.ts new file mode 100644 index 0000000000..0110ddd6e1 --- /dev/null +++ b/packages/cli/src/cmds/archive/unpackArchive.ts @@ -0,0 +1,40 @@ +import { exec } from 'child_process'; +import { mkdir, readFile, rename, rmdir, unlink } from 'fs/promises'; +import { basename, join, resolve } from 'path'; +import { Metadata } from './Metadata'; + +export default async function unpackArchive(outputDir: any, archivePath: string) { + await mkdir(outputDir, { recursive: true }); + + const dir = process.cwd(); + try { + process.chdir(outputDir); + + await new Promise((resolveCB, rejectCB) => { + exec(`tar xf ${resolve(dir, archivePath)}`, (error) => { + if (error) rejectCB(error); + + resolveCB(); + }); + }); + await new Promise((resolveCB, rejectCB) => { + exec(`tar xf appmaps.tar.gz`, (error) => { + if (error) rejectCB(error); + + resolveCB(); + }); + }); + await unlink('appmaps.tar.gz'); + + const metadata: Metadata = JSON.parse(await readFile('appmap_archive.json', 'utf8')); + await rename('appmap_archive.json', `appmap_archive.${metadata.revision}.json`); + + const deletedAppMaps = metadata.deletedAppMaps || []; + for (const deletedAppMap of deletedAppMaps) { + await unlink(deletedAppMap); + await rmdir(basename(deletedAppMap, '.appmap.json'), { recursive: true }); + } + } finally { + process.chdir(dir); + } +} From 38cff99946fb75699f018913a4d2d50fc4070250 Mon Sep 17 00:00:00 2001 From: Kevin Gilpin Date: Thu, 16 Mar 2023 13:37:13 -0400 Subject: [PATCH 04/24] feat: Update AppMaps by running out of date tests --- packages/cli/src/cli.ts | 2 + .../cli/src/cmds/archive/gitDeletedFiles.ts | 17 +++ .../cli/src/cmds/archive/gitModifiedFiles.ts | 17 +++ packages/cli/src/cmds/archive/gitNewFiles.ts | 14 ++ packages/cli/src/cmds/archive/runTests.ts | 41 ++++++ packages/cli/src/cmds/update.ts | 130 ++++++++++++++++++ packages/cli/src/lib/loadAppMapConfig.ts | 3 +- 7 files changed, 223 insertions(+), 1 deletion(-) create mode 100644 packages/cli/src/cmds/archive/gitDeletedFiles.ts create mode 100644 packages/cli/src/cmds/archive/gitModifiedFiles.ts create mode 100644 packages/cli/src/cmds/archive/gitNewFiles.ts create mode 100644 packages/cli/src/cmds/archive/runTests.ts create mode 100644 packages/cli/src/cmds/update.ts diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 741c0744ad..29745b3595 100755 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -35,6 +35,7 @@ const StatsCommand = require('./cmds/stats/stats'); import UploadCommand from './cmds/upload'; const BuildArchive = require('./cmds/archive/archive'); const RestoreArchive = require('./cmds/archive/restore'); +const UpdateAppMaps = require('./cmds/update'); class DiffCommand { public appMapNames: any; @@ -459,6 +460,7 @@ yargs(process.argv.slice(2)) .command(UploadCommand) .command(BuildArchive) .command(RestoreArchive) + .command(UpdateAppMaps) .strict() .demandCommand() .help().argv; diff --git a/packages/cli/src/cmds/archive/gitDeletedFiles.ts b/packages/cli/src/cmds/archive/gitDeletedFiles.ts new file mode 100644 index 0000000000..857dbb6316 --- /dev/null +++ b/packages/cli/src/cmds/archive/gitDeletedFiles.ts @@ -0,0 +1,17 @@ +import { exec } from 'child_process'; + +export default async function gitDeletedFiles( + revision: string, + folders?: string[] +): Promise { + const command = [`git diff --name-only --no-renames --diff-filter=D ${revision}`]; + if (folders) command.push(...folders); + + return new Promise((resolve, reject) => { + exec(command.join(' '), (error, stdout) => { + if (error) reject(error); + + resolve(stdout.trim().split('\n').filter(Boolean)); + }); + }); +} diff --git a/packages/cli/src/cmds/archive/gitModifiedFiles.ts b/packages/cli/src/cmds/archive/gitModifiedFiles.ts new file mode 100644 index 0000000000..0b439d4bca --- /dev/null +++ b/packages/cli/src/cmds/archive/gitModifiedFiles.ts @@ -0,0 +1,17 @@ +import { exec } from 'child_process'; + +export default async function gitModifiedFiles( + revision: string, + folders?: string[] +): Promise { + const command = [`git diff --name-only --no-renames --diff-filter=d ${revision}`]; + if (folders) command.push(...folders); + + return new Promise((resolve, reject) => { + exec(command.join(' '), (error, stdout) => { + if (error) reject(error); + + resolve(stdout.trim().split('\n').filter(Boolean)); + }); + }); +} diff --git a/packages/cli/src/cmds/archive/gitNewFiles.ts b/packages/cli/src/cmds/archive/gitNewFiles.ts new file mode 100644 index 0000000000..8bb19a692e --- /dev/null +++ b/packages/cli/src/cmds/archive/gitNewFiles.ts @@ -0,0 +1,14 @@ +import { exec } from 'child_process'; + +export default async function gitNewFiles(folders?: string[]): Promise { + const command = ['git ls-files -o --exclude-standard']; + if (folders) command.push(...folders); + + return new Promise((resolve, reject) => { + exec(command.join(' '), (error, stdout) => { + if (error) reject(error); + + resolve(stdout.trim().split('\n').filter(Boolean)); + }); + }); +} diff --git a/packages/cli/src/cmds/archive/runTests.ts b/packages/cli/src/cmds/archive/runTests.ts new file mode 100644 index 0000000000..3290f46267 --- /dev/null +++ b/packages/cli/src/cmds/archive/runTests.ts @@ -0,0 +1,41 @@ +import chalk from 'chalk'; +import { exec, spawn } from 'child_process'; +import { mkdtemp, rmdir, writeFile } from 'fs/promises'; +import { tmpdir } from 'os'; +import { join } from 'path'; + +export default async function runTests(baseCommand: string, testFiles: string[]) { + const injectTestFiles = (): string | undefined => { + if (!baseCommand.includes('{testFiles}')) return; + + const [prefix, suffix] = baseCommand.split('{testFiles}'); + return [prefix, testFiles.join(' '), suffix].join(''); + }; + + const argumentTestFiles = async (): Promise => { + const tempDir = await mkdtemp(join(tmpdir(), 'appmap_')); + + process.on('exit', () => rmdir(tempDir, { recursive: true })); + + await writeFile(join(tempDir, 'added'), testFiles.join('\n')); + + return [baseCommand, join(tempDir, 'added')].join(' '); + }; + + const command = injectTestFiles() || (await argumentTestFiles()); + + console.log(chalk.magenta(`Running tests: ${command}`)); + + return new Promise((resolve) => { + const cmd = spawn(command.split(' ')[0], command.split(' ').slice(1)); + cmd.stderr.pipe(process.stderr); + cmd.stdout.pipe(process.stdout); + cmd.on('close', (code) => { + if (code) { + console.error(chalk.red(`Test command failed (${code})`)); + } + + resolve(); + }); + }); +} diff --git a/packages/cli/src/cmds/update.ts b/packages/cli/src/cmds/update.ts new file mode 100644 index 0000000000..35cdb60514 --- /dev/null +++ b/packages/cli/src/cmds/update.ts @@ -0,0 +1,130 @@ +import chalk from 'chalk'; +import { readFile } from 'fs/promises'; +import { glob } from 'glob'; +import { join } from 'path'; +import { promisify } from 'util'; +import yargs from 'yargs'; +import { handleWorkingDirectory } from '../lib/handleWorkingDirectory'; +import loadAppMapConfig from '../lib/loadAppMapConfig'; +import { verbose } from '../utils'; +import gitDeletedFiles from './archive/gitDeletedFiles'; +import gitModifiedFiles from './archive/gitModifiedFiles'; +import gitNewFiles from './archive/gitNewFiles'; +import { Metadata } from './archive/Metadata'; +import runTests from './archive/runTests'; + +export const command = 'update'; +export const describe = 'Update AppMaps by running new and out-of-date tests'; + +export const builder = (args: yargs.Argv) => { + args.option('directory', { + describe: 'program working directory', + type: 'string', + alias: 'd', + }); + args.option('appmap-dir', { + describe: 'directory to recursively inspect for AppMaps', + }); + + args.option('base-appmap-dir', { + describe: 'directory in which base AppMaps are available', + type: 'string', + demandOption: true, + }); + + args.option('test-folder', { + describe: 'folder in which tests are located', + array: true, + }); + + args.option('test-command', { + describe: 'command to invoke with the updated (added + modified) tests', + array: true, + }); + + return args.strict(); +}; + +export const handler = async (argv: any) => { + verbose(argv.verbose); + + const { baseAppmapDir } = argv; + + handleWorkingDirectory(argv.directory); + const appmapConfig = await loadAppMapConfig(); + if (!appmapConfig) throw new Error(`Unable to load appmap.yml config file`); + + const testFoldersArgument = (): string[] | undefined => argv.testFolder; + const testFoldersConfig = (): string[] | undefined => appmapConfig.preflight?.test_folders; + + const testFolders = testFoldersArgument() || testFoldersConfig(); + if (!testFolders) + console.warn( + chalk.yellow(`No test folders specified. Use --test-folder arguments, or configure +test_folders in appmap.yml. In the absence of this information, all added and changed files will +be included in the file arguments passed to the test runner.`) + ); + + const testCommandsArgument = (): Record | undefined => { + const testCommands = argv.testCommand as string[] | undefined; + if (!testCommands) return; + + return testCommands.reduce((memo, line) => { + const match = /\[([^\]]+)\]\s+(.*)/.exec(line); + if (!match) { + console.warn(chalk.red(`Invalid test command: ${line}`)); + } else { + const folder = match[1]; + const command = match[2]; + memo[folder] = command; + } + return memo; + }, {}); + }; + const testCommandsConfig = (): Record | undefined => + appmapConfig.preflight?.test_commands; + const testCommands = testCommandsArgument() || testCommandsConfig(); + + const fullArchives = ( + await Promise.all( + ( + await promisify(glob)(join(baseAppmapDir, 'appmap_archive.*.json')) + ).map(async (metadataFile) => JSON.parse(await readFile(metadataFile, 'utf8'))) + ) + ).filter((metadata) => !metadata.baseRevision); + + if (fullArchives.length !== 1) + throw new Error( + `Expecting exactly one full archive in ${baseAppmapDir}, found ${fullArchives.length}.` + ); + + const baseRevision = fullArchives[0].revision; + + const addedTestFiles = await gitNewFiles(testFolders); + const modifiedTestFiles = await gitModifiedFiles(baseRevision, testFolders); + const deletedTestFiles = await gitDeletedFiles(baseRevision, testFolders); + const updatedTestFiles = [...new Set([...addedTestFiles, ...modifiedTestFiles])].sort(); + + if (addedTestFiles.length > 0) console.log(`Added tests: ${addedTestFiles.join(' ')}`); + if (modifiedTestFiles.length > 0) console.log(`Modified tests: ${modifiedTestFiles.join(' ')}`); + if (deletedTestFiles.length > 0) console.log(`Deleted tests: ${deletedTestFiles.join(' ')}`); + if (updatedTestFiles.length > 0) + console.log(`Updated (added+modified) tests: ${updatedTestFiles.join(' ')}`); + + const testFilesByFolder = new Map(); + updatedTestFiles.forEach((testFile) => { + testFolders?.forEach((folder) => { + if (testFile.startsWith(folder)) + testFilesByFolder.set(folder, [...(testFilesByFolder.get(folder) || []), testFile]); + }); + }); + + for (const [folder, testFiles] of testFilesByFolder.entries()) { + const command = testCommands?.[folder]; + if (!command) { + console.warn(chalk.yellow(`No test command configured for folder ${folder}`)); + continue; + } + runTests(command, testFiles); + } +}; diff --git a/packages/cli/src/lib/loadAppMapConfig.ts b/packages/cli/src/lib/loadAppMapConfig.ts index c98ec46dd3..e3dbeca730 100644 --- a/packages/cli/src/lib/loadAppMapConfig.ts +++ b/packages/cli/src/lib/loadAppMapConfig.ts @@ -8,7 +8,8 @@ interface PreflightFilterConfig { interface PreflightConfig { base_branch?: string; - test_command?: string; + test_folders?: string[]; + test_commands?: Record; filter?: PreflightFilterConfig; } From 61c1c233b851cafc460d37ee5988e242b8f6ef45 Mon Sep 17 00:00:00 2001 From: Kevin Gilpin Date: Thu, 16 Mar 2023 16:22:33 -0400 Subject: [PATCH 05/24] wip: Add out-of-date AppMaps to the test run --- .../cli/src/cmds/archive/gitModifiedFiles.ts | 5 +- packages/cli/src/cmds/archive/runTests.ts | 2 +- packages/cli/src/cmds/update.ts | 79 ++++++++++++++++--- packages/cli/src/depends.js | 2 +- 4 files changed, 76 insertions(+), 12 deletions(-) diff --git a/packages/cli/src/cmds/archive/gitModifiedFiles.ts b/packages/cli/src/cmds/archive/gitModifiedFiles.ts index 0b439d4bca..e1ffb48b78 100644 --- a/packages/cli/src/cmds/archive/gitModifiedFiles.ts +++ b/packages/cli/src/cmds/archive/gitModifiedFiles.ts @@ -2,9 +2,12 @@ import { exec } from 'child_process'; export default async function gitModifiedFiles( revision: string, + diffFilters: string[], folders?: string[] ): Promise { - const command = [`git diff --name-only --no-renames --diff-filter=d ${revision}`]; + const command = [`git diff --name-only --no-renames `]; + if (diffFilters.length > 0) command.push(`--diff-filter=${diffFilters.join('')}`); + command.push(revision); if (folders) command.push(...folders); return new Promise((resolve, reject) => { diff --git a/packages/cli/src/cmds/archive/runTests.ts b/packages/cli/src/cmds/archive/runTests.ts index 3290f46267..caab643da6 100644 --- a/packages/cli/src/cmds/archive/runTests.ts +++ b/packages/cli/src/cmds/archive/runTests.ts @@ -1,5 +1,5 @@ import chalk from 'chalk'; -import { exec, spawn } from 'child_process'; +import { spawn } from 'child_process'; import { mkdtemp, rmdir, writeFile } from 'fs/promises'; import { tmpdir } from 'os'; import { join } from 'path'; diff --git a/packages/cli/src/cmds/update.ts b/packages/cli/src/cmds/update.ts index 35cdb60514..241b780574 100644 --- a/packages/cli/src/cmds/update.ts +++ b/packages/cli/src/cmds/update.ts @@ -1,17 +1,21 @@ import chalk from 'chalk'; import { readFile } from 'fs/promises'; import { glob } from 'glob'; -import { join } from 'path'; +import { dirname, join, relative, resolve } from 'path'; import { promisify } from 'util'; import yargs from 'yargs'; +import { Metadata as AppMapMetadata } from '@appland/models'; +import Depends from '../depends'; import { handleWorkingDirectory } from '../lib/handleWorkingDirectory'; import loadAppMapConfig from '../lib/loadAppMapConfig'; +import { locateAppMapDir } from '../lib/locateAppMapDir'; import { verbose } from '../utils'; import gitDeletedFiles from './archive/gitDeletedFiles'; import gitModifiedFiles from './archive/gitModifiedFiles'; import gitNewFiles from './archive/gitNewFiles'; import { Metadata } from './archive/Metadata'; import runTests from './archive/runTests'; +import FingerprintDirectoryCommand from '../fingerprint/fingerprintDirectoryCommand'; export const command = 'update'; export const describe = 'Update AppMaps by running new and out-of-date tests'; @@ -51,6 +55,7 @@ export const handler = async (argv: any) => { const { baseAppmapDir } = argv; handleWorkingDirectory(argv.directory); + const appmapDir = await locateAppMapDir(); const appmapConfig = await loadAppMapConfig(); if (!appmapConfig) throw new Error(`Unable to load appmap.yml config file`); @@ -101,21 +106,75 @@ be included in the file arguments passed to the test runner.`) const baseRevision = fullArchives[0].revision; const addedTestFiles = await gitNewFiles(testFolders); - const modifiedTestFiles = await gitModifiedFiles(baseRevision, testFolders); + const modifiedFiles = await gitModifiedFiles(baseRevision, []); + const modifiedTestFiles = await gitModifiedFiles(baseRevision, ['d'], testFolders); const deletedTestFiles = await gitDeletedFiles(baseRevision, testFolders); - const updatedTestFiles = [...new Set([...addedTestFiles, ...modifiedTestFiles])].sort(); + const outOfDateTestFiles = new Set([...addedTestFiles, ...modifiedTestFiles]); if (addedTestFiles.length > 0) console.log(`Added tests: ${addedTestFiles.join(' ')}`); if (modifiedTestFiles.length > 0) console.log(`Modified tests: ${modifiedTestFiles.join(' ')}`); if (deletedTestFiles.length > 0) console.log(`Deleted tests: ${deletedTestFiles.join(' ')}`); - if (updatedTestFiles.length > 0) - console.log(`Updated (added+modified) tests: ${updatedTestFiles.join(' ')}`); + + const indexAppMaps = async () => { + process.stdout.write(`Indexing AppMaps...`); + const numIndexed = await new FingerprintDirectoryCommand(appmapDir).execute(); + process.stdout.write(`done (${numIndexed})\n`); + }; + + await indexAppMaps(); + + const dirtyTests = new Set([...modifiedTestFiles, ...deletedTestFiles]); + const deletedAppMaps = new Array(); + // Remove AppMaps for deleted and modified tests + await Promise.all( + ( + await promisify(glob)(join(appmapDir, '**/metadata.json')) + ).map(async (metadataFile) => { + const metadata = JSON.parse(await readFile(metadataFile, 'utf-8')); + const { source_location: sourceLocation } = metadata; + if (!sourceLocation) return; + + const [path] = sourceLocation.split(':'); + + if (dirtyTests.has(path)) { + const appmapName = dirname(metadataFile); + console.warn( + `AppMap ${appmapName} is out of date because ${path} has been ${ + modifiedTestFiles.includes(path) ? 'modified' : 'deleted' + }` + ); + deletedAppMaps.push(appmapName); + } + }) + ); + + const depends = new Depends(appmapDir); + depends.files = modifiedFiles; + const outOfDateAppMaps = await depends.depends(); + if (outOfDateAppMaps.length > 0) { + console.log(`Out-of-date AppMaps: ${outOfDateAppMaps.join(' ')}`); + for (const appmap of outOfDateAppMaps) { + const metadata = JSON.parse( + await readFile(join(appmap, 'metadata.json'), 'utf-8') + ) as AppMapMetadata; + if (metadata.source_location) outOfDateTestFiles.add(metadata.source_location.split(':')[0]); + } + } + + if (outOfDateTestFiles.size > 0) + console.log(`Updated (added+modified) tests: ${[...outOfDateTestFiles].join(' ')}`); + + if (outOfDateTestFiles.size === 0) { + console.log('No changes detected to the code or tests.'); + return; + } const testFilesByFolder = new Map(); - updatedTestFiles.forEach((testFile) => { + outOfDateTestFiles.forEach((testFile) => { + const localPath = relative(process.cwd(), testFile); testFolders?.forEach((folder) => { - if (testFile.startsWith(folder)) - testFilesByFolder.set(folder, [...(testFilesByFolder.get(folder) || []), testFile]); + if (localPath.startsWith(folder)) + testFilesByFolder.set(folder, [...(testFilesByFolder.get(folder) || []), localPath]); }); }); @@ -125,6 +184,8 @@ be included in the file arguments passed to the test runner.`) console.warn(chalk.yellow(`No test command configured for folder ${folder}`)); continue; } - runTests(command, testFiles); + await runTests(command, testFiles); } + + await indexAppMaps(); }; diff --git a/packages/cli/src/depends.js b/packages/cli/src/depends.js index 2845b1df7f..c08e26fe81 100644 --- a/packages/cli/src/depends.js +++ b/packages/cli/src/depends.js @@ -66,7 +66,7 @@ class Depends { * @param {function} callback * @returns string[] */ - async depends(callback) { + async depends(callback = undefined) { const outOfDateNames = new Set(); const checkClassMap = async (fileName) => { From 4e67c6b46a6d5d6ac731f32b7bf507b3209befc2 Mon Sep 17 00:00:00 2001 From: Kevin Gilpin Date: Fri, 17 Mar 2023 15:49:44 -0400 Subject: [PATCH 06/24] test: Fix OpenAPI spec --- packages/cli/tests/unit/openapi.spec.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/cli/tests/unit/openapi.spec.ts b/packages/cli/tests/unit/openapi.spec.ts index 0c54ac1fcc..df8499ecca 100644 --- a/packages/cli/tests/unit/openapi.spec.ts +++ b/packages/cli/tests/unit/openapi.spec.ts @@ -1,7 +1,8 @@ import { verbose } from '../../src/utils'; -import { default as openapi, fileSizeFilter } from '../../src/cmds/openapi'; +import { default as openapi } from '../../src/cmds/openapi'; import assert from 'assert'; import path from 'path'; +import { fileSizeFilter } from '../../src/lib/fileSizeFilter'; describe('OpenAPI', () => { const malformedHTTPServerRequestDir = 'tests/unit/fixtures/malformedHTTPServerRequest'; From 2428e561b154dc198a4dfa7646d1efcd669953dd Mon Sep 17 00:00:00 2001 From: Kevin Gilpin Date: Fri, 17 Mar 2023 15:50:12 -0400 Subject: [PATCH 07/24] fix: Fingerprint an empty directory --- packages/cli/src/fingerprint/fingerprintDirectoryCommand.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/src/fingerprint/fingerprintDirectoryCommand.js b/packages/cli/src/fingerprint/fingerprintDirectoryCommand.js index 14d6774715..d331b424c2 100644 --- a/packages/cli/src/fingerprint/fingerprintDirectoryCommand.js +++ b/packages/cli/src/fingerprint/fingerprintDirectoryCommand.js @@ -18,7 +18,7 @@ class FingerprintDirectoryCommand { count += 1; return fpQueue.push(file); }); - await fpQueue.process(); + if (count > 0) await fpQueue.process(); return count; } From 28f5dfaeb7a478876a9456dd5db30971f43cd5bd Mon Sep 17 00:00:00 2001 From: Kevin Gilpin Date: Fri, 17 Mar 2023 15:50:29 -0400 Subject: [PATCH 08/24] misc: Scripts to build archives of known repos AppMap server Rails sample app 6th ed --- packages/cli/contrib/build_appland_archive | 54 ++++++++++++++++ .../build_rails_sample_app_6th_ed_archive | 61 +++++++++++++++++++ 2 files changed, 115 insertions(+) create mode 100755 packages/cli/contrib/build_appland_archive create mode 100755 packages/cli/contrib/build_rails_sample_app_6th_ed_archive diff --git a/packages/cli/contrib/build_appland_archive b/packages/cli/contrib/build_appland_archive new file mode 100755 index 0000000000..bfef9e7e8d --- /dev/null +++ b/packages/cli/contrib/build_appland_archive @@ -0,0 +1,54 @@ +#!/usr/bin/env ruby + +require 'fileutils' + +revision = ARGV[0] +no_test = true if ARGV[1..-1].member?('--no-test') + +cli_dir = Dir.pwd + +sys_command = ->(cmd, env = nil) do + Array(cmd).each do |c| + args = [ env, c ].compact + p args + system *args or raise "Command failed: #{c}" + end +end + +node_version = `node -v` +raise "Node version should be 16, got #{node_version}" unless node_version.index('v16.') + +Dir.chdir File.expand_path('~/source/appland/appland') +appland_dir = Dir.pwd + +required_ruby_version = File.read('.ruby-version').strip +ruby_version = `ruby -v` +raise "Ruby version must be #{required_ruby_version}, is #{ruby_version}" unless ruby_version.index(required_ruby_version) + +sys_command.call [ + "git checkout #{revision}", +] + +unless no_test + sys_command.call 'git lfs track "*.tar"' \ + unless File.read('.gitattributes').index('*.tar filter=lfs diff=lfs merge=lfs -text') + + env = `bundle exec rake dev:db`.split("\n").select { |line| line.index('=') }.map { |line| k, v = line.split('='); [ k['export '.length..-1], v] }.to_h + + FileUtils.rm_rf 'tmp/appmap' + FileUtils.mkdir_p 'tmp/appmap/rspec' + FileUtils.mkdir_p '.appmap/archive/full' + + sys_command.call [ + 'npm install', + './bin/webpack', + 'psql -U postgres -h localhost -c "create database appland_test" || true', + 'rails db:migrate', + 'rails db:test:prepare', + 'bundle exec rspec' + ], env.merge('RAILS_ENV' => 'test', 'DISABLE_SPRING' => 'true') +end + +Dir.chdir cli_dir + +system "node ./built/src/cli.js archive" diff --git a/packages/cli/contrib/build_rails_sample_app_6th_ed_archive b/packages/cli/contrib/build_rails_sample_app_6th_ed_archive new file mode 100755 index 0000000000..3d46ccef1b --- /dev/null +++ b/packages/cli/contrib/build_rails_sample_app_6th_ed_archive @@ -0,0 +1,61 @@ +#!/usr/bin/env ruby + +require 'fileutils' + +revision = ARGV[0] +no_test = true if ARGV[1..-1]&.member?('--no-test') + +cli_dir = Dir.pwd + +sys_command = ->(cmd, env = nil) do + Array(cmd).each do |c| + args = [ env, c ].compact + p args + system *args or raise "Command failed: #{c}" + end +end + +node_version = `node -v` +raise "Node version should be 14, got #{node_version}" unless node_version.index('v14.') + +Dir.chdir File.expand_path('~/source/land-of-apps/sample_app_6th_ed') +project_dir = Dir.pwd + +required_ruby_version = '2.7.3' +ruby_version = `ruby -v` +raise "Ruby version must be #{required_ruby_version}, is #{ruby_version}" unless ruby_version.index(required_ruby_version) + +sys_command.call [ + "git checkout #{revision}", +] + +unless no_test + sys_command.call 'git lfs track "*.tar"' \ + unless File.read('.gitattributes').index('*.tar filter=lfs diff=lfs merge=lfs -text') + + File.write 'appmap.yml', <<~APPMAP_CONFIG +name: sample_app_6th_ed +language: ruby +appmap_dir: tmp/appmap +packages: + - path: app + - path: lib +APPMAP_CONFIG + + FileUtils.rm_rf 'tmp/appmap' + FileUtils.mkdir_p 'tmp/appmap/minitest' + FileUtils.mkdir_p '.appmap/archive/full' + + sys_command.call [ + 'bundle update mimemagic puma', + 'bundle info appmap || bundle add appmap', + 'bundle update appmap', + 'yarn install', + 'rails db:test:prepare', + 'rails test' + ], { 'RAILS_ENV' => 'test', 'DISABLE_SPRING' => 'true' } +end + +Dir.chdir cli_dir + +system "node ./built/src/cli.js archive -d #{project_dir}" From a1280b98837056aa5bcfc30077335eb7dc3d6b5a Mon Sep 17 00:00:00 2001 From: Kevin Gilpin Date: Sat, 18 Mar 2023 15:59:40 -0400 Subject: [PATCH 09/24] feat: Update archive/restore/update commands --- packages/cli/src/cmds/archive/archive.ts | 29 ++++--- packages/cli/src/cmds/archive/gitAncestors.ts | 2 +- packages/cli/src/cmds/archive/restore.ts | 2 +- packages/cli/src/cmds/update.ts | 75 ++++++++++++------- 4 files changed, 67 insertions(+), 41 deletions(-) diff --git a/packages/cli/src/cmds/archive/archive.ts b/packages/cli/src/cmds/archive/archive.ts index 12cac0dd01..44c5a44697 100644 --- a/packages/cli/src/cmds/archive/archive.ts +++ b/packages/cli/src/cmds/archive/archive.ts @@ -68,14 +68,13 @@ commit of the current git revision may not be the one that triggered the build.` }); args.option('output-dir', { - describe: 'directory in which to save the output file', + describe: `directory in which to save the output file. By default, it's .appmap/archive/.`, type: 'string', }); args.option('output-file', { - describe: 'output file name', + describe: 'output file name. Default output name is .tar', type: 'string', - default: 'appmap_archive.tar', alias: 'f', }); @@ -107,15 +106,15 @@ export const handler = async (argv: any) => { const { concurrency, maxSize, - type, + type: typeArg, revision: defaultRevision, - outputFile: outputFileName, + outputFile: outputFileNameArg, } = argv; - const { outputDir } = argv || '.'; + const { outputDirArg } = argv; const maxAppMapSizeInBytes = Math.round(parseFloat(maxSize) * 1024 * 1024); - console.log(`Building '${type}' archive from ${appMapDir}`); + console.log(`Building '${typeArg}' archive from ${appMapDir}`); const revision = await gitRevision(defaultRevision); @@ -203,16 +202,17 @@ export const handler = async (argv: any) => { config: appmapConfig, }; + let type: string; if (await exists('appmap_archive.json')) { const existingMetadata = JSON.parse(await readFile('appmap_archive.json', 'utf8')); const { revision: baseRevision } = existingMetadata; - if (type === 'auto') { + if (typeArg === 'auto') { console.log( `The AppMap directory contains appmap_archive.json, so the archive type will be 'incremental'. The base revision is ${baseRevision}.` ); } - if (type === 'full') { + if (typeArg === 'full') { console.warn( chalk.yellow( `\nThe AppMap directory contains appmap_archive.json, so it looks like the directory contains incremental AppMaps @@ -223,17 +223,19 @@ remove appmap_archive.json if your intention is to build a full archive. Otherwi } else { metadata.baseRevision = baseRevision; } + type = 'incremental'; } else { - if (type === 'auto') { + if (typeArg === 'auto') { console.log( `The AppMap directory does not contain appmap_archive.json, so the archive type will be 'full'.` ); - } else if (type === 'incremental') { + } else if (typeArg === 'incremental') { throw new Error( `AppMap directory does not contain appmap_archive.json, but you've specified --type=incremental. The base revision cannot be determined, so either use --type=auto or --type=full.` ); } + type = 'full'; } await new Promise((resolveCB, rejectCB) => { @@ -251,6 +253,11 @@ The base revision cannot be determined, so either use --type=auto or --type=full await writeFile('appmap_archive.json', JSON.stringify(metadata, null, 2)); + const outputFileName = outputFileNameArg || `${revision}.tar`; + + const defaultOutputDir = () => join('.appmap', 'archive', type); + const outputDir = outputDirArg || defaultOutputDir(); + await new Promise((resolveCB, rejectCB) => { exec( `tar cf ${join( diff --git a/packages/cli/src/cmds/archive/gitAncestors.ts b/packages/cli/src/cmds/archive/gitAncestors.ts index 1e2b7b1fb0..8a29c5ee53 100644 --- a/packages/cli/src/cmds/archive/gitAncestors.ts +++ b/packages/cli/src/cmds/archive/gitAncestors.ts @@ -1,6 +1,6 @@ import { exec } from 'child_process'; -export default async function gitAncestors(defaultRevision?: string): Promise { +export default async function gitAncestors(): Promise { return new Promise((resolve, reject) => { exec('git rev-list HEAD', (error, stdout) => { if (error) reject(error); diff --git a/packages/cli/src/cmds/archive/restore.ts b/packages/cli/src/cmds/archive/restore.ts index 44984695bc..39e6620d19 100644 --- a/packages/cli/src/cmds/archive/restore.ts +++ b/packages/cli/src/cmds/archive/restore.ts @@ -51,7 +51,7 @@ export const handler = async (argv: any) => { console.log(`Restoring AppMaps of revision ${revision} to ${outputDir}`); - const ancestors = await gitAncestors(defaultRevision); + const ancestors = await gitAncestors(); // Find the AppMap tarball that's closest in the git ancestry. const fullArchivesAvailable = await promisify(glob)(join(archiveDir, 'full', '*.tar')); diff --git a/packages/cli/src/cmds/update.ts b/packages/cli/src/cmds/update.ts index 241b780574..d300a6c4d1 100644 --- a/packages/cli/src/cmds/update.ts +++ b/packages/cli/src/cmds/update.ts @@ -1,5 +1,5 @@ import chalk from 'chalk'; -import { readFile } from 'fs/promises'; +import { mkdir, readFile, writeFile } from 'fs/promises'; import { glob } from 'glob'; import { dirname, join, relative, resolve } from 'path'; import { promisify } from 'util'; @@ -9,13 +9,15 @@ import Depends from '../depends'; import { handleWorkingDirectory } from '../lib/handleWorkingDirectory'; import loadAppMapConfig from '../lib/loadAppMapConfig'; import { locateAppMapDir } from '../lib/locateAppMapDir'; -import { verbose } from '../utils'; +import { exists, verbose } from '../utils'; import gitDeletedFiles from './archive/gitDeletedFiles'; import gitModifiedFiles from './archive/gitModifiedFiles'; import gitNewFiles from './archive/gitNewFiles'; import { Metadata } from './archive/Metadata'; import runTests from './archive/runTests'; import FingerprintDirectoryCommand from '../fingerprint/fingerprintDirectoryCommand'; +import gitAncestors from './archive/gitAncestors'; +import assert from 'assert'; export const command = 'update'; export const describe = 'Update AppMaps by running new and out-of-date tests'; @@ -90,20 +92,22 @@ be included in the file arguments passed to the test runner.`) appmapConfig.preflight?.test_commands; const testCommands = testCommandsArgument() || testCommandsConfig(); - const fullArchives = ( - await Promise.all( - ( - await promisify(glob)(join(baseAppmapDir, 'appmap_archive.*.json')) - ).map(async (metadataFile) => JSON.parse(await readFile(metadataFile, 'utf8'))) - ) - ).filter((metadata) => !metadata.baseRevision); - - if (fullArchives.length !== 1) - throw new Error( - `Expecting exactly one full archive in ${baseAppmapDir}, found ${fullArchives.length}.` - ); + const archiveMetadata = await Promise.all( + ( + await promisify(glob)(join(baseAppmapDir, 'appmap_archive.*.json')) + ).map(async (metadataFile) => JSON.parse(await readFile(metadataFile, 'utf8'))) + ); + + const archiveRevisions = archiveMetadata.map((metadata) => metadata.revision); + const repoRevisions = await gitAncestors(); + const baseRevision = repoRevisions.find((revision) => archiveRevisions.includes(revision)); - const baseRevision = fullArchives[0].revision; + if (!baseRevision) throw new Error(`No AppMap archive found for any ancestor revision.`); + + console.log(`Comparing code changes with revision ${baseRevision}`); + + const baseArchive = archiveMetadata.find((metadata) => metadata.revision === baseRevision); + assert(baseArchive); const addedTestFiles = await gitNewFiles(testFolders); const modifiedFiles = await gitModifiedFiles(baseRevision, []); @@ -111,17 +115,21 @@ be included in the file arguments passed to the test runner.`) const deletedTestFiles = await gitDeletedFiles(baseRevision, testFolders); const outOfDateTestFiles = new Set([...addedTestFiles, ...modifiedTestFiles]); - if (addedTestFiles.length > 0) console.log(`Added tests: ${addedTestFiles.join(' ')}`); - if (modifiedTestFiles.length > 0) console.log(`Modified tests: ${modifiedTestFiles.join(' ')}`); - if (deletedTestFiles.length > 0) console.log(`Deleted tests: ${deletedTestFiles.join(' ')}`); + console.log(`${outOfDateTestFiles.size} test case files are added or modified.`); + + if (verbose()) { + if (addedTestFiles.length > 0) console.log(`Added tests: ${addedTestFiles.join(' ')}`); + if (modifiedTestFiles.length > 0) console.log(`Modified tests: ${modifiedTestFiles.join(' ')}`); + if (deletedTestFiles.length > 0) console.log(`Deleted tests: ${deletedTestFiles.join(' ')}`); + } - const indexAppMaps = async () => { + const indexAppMaps = async (dir: string) => { process.stdout.write(`Indexing AppMaps...`); - const numIndexed = await new FingerprintDirectoryCommand(appmapDir).execute(); + const numIndexed = await new FingerprintDirectoryCommand(dir).execute(); process.stdout.write(`done (${numIndexed})\n`); }; - await indexAppMaps(); + await indexAppMaps(baseAppmapDir); const dirtyTests = new Set([...modifiedTestFiles, ...deletedTestFiles]); const deletedAppMaps = new Array(); @@ -148,11 +156,12 @@ be included in the file arguments passed to the test runner.`) }) ); - const depends = new Depends(appmapDir); + const depends = new Depends(baseAppmapDir); depends.files = modifiedFiles; const outOfDateAppMaps = await depends.depends(); if (outOfDateAppMaps.length > 0) { - console.log(`Out-of-date AppMaps: ${outOfDateAppMaps.join(' ')}`); + console.log(`${outOfDateAppMaps.length} AppMaps are out of date.`); + if (verbose()) console.log(`Out-of-date AppMaps: ${outOfDateAppMaps.join(' ')}`); for (const appmap of outOfDateAppMaps) { const metadata = JSON.parse( await readFile(join(appmap, 'metadata.json'), 'utf-8') @@ -161,22 +170,32 @@ be included in the file arguments passed to the test runner.`) } } - if (outOfDateTestFiles.size > 0) - console.log(`Updated (added+modified) tests: ${[...outOfDateTestFiles].join(' ')}`); + if (verbose()) { + if (outOfDateTestFiles.size > 0) + console.log(`Updated (added+modified) tests: ${[...outOfDateTestFiles].join(' ')}`); + } if (outOfDateTestFiles.size === 0) { console.log('No changes detected to the code or tests.'); return; } + console.log(`${outOfDateTestFiles.size} tests will be re-run.`); + + await mkdir(appmapDir, { recursive: true }); + const testFilesByFolder = new Map(); - outOfDateTestFiles.forEach((testFile) => { + for (const testFile of outOfDateTestFiles) { const localPath = relative(process.cwd(), testFile); + if (!(await exists(testFile))) { + console.warn(chalk.yellow(`Test file ${testFile} does not exist`)); + continue; + } testFolders?.forEach((folder) => { if (localPath.startsWith(folder)) testFilesByFolder.set(folder, [...(testFilesByFolder.get(folder) || []), localPath]); }); - }); + } for (const [folder, testFiles] of testFilesByFolder.entries()) { const command = testCommands?.[folder]; @@ -187,5 +206,5 @@ be included in the file arguments passed to the test runner.`) await runTests(command, testFiles); } - await indexAppMaps(); + await writeFile(join(appmapDir, 'appmap_archive.json'), JSON.stringify(baseArchive, null, 2)); }; From fd5397ac662740121ec4938d742f4ccdfba74a83 Mon Sep 17 00:00:00 2001 From: Kevin Gilpin Date: Wed, 4 Jan 2023 16:23:48 -0500 Subject: [PATCH 10/24] feat: Describe a commit by comparing seq diagrams Checkout a base commit, and generate seq diagrams. Checkout a head commit, and generate seq diagrams. Compare the diagrams and summarize the results. --- packages/cli/src/cli.ts | 3 +- packages/cli/src/cmds/describeChange.ts | 508 ++++++++++++++++++++++++ packages/cli/src/touchFile.ts | 16 + 3 files changed, 526 insertions(+), 1 deletion(-) create mode 100644 packages/cli/src/cmds/describeChange.ts create mode 100644 packages/cli/src/touchFile.ts diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 29745b3595..540e76a352 100755 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -32,10 +32,10 @@ const InspectCommand = require('./cmds/inspect/inspect'); const SequenceDiagramCommand = require('./cmds/sequenceDiagram'); const SequenceDiagramDiffCommand = require('./cmds/sequenceDiagramDiff'); const StatsCommand = require('./cmds/stats/stats'); -import UploadCommand from './cmds/upload'; const BuildArchive = require('./cmds/archive/archive'); const RestoreArchive = require('./cmds/archive/restore'); const UpdateAppMaps = require('./cmds/update'); +const DescribeChange = require('./cmds/describeChange'); class DiffCommand { public appMapNames: any; @@ -458,6 +458,7 @@ yargs(process.argv.slice(2)) .command(SequenceDiagramDiffCommand) .command(PruneCommand) .command(UploadCommand) + .command(DescribeChange) .command(BuildArchive) .command(RestoreArchive) .command(UpdateAppMaps) diff --git a/packages/cli/src/cmds/describeChange.ts b/packages/cli/src/cmds/describeChange.ts new file mode 100644 index 0000000000..fe33293460 --- /dev/null +++ b/packages/cli/src/cmds/describeChange.ts @@ -0,0 +1,508 @@ +import { mkdir, readFile, rename, rm, writeFile } from 'fs/promises'; +import yargs from 'yargs'; +import readline from 'readline'; + +import { handleWorkingDirectory } from '../lib/handleWorkingDirectory'; +import { locateAppMapDir } from '../lib/locateAppMapDir'; +import { fingerprintDirectory } from '../fingerprint'; +import Depends from '../depends'; +import { exists, verbose } from '../utils'; +import { exec } from 'child_process'; +import { cwd, nextTick, rawListeners } from 'process'; +import { queue } from 'async'; +import { basename, dirname, join } from 'path'; +import { existsSync } from 'fs'; +import { tmpdir } from 'os'; +import { + buildDiagram, + Diagram, + format, + FormatType, + SequenceDiagramOptions, + Specification, +} from '@appland/sequence-diagram'; +import { glob } from 'glob'; +import { promisify } from 'util'; +import { buildAppMap } from '@appland/models'; +import { DiffDiagrams } from '../sequenceDiagramDiff/DiffDiagrams'; +import assert from 'assert'; +import { touch } from '../touchFile'; +import chalk from 'chalk'; + +export const command = 'describe-change'; +export const describe = 'Compare base and head revisions'; + +export const builder = (args: yargs.Argv) => { + args.option('directory', { + describe: 'program working directory', + type: 'string', + alias: 'd', + }); + + args.option('base-revision', { + describe: 'base revision name or commit SHA (default is the previous commit)', + alias: 'base', + }); + args.option('head-revision', { + describe: 'head revision name or commit SHA (default is the current commit)', + alias: 'head', + }); + + args.option('output-dir', { + describe: 'directory in which to save the report files', + }); + args.option('clobber-output-dir', { + describe: 'remove the output directory if it exists', + type: 'boolean', + default: false, + }); + + args.option('base-command', { + describe: 'command to use to run the base tests', + }); + args.option('head-command', { + describe: 'command to use to run the head tests', + }); + + args.option('touch', { + describe: 'touch out-of-date test files when switching to a new revision', + type: 'boolean', + default: false, + }); + + args.option('plantuml-jar', { + describe: 'location of PlantUML JAR file', + default: 'plantuml.jar', + }); + + args.option('include', { + describe: 'code objects to include in the comparison (inclusive of descendants)', + }); + args.option('exclude', { + describe: 'code objects to exclude from the comparison', + }); + + return args.strict(); +}; + +export const handler = async (argv: any) => { + verbose(argv.verbose); + + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, + }); + rl.on('close', function () { + yargs.exit(0, new Error()); + }); + + handleWorkingDirectory(argv.directory); + const appmapDir = await locateAppMapDir(); + + const { touch: touchOutDateTestFiles, plantumlJar, baseCommand, headCommand } = argv; + + if (!(await exists(plantumlJar))) { + yargs.exit( + 1, + new Error( + `PlantUML JAR file ${plantumlJar} does not exist. Use --plantuml-jar option to provide the JAR file location.` + ) + ); + } + + if (argv.outputDir) { + if (await exists(argv.outputDir)) { + if ( + argv.clobberOutputDir || + !(await confirm(`Remove existing output directory ${argv.outputDir}?`, rl)) + ) { + yargs.exit(1, new Error(`Aborted`)); + } + await rm(argv.outputDir, { recursive: true, force: true }); + } + await mkdir(argv.outputDir, { recursive: true }); + } + + let currentBranch = (await executeCommand(`git branch --show-current`)).trim(); + if (currentBranch === '') { + currentBranch = (await executeCommand(`git show --format=oneline --abbrev-commit`)).split( + /\s/ + )[0]; + } + + console.log(prominentStyle(`Current revision is: ${currentBranch}`)); + console.log( + mutedStyle( + `Note: The repository will be returned to this revision if this command succeeds, but not if it fails.` + ) + ); + + const baseRevision = argv.baseRevision || 'HEAD~1'; + const headRevision = argv.headRevision || (currentBranch || 'HEAD').trim(); + + let baseDiagrams: Map; + let headDiagrams: Map; + + await stashAll(); + { + await checkout('base', baseRevision); + await createBaselineAppMaps(rl, baseCommand); + process.stdout.write(`Processing AppMaps...`); + baseDiagrams = await buildDiagrams(appmapDir); + process.stdout.write(`done (${baseDiagrams.size})\n`); + } + + { + await checkout('head', headRevision); + await updateAppMaps(appmapDir, rl, touchOutDateTestFiles, headCommand); + process.stdout.write(`Processing AppMaps...`); + headDiagrams = await buildDiagrams(appmapDir); + process.stdout.write(`done (${headDiagrams.size})\n`); + } + await unstashAll(); + + const appmapNameSet = new Set(); + [...baseDiagrams.keys()].forEach((file) => appmapNameSet.add(file)); + [...headDiagrams.keys()].forEach((file) => appmapNameSet.add(file)); + const appmapNames = [...appmapNameSet].sort(); + + const changedAppMaps = new Map(); + const diffSnippets = new Set(); + const reportLines = new Array(); + for (let i = 0; i < appmapNames.length; i++) { + const appmapName = appmapNames[i]; + + const baseDiagram = baseDiagrams.get(appmapName); + const headDiagram = headDiagrams.get(appmapName); + + if (!baseDiagram) { + assert(headDiagram); + + console.log(`${appmapName} is new in the head revision`); + let diagramFile = await showDiagram(rl, plantumlJar, headDiagram); + await reportDiagram( + appmapName, + reportLines, + argv.outputDir, + 'is new in the head revision', + diagramFile, + rl, + plantumlJar, + headDiagram + ); + continue; + } + if (!headDiagram) { + console.log(`${appmapName} is removed in the head revision`); + let diagramFile = await showDiagram(rl, plantumlJar, baseDiagram); + await reportDiagram( + appmapName, + reportLines, + argv.outputDir, + 'is removed from the head revision', + diagramFile, + rl, + plantumlJar, + baseDiagram + ); + continue; + } + + const diffDiagrams = new DiffDiagrams(); + + if (argv.include) + (Array.isArray(argv.include) ? argv.include : [argv.include]).forEach((expr: string) => + diffDiagrams.include(expr) + ); + if (argv.exclude) + (Array.isArray(argv.exclude) ? argv.exclude : [argv.exclude]).forEach((expr: string) => + diffDiagrams.exclude(expr) + ); + + let diffDiagram: Diagram | undefined; + try { + diffDiagram = diffDiagrams.diff(baseDiagram, headDiagram); + if (!diffDiagram) continue; + } catch (e) { + console.warn(`Error comparing ${appmapName} ${baseRevision}..${headRevision}: ${e}`); + continue; + } + + const diffText = format(FormatType.Text, diffDiagram, `Compare ${appmapName}`).diagram.trim(); + if (diffSnippets.has(diffText)) continue; + + diffSnippets.add(diffText); + changedAppMaps.set(appmapName, { diffText, diffDiagram }); + } + + const changedAppMapNames = [...changedAppMaps.keys()].sort(); + console.log( + prominentStyle( + `${changedAppMapNames.length} AppMaps have changed between these two code versions.` + ) + ); + console.log(); + + for (let i = 0; i < changedAppMapNames.length; i++) { + const appmapName = changedAppMapNames[i]; + const entry = changedAppMaps.get(appmapName); + assert(entry); + + console.log(prominentStyle(`${appmapName} changed:`)); + console.log(entry.diffText); + console.log(); + let diagramFile = await showDiagram(rl, plantumlJar, entry.diffDiagram); + + await reportDiagram( + appmapName, + reportLines, + argv.outputDir, + 'has changed', + diagramFile, + rl, + plantumlJar, + entry.diffDiagram + ); + } + + if (argv.outputDir) { + await writeFile(join(argv.outputDir, 'report.md'), reportLines.join('\n')); + } + + rl.close(); +}; + +async function ask(rl: readline.Interface, q: string): Promise { + return new Promise((resolve) => { + rl.question(q, resolve); + }); +} + +async function createBaselineAppMaps(rl: readline.Interface, testCommand?: string): Promise { + if (testCommand) { + await executeCommand(testCommand); + return; + } + + console.log( + prominentStyle(`Run the tests for this revision in a separate terminal. For example:`) + ); + console.log(` rails test`); + await waitForEnter(rl); +} + +async function updateAppMaps( + appmapDir: string, + rl: readline.Interface, + touchOutDateTestFiles: boolean, + testCommand?: string +): Promise { + await indexAppMaps(appmapDir); + const testFileNames = await enumerateOutOfDateTestFiles(appmapDir, rl); + if (testFileNames.length === 0) return 0; + + console.log(prominentStyle(`${testFileNames.length} tests are out of date.`)); + + const fileName = makeTempFile('outOfDateTests.txt'); + writeFile(fileName, testFileNames.sort().join('\n')); + if (touchOutDateTestFiles) { + testFileNames.forEach(touch); + } + + if (testCommand) { + await executeCommand(testCommand); + return testFileNames.length; + } + + console.log(`A list of the test cases that are out-of-date has been written to a temp file:`); + console.log(`\t${fileName}`); + if (touchOutDateTestFiles) { + console.log(`Each file has also been "touched".`); + } + + console.log(`Re-run these tests in a separate terminal. For example:`); + console.log(` cat ${fileName} | xargs rails test`); + console.log(); + await waitForEnter(rl); + return testFileNames.length; +} + +async function enumerateOutOfDateTestFiles( + appmapDir: string, + rl: readline.Interface +): Promise { + const depends = new Depends(appmapDir); + const outOfDateAppMapNames = await depends.depends((appmapName: string) => {}); + const testFileNames = new Set(); + if (outOfDateAppMapNames.length > 0) { + const q = queue(async (appMapBaseName: string) => { + const data = await readFile(join(appMapBaseName, 'metadata.json'), 'utf-8'); + const metadata = JSON.parse(data); + const value = metadata['source_location'] as string; + if (value) { + const tokens = value.split(':'); + testFileNames.add(tokens[0]); + } else { + console.warn(warningStyle(`No source_location in ${appMapBaseName}`)); + } + }, 5); + outOfDateAppMapNames.forEach((name) => q.push(name)); + await q.drain(); + } + + return [...testFileNames].sort(); +} + +async function checkout(revisionName: string, revision: string): Promise { + console.log(); + console.log(actionStyle(`Switching to ${revisionName} revision: ${revision}`)); + await executeCommand(`git checkout ${revision}`, false, false); + console.log(); +} + +async function indexAppMaps(appmapDir: string): Promise { + console.log(mutedStyle(`Indexing AppMaps in ${cwd()}`)); + await fingerprintDirectory(appmapDir); +} + +function makeTempFile(fileName: string): string { + if (existsSync('tmp')) return join('tmp', fileName); + + return join(tmpdir(), fileName); +} + +async function buildDiagrams(appmapDir: string): Promise> { + const result = new Map(); + const specOptions = {} as SequenceDiagramOptions; + const appmapFileNames = await promisify(glob)(`${appmapDir}/**/*.appmap.json`); + for (let i = 0; i < appmapFileNames.length; i++) { + const appmapFileName = appmapFileNames[i]; + const appmapData = JSON.parse(await readFile(appmapFileName, 'utf-8')); + const appmap = buildAppMap().source(appmapData).build(); + const specification = Specification.build(appmap, specOptions); + const diagram = buildDiagram(appmapFileName, appmap, specification); + result.set(appmapFileName, diagram); + } + return result; +} + +async function confirm(prompt: string, rl: readline.Interface): Promise { + return (await ask(rl, `${prompt} (y/n) `)) === 'y'; +} + +async function showDiagram( + rl: readline.Interface, + plantumlJar: string, + diagram: Diagram | undefined +): Promise { + if (!diagram) return; + + if (!(await confirm('Open diagram?', rl))) { + return; + } + + const fileNameSVG = await renderDiagram(plantumlJar, diagram); + await executeCommand(`open ${fileNameSVG}`); + await waitForEnter(rl); + return fileNameSVG; +} + +async function stashAll(): Promise { + console.log(`Stashing all modified and untracked files`); + await executeCommand(`git stash --include-untracked`); +} + +async function unstashAll(): Promise { + console.log(`Restoring modified and untracked files`); + try { + await executeCommand(`git stash pop`); + } catch { + console.log(warningStyle(`Command failed. Continuing optimistically...`)); + } +} + +function executeCommand(cmd: string, printStdout = true, printStderr = true): Promise { + console.log(commandStyle(cmd)); + const command = exec(cmd); + const result: string[] = []; + if (command.stdout) { + command.stdout.addListener('data', (data) => { + if (printStdout) process.stdout.write(data); + result.push(data); + }); + } + if (printStderr && command.stderr) command.stderr.pipe(process.stdout); + return new Promise((resolve, reject) => { + command.addListener('exit', (code) => { + if (code === 0) { + resolve(result.join('')); + } else { + reject(new Error(`Command failed with code ${code}`)); + } + }); + }); +} + +function warningStyle(message: string): string { + return chalk.yellow(message); +} +function actionStyle(message: string): string { + return chalk.bold(chalk.green(message)); +} +function prominentStyle(message: string): string { + return chalk.bold(message); +} +function mutedStyle(message: string): string { + return chalk.dim(message); +} +function commandStyle(message: string): string { + return chalk.gray(`$ ${message}`); +} +async function waitForEnter(rl: readline.Interface): Promise { + console.log(prominentStyle(`Press Enter to continue...`)); + await new Promise((resolve) => { + const listener = () => { + rl.removeListener('line', listener); + resolve(); + }; + rl.addListener('line', listener); + }); +} + +async function renderDiagram(plantumlJar: string, diagram: Diagram): Promise { + const diffPlantUML = format(FormatType.PlantUML, diagram, `Diagram`); + const fileNameUML = makeTempFile(`sequence${diffPlantUML.extension}`); + const fileNameSVG = makeTempFile(`sequence.svg`); + await writeFile(fileNameUML, diffPlantUML.diagram); + await executeCommand(`java -jar ${plantumlJar} -tsvg ${fileNameUML}`); + return fileNameSVG; +} + +async function reportDiagram( + appmapName: string, + reportLines: string[], + outputDir: string, + message: string, + diagramFile: string | undefined, + rl: readline.Interface, + plantumlJar: any, + diagram: Diagram +): Promise { + if (outputDir && (await confirm(`Include in the change report?`, rl))) { + if (!diagramFile) diagramFile = await renderDiagram(plantumlJar, diagram); + await mkdir(join(outputDir, 'sequence-diagrams'), { recursive: true }); + const fileDirName = dirname(appmapName); + const fileBaseName = basename(appmapName, '.appmap.json'); + const metadata = JSON.parse( + await readFile(join(fileDirName, fileBaseName, `metadata.json`), 'utf-8') + ); + const sourceLocation = metadata.source_location; + await rename(diagramFile, join(outputDir, 'sequence-diagrams', `${fileBaseName}.svg`)); + reportLines.push(`## [${fileBaseName}](./sequence-diagrams/${fileBaseName}.svg) ${message}.`); + reportLines.push(''); + reportLines.push(`[${sourceLocation}](../${sourceLocation})`); + reportLines.push(''); + reportLines.push(`![${fileBaseName}](./sequence-diagrams/${fileBaseName}.svg)`); + reportLines.push(''); + } +} diff --git a/packages/cli/src/touchFile.ts b/packages/cli/src/touchFile.ts new file mode 100644 index 0000000000..6c7c2c77b1 --- /dev/null +++ b/packages/cli/src/touchFile.ts @@ -0,0 +1,16 @@ +import { close, open, utimes } from 'fs'; + +export async function touch(path: string): Promise { + return new Promise((resolve, reject) => { + const time = new Date(); + utimes(path, time, time, (err) => { + if (err) { + return open(path, 'w', (err, fd) => { + if (err) return reject(err); + close(fd, (err) => (err ? reject(err) : resolve())); + }); + } + resolve(); + }); + }); +} From 34e6f212b66413324b642d67a39e5dee572e2951 Mon Sep 17 00:00:00 2001 From: Kevin Gilpin Date: Thu, 5 Jan 2023 18:48:03 -0500 Subject: [PATCH 11/24] feat: Print revision change report --- packages/cli/src/cmds/describeChange.ts | 358 +++++++++++++++++------- 1 file changed, 252 insertions(+), 106 deletions(-) diff --git a/packages/cli/src/cmds/describeChange.ts b/packages/cli/src/cmds/describeChange.ts index fe33293460..7ecfdf70b5 100644 --- a/packages/cli/src/cmds/describeChange.ts +++ b/packages/cli/src/cmds/describeChange.ts @@ -1,4 +1,4 @@ -import { mkdir, readFile, rename, rm, writeFile } from 'fs/promises'; +import { copyFile, mkdir, readFile, rename, rm, writeFile } from 'fs/promises'; import yargs from 'yargs'; import readline from 'readline'; @@ -10,10 +10,11 @@ import { exists, verbose } from '../utils'; import { exec } from 'child_process'; import { cwd, nextTick, rawListeners } from 'process'; import { queue } from 'async'; -import { basename, dirname, join } from 'path'; +import { basename, isAbsolute, join, relative } from 'path'; import { existsSync } from 'fs'; import { tmpdir } from 'os'; import { + Action, buildDiagram, Diagram, format, @@ -21,10 +22,11 @@ import { SequenceDiagramOptions, Specification, } from '@appland/sequence-diagram'; +import { AppMap, CodeObject, buildAppMap } from '@appland/models'; import { glob } from 'glob'; import { promisify } from 'util'; -import { buildAppMap } from '@appland/models'; import { DiffDiagrams } from '../sequenceDiagramDiff/DiffDiagrams'; +import { readDiagramFile } from './sequenceDiagram/readDiagramFile'; import assert from 'assert'; import { touch } from '../touchFile'; import chalk from 'chalk'; @@ -85,6 +87,74 @@ export const builder = (args: yargs.Argv) => { return args.strict(); }; +enum RevisionName { + Base = 'base', + Head = 'head', + Diff = 'diff', +} + +class AppMapReference { + public sourcePaths = new Set(); + public sourceLocation: string | undefined; + public appmapName: string | undefined; + + constructor(public outputDir: string, public appmapFileName: string) {} + + sequenceDiagramFileName(format: string): string { + return [basename(this.appmapFileName, '.appmap.json'), `sequence.${format}`].join('.'); + } + + sequenceDiagramFilePath( + revisionName: RevisionName, + format: FormatType | string, + includeOutputDir: boolean + ): string { + const tokens = [revisionName, this.sequenceDiagramFileName(format)]; + if (includeOutputDir) tokens.unshift(this.outputDir); + return join(...tokens); + } + + archivedAppMapFilePath(revisionName: RevisionName): string { + return join(this.outputDir, revisionName, basename(this.appmapFileName)); + } + + async loadSequenceDiagramJSON(revisionName: RevisionName): Promise { + return readDiagramFile(this.sequenceDiagramFilePath(revisionName, FormatType.JSON, true)); + } + + async loadSequenceDiagramText(revisionName: RevisionName): Promise { + return await readFile( + this.sequenceDiagramFilePath(revisionName, FormatType.Text, true), + 'utf-8' + ); + } + + async buildSequenceDiagram(): Promise { + const specOptions = { loops: false } as SequenceDiagramOptions; + const appmap = await this.buildAppMap(); + const specification = Specification.build(appmap, specOptions); + return buildDiagram(this.appmapFileName, appmap, specification); + } + + async buildAppMap(): Promise { + const appmapData = JSON.parse(await readFile(this.appmapFileName, 'utf-8')); + const appmap = buildAppMap().source(appmapData).build(); + if (appmap.metadata) { + if (!this.sourceLocation) this.sourceLocation = (appmap.metadata as any).source_location; + if (!this.appmapName) this.appmapName = appmap.metadata.name; + } + const collectSourcePath = (codeObject: CodeObject) => { + const location = codeObject.location; + if (location) { + const path = location.split(':')[0]; + if (!isAbsolute(path)) this.sourcePaths.add(path); + } + }; + appmap.classMap.visit(collectSourcePath); + return appmap; + } +} + export const handler = async (argv: any) => { verbose(argv.verbose); @@ -100,6 +170,7 @@ export const handler = async (argv: any) => { const appmapDir = await locateAppMapDir(); const { touch: touchOutDateTestFiles, plantumlJar, baseCommand, headCommand } = argv; + let { outputDir } = argv; if (!(await exists(plantumlJar))) { yargs.exit( @@ -110,19 +181,6 @@ export const handler = async (argv: any) => { ); } - if (argv.outputDir) { - if (await exists(argv.outputDir)) { - if ( - argv.clobberOutputDir || - !(await confirm(`Remove existing output directory ${argv.outputDir}?`, rl)) - ) { - yargs.exit(1, new Error(`Aborted`)); - } - await rm(argv.outputDir, { recursive: true, force: true }); - } - await mkdir(argv.outputDir, { recursive: true }); - } - let currentBranch = (await executeCommand(`git branch --show-current`)).trim(); if (currentBranch === '') { currentBranch = (await executeCommand(`git show --format=oneline --abbrev-commit`)).split( @@ -140,70 +198,76 @@ export const handler = async (argv: any) => { const baseRevision = argv.baseRevision || 'HEAD~1'; const headRevision = argv.headRevision || (currentBranch || 'HEAD').trim(); - let baseDiagrams: Map; - let headDiagrams: Map; + if (!outputDir) { + outputDir = `revision-report/${sanitizeRevision(baseRevision)}-${sanitizeRevision( + headRevision + )}`; + } - await stashAll(); + if (await exists(outputDir)) { + if ( + argv.clobberOutputDir || + (await confirm(`Remove existing output directory ${outputDir}?`, rl)) + ) { + await rm(outputDir, { recursive: true, force: true }); + // Rapid rm and then mkdir will silently fail in practice. + await new Promise((resolve) => setTimeout(resolve, 100)); + } + } + + // stashAll() + + await mkdir(outputDir, { recursive: true }); + await mkdir(join(outputDir, 'base'), { recursive: true }); + await mkdir(join(outputDir, 'head'), { recursive: true }); + await mkdir(join(outputDir, 'diff'), { recursive: true }); + + const appmapReferences = new Map(); + let baseAppMapFileNames: Set; + let headAppMapFileNames: Set; { await checkout('base', baseRevision); await createBaselineAppMaps(rl, baseCommand); process.stdout.write(`Processing AppMaps...`); - baseDiagrams = await buildDiagrams(appmapDir); - process.stdout.write(`done (${baseDiagrams.size})\n`); + baseAppMapFileNames = new Set([ + ...(await processAppMaps(appmapDir, outputDir, RevisionName.Base, appmapReferences)), + ]); + process.stdout.write(`done (${baseAppMapFileNames.size})\n`); } { await checkout('head', headRevision); await updateAppMaps(appmapDir, rl, touchOutDateTestFiles, headCommand); process.stdout.write(`Processing AppMaps...`); - headDiagrams = await buildDiagrams(appmapDir); - process.stdout.write(`done (${headDiagrams.size})\n`); + headAppMapFileNames = new Set([ + ...(await processAppMaps(appmapDir, outputDir, RevisionName.Head, appmapReferences)), + ]); + process.stdout.write(`done (${headAppMapFileNames.size})\n`); } - await unstashAll(); - const appmapNameSet = new Set(); - [...baseDiagrams.keys()].forEach((file) => appmapNameSet.add(file)); - [...headDiagrams.keys()].forEach((file) => appmapNameSet.add(file)); - const appmapNames = [...appmapNameSet].sort(); + // unstashAll() - const changedAppMaps = new Map(); + const headAppMapFileNameArray = [...headAppMapFileNames].sort(); + const changedAppMaps = new Array(); const diffSnippets = new Set(); const reportLines = new Array(); - for (let i = 0; i < appmapNames.length; i++) { - const appmapName = appmapNames[i]; + for (let i = 0; i < headAppMapFileNameArray.length; i++) { + const appmapFileName = headAppMapFileNameArray[i]; + const appmapReference = appmapReferences.get(appmapFileName); + assert(appmapReference); - const baseDiagram = baseDiagrams.get(appmapName); - const headDiagram = headDiagrams.get(appmapName); - - if (!baseDiagram) { - assert(headDiagram); - - console.log(`${appmapName} is new in the head revision`); - let diagramFile = await showDiagram(rl, plantumlJar, headDiagram); + if (!baseAppMapFileNames.has(appmapFileName)) { + console.log(`${appmapFileName} is new in the head revision`); await reportDiagram( - appmapName, + RevisionName.Head, + outputDir, + baseRevision, + appmapReference, reportLines, - argv.outputDir, 'is new in the head revision', - diagramFile, - rl, - plantumlJar, - headDiagram - ); - continue; - } - if (!headDiagram) { - console.log(`${appmapName} is removed in the head revision`); - let diagramFile = await showDiagram(rl, plantumlJar, baseDiagram); - await reportDiagram( - appmapName, - reportLines, - argv.outputDir, - 'is removed from the head revision', - diagramFile, rl, plantumlJar, - baseDiagram + undefined ); continue; } @@ -220,53 +284,91 @@ export const handler = async (argv: any) => { ); let diffDiagram: Diagram | undefined; + const baseDiagram = await appmapReference.loadSequenceDiagramJSON(RevisionName.Base); + const headDiagram = await appmapReference.loadSequenceDiagramJSON(RevisionName.Head); try { diffDiagram = diffDiagrams.diff(baseDiagram, headDiagram); if (!diffDiagram) continue; } catch (e) { - console.warn(`Error comparing ${appmapName} ${baseRevision}..${headRevision}: ${e}`); + console.warn(`Error comparing ${appmapFileName} ${baseRevision}..${headRevision}: ${e}`); continue; } - const diffText = format(FormatType.Text, diffDiagram, `Compare ${appmapName}`).diagram.trim(); - if (diffSnippets.has(diffText)) continue; + const diffActions = new Set(); + const markDiffActions = (action: Action): void => { + if (action.diffMode) { + let ancestor: Action | undefined = action; + while (ancestor) { + diffActions.add(ancestor); + ancestor = ancestor.parent; + } + } + action.children.forEach(markDiffActions); + }; + + diffDiagram.rootActions.forEach(markDiffActions); + + const filterDiffActions = (actions: Action[]): Action[] => { + const result = actions.filter((action) => diffActions.has(action)); + result.forEach((action) => { + action.children = filterDiffActions(action.children); + }); + return result; + }; + diffDiagram.rootActions = filterDiffActions(diffDiagram.rootActions); + + await writeFile( + appmapReference.sequenceDiagramFilePath(RevisionName.Diff, FormatType.JSON, true), + format(FormatType.JSON, diffDiagram, `Compare ${appmapFileName}`).diagram + ); + + const diffText = format( + FormatType.Text, + diffDiagram, + `Compare ${appmapFileName}` + ).diagram.trim(); + + await writeFile( + appmapReference.sequenceDiagramFilePath(RevisionName.Diff, FormatType.Text, true), + diffText + ); + if (diffSnippets.has(diffText)) continue; diffSnippets.add(diffText); - changedAppMaps.set(appmapName, { diffText, diffDiagram }); + + changedAppMaps.push(appmapReference); } - const changedAppMapNames = [...changedAppMaps.keys()].sort(); console.log( - prominentStyle( - `${changedAppMapNames.length} AppMaps have changed between these two code versions.` - ) + prominentStyle(`${changedAppMaps.length} AppMaps have changed between these two code versions.`) ); console.log(); - for (let i = 0; i < changedAppMapNames.length; i++) { - const appmapName = changedAppMapNames[i]; - const entry = changedAppMaps.get(appmapName); - assert(entry); + for (let i = 0; i < changedAppMaps.length; i++) { + const appmapReference = changedAppMaps[i]; + const diagramText = await appmapReference.loadSequenceDiagramText(RevisionName.Diff); + const diffDiagram = await appmapReference.loadSequenceDiagramJSON(RevisionName.Diff); - console.log(prominentStyle(`${appmapName} changed:`)); - console.log(entry.diffText); + console.log(prominentStyle(`${appmapReference.appmapFileName} changed:`)); + console.log(diagramText); console.log(); - let diagramFile = await showDiagram(rl, plantumlJar, entry.diffDiagram); + let diagramFile = await showDiagram(rl, plantumlJar, diffDiagram); await reportDiagram( - appmapName, + RevisionName.Diff, + outputDir, + baseRevision, + appmapReference, reportLines, - argv.outputDir, 'has changed', - diagramFile, rl, plantumlJar, - entry.diffDiagram + diagramFile ); } - if (argv.outputDir) { - await writeFile(join(argv.outputDir, 'report.md'), reportLines.join('\n')); + if (outputDir) { + await writeFile(join(outputDir, 'report.md'), reportLines.join('\n')); } rl.close(); @@ -371,19 +473,27 @@ function makeTempFile(fileName: string): string { return join(tmpdir(), fileName); } -async function buildDiagrams(appmapDir: string): Promise> { - const result = new Map(); - const specOptions = {} as SequenceDiagramOptions; +async function processAppMaps( + appmapDir: string, + outputDir: string, + revisionName: RevisionName, + appmapReferences: Map +): Promise { const appmapFileNames = await promisify(glob)(`${appmapDir}/**/*.appmap.json`); for (let i = 0; i < appmapFileNames.length; i++) { const appmapFileName = appmapFileNames[i]; - const appmapData = JSON.parse(await readFile(appmapFileName, 'utf-8')); - const appmap = buildAppMap().source(appmapData).build(); - const specification = Specification.build(appmap, specOptions); - const diagram = buildDiagram(appmapFileName, appmap, specification); - result.set(appmapFileName, diagram); + const appmapReference = new AppMapReference(outputDir, appmapFileName); + const diagram = await appmapReference.buildSequenceDiagram(); + await copyFile(appmapFileName, appmapReference.archivedAppMapFilePath(revisionName)); + await writeFile( + appmapReference.sequenceDiagramFilePath(revisionName, FormatType.JSON, true), + format(FormatType.JSON, diagram, appmapFileName).diagram + ); + if (!appmapReferences.get(appmapFileName)) { + appmapReferences.set(appmapFileName, appmapReference); + } } - return result; + return appmapFileNames; } async function confirm(prompt: string, rl: readline.Interface): Promise { @@ -407,6 +517,7 @@ async function showDiagram( return fileNameSVG; } +/* async function stashAll(): Promise { console.log(`Stashing all modified and untracked files`); await executeCommand(`git stash --include-untracked`); @@ -420,6 +531,7 @@ async function unstashAll(): Promise { console.log(warningStyle(`Command failed. Continuing optimistically...`)); } } +*/ function executeCommand(cmd: string, printStdout = true, printStderr = true): Promise { console.log(commandStyle(cmd)); @@ -470,39 +582,73 @@ async function waitForEnter(rl: readline.Interface): Promise { } async function renderDiagram(plantumlJar: string, diagram: Diagram): Promise { - const diffPlantUML = format(FormatType.PlantUML, diagram, `Diagram`); - const fileNameUML = makeTempFile(`sequence${diffPlantUML.extension}`); + const plantUML = format(FormatType.PlantUML, diagram, `Diagram`); + const fileNameUML = makeTempFile(`sequence${plantUML.extension}`); const fileNameSVG = makeTempFile(`sequence.svg`); - await writeFile(fileNameUML, diffPlantUML.diagram); + await writeFile(fileNameUML, plantUML.diagram); await executeCommand(`java -jar ${plantumlJar} -tsvg ${fileNameUML}`); return fileNameSVG; } async function reportDiagram( - appmapName: string, - reportLines: string[], + revisionName: RevisionName, outputDir: string, + baseRevision: string, + appmapReference: AppMapReference, + reportLines: string[], message: string, - diagramFile: string | undefined, rl: readline.Interface, plantumlJar: any, - diagram: Diagram + diagramFile?: string ): Promise { - if (outputDir && (await confirm(`Include in the change report?`, rl))) { + if (await confirm(`Include in the change report?`, rl)) { + const diagram = await appmapReference.loadSequenceDiagramJSON(revisionName); + const diagramText = await appmapReference.loadSequenceDiagramText(revisionName); if (!diagramFile) diagramFile = await renderDiagram(plantumlJar, diagram); - await mkdir(join(outputDir, 'sequence-diagrams'), { recursive: true }); - const fileDirName = dirname(appmapName); - const fileBaseName = basename(appmapName, '.appmap.json'); - const metadata = JSON.parse( - await readFile(join(fileDirName, fileBaseName, `metadata.json`), 'utf-8') + + const appmapName = appmapReference.appmapName || appmapReference.appmapFileName; + + await rename(diagramFile, appmapReference.sequenceDiagramFilePath(revisionName, 'svg', true)); + reportLines.push( + `## [${appmapName}](${appmapReference.sequenceDiagramFilePath( + revisionName, + 'svg', + false + )}) ${message}.` ); - const sourceLocation = metadata.source_location; - await rename(diagramFile, join(outputDir, 'sequence-diagrams', `${fileBaseName}.svg`)); - reportLines.push(`## [${fileBaseName}](./sequence-diagrams/${fileBaseName}.svg) ${message}.`); + + if (appmapReference.sourceLocation) { + const sourcePath = appmapReference.sourceLocation.split(':')[0]; + const fileURL = isAbsolute(sourcePath) + ? relative(outputDir, sourcePath) + : relative(outputDir, join(process.cwd(), sourcePath)); + reportLines.push(''); + reportLines.push(`[${relative(process.cwd(), appmapReference.sourceLocation)}](${fileURL})`); + reportLines.push(''); + } + + if (appmapReference.sourcePaths.size > 0) { + const existingSourcePaths = [...appmapReference.sourcePaths].filter(existsSync).sort(); + const sourceDiff = await executeCommand( + `git diff ${baseRevision} -- ${existingSourcePaths.join(' ')}` + ); + reportLines.push('```'); + reportLines.push(sourceDiff); + reportLines.push('```'); + } + + reportLines.push(''); reportLines.push(''); - reportLines.push(`[${sourceLocation}](../${sourceLocation})`); + reportLines.push('```'); + reportLines.push(diagramText); + reportLines.push('```'); reportLines.push(''); - reportLines.push(`![${fileBaseName}](./sequence-diagrams/${fileBaseName}.svg)`); + reportLines.push( + `![${appmapName}](${appmapReference.sequenceDiagramFilePath(revisionName, 'svg', false)})` + ); reportLines.push(''); } } +function sanitizeRevision(revision: string): string { + return revision.replace(/[^a-zA-Z0-9_]/g, '_'); +} From 5bdeb2507f63c69226e39d699d35401afeadb031 Mon Sep 17 00:00:00 2001 From: Kevin Gilpin Date: Fri, 6 Jan 2023 15:32:33 -0500 Subject: [PATCH 12/24] feat: Remove unused actors When showing a diff diagram, remove actors that are present the source diagrams but unused in the diff. --- packages/cli/src/cmds/describeChange.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/cli/src/cmds/describeChange.ts b/packages/cli/src/cmds/describeChange.ts index 7ecfdf70b5..5f54142891 100644 --- a/packages/cli/src/cmds/describeChange.ts +++ b/packages/cli/src/cmds/describeChange.ts @@ -15,6 +15,7 @@ import { existsSync } from 'fs'; import { tmpdir } from 'os'; import { Action, + actionActors, buildDiagram, Diagram, format, @@ -308,14 +309,17 @@ export const handler = async (argv: any) => { diffDiagram.rootActions.forEach(markDiffActions); + const diffActors = new Set(); const filterDiffActions = (actions: Action[]): Action[] => { const result = actions.filter((action) => diffActions.has(action)); result.forEach((action) => { + actionActors(action).forEach((actor) => (actor ? diffActors.add(actor.name) : undefined)); action.children = filterDiffActions(action.children); }); return result; }; diffDiagram.rootActions = filterDiffActions(diffDiagram.rootActions); + diffDiagram.actors = diffDiagram.actors.filter((actor) => diffActors.has(actor.name)); await writeFile( appmapReference.sequenceDiagramFilePath(RevisionName.Diff, FormatType.JSON, true), From b4b9f02edcbf52074d5a8e53e7e25e655e4ce64e Mon Sep 17 00:00:00 2001 From: Kevin Gilpin Date: Fri, 6 Jan 2023 15:33:00 -0500 Subject: [PATCH 13/24] feat: Re-process change data When a describe-change output directory contains appmaps and sequence diagrams, re-use this data rather than regenerating it. --- packages/cli/src/cmds/describeChange.ts | 95 +++++++++++++++++++------ 1 file changed, 74 insertions(+), 21 deletions(-) diff --git a/packages/cli/src/cmds/describeChange.ts b/packages/cli/src/cmds/describeChange.ts index 5f54142891..75467ffc68 100644 --- a/packages/cli/src/cmds/describeChange.ts +++ b/packages/cli/src/cmds/describeChange.ts @@ -140,6 +140,11 @@ class AppMapReference { async buildAppMap(): Promise { const appmapData = JSON.parse(await readFile(this.appmapFileName, 'utf-8')); const appmap = buildAppMap().source(appmapData).build(); + this.populateMetadata(appmap); + return appmap; + } + + populateMetadata(appmap: AppMap): void { if (appmap.metadata) { if (!this.sourceLocation) this.sourceLocation = (appmap.metadata as any).source_location; if (!this.appmapName) this.appmapName = appmap.metadata.name; @@ -152,7 +157,6 @@ class AppMapReference { } }; appmap.classMap.visit(collectSourcePath); - return appmap; } } @@ -199,6 +203,10 @@ export const handler = async (argv: any) => { const baseRevision = argv.baseRevision || 'HEAD~1'; const headRevision = argv.headRevision || (currentBranch || 'HEAD').trim(); + if (baseRevision === headRevision) { + yargs.exit(1, new Error(`Base and head revisions are the same: ${baseRevision}`)); + } + if (!outputDir) { outputDir = `revision-report/${sanitizeRevision(baseRevision)}-${sanitizeRevision( headRevision @@ -219,32 +227,53 @@ export const handler = async (argv: any) => { // stashAll() await mkdir(outputDir, { recursive: true }); - await mkdir(join(outputDir, 'base'), { recursive: true }); - await mkdir(join(outputDir, 'head'), { recursive: true }); - await mkdir(join(outputDir, 'diff'), { recursive: true }); + await mkdir(join(outputDir, RevisionName.Diff), { recursive: true }); const appmapReferences = new Map(); let baseAppMapFileNames: Set; let headAppMapFileNames: Set; - { - await checkout('base', baseRevision); - await createBaselineAppMaps(rl, baseCommand); - process.stdout.write(`Processing AppMaps...`); - baseAppMapFileNames = new Set([ - ...(await processAppMaps(appmapDir, outputDir, RevisionName.Base, appmapReferences)), + + const processAppMaps = async ( + revisionName: RevisionName, + revision: string, + command?: string + ): Promise> => { + await mkdir(join(outputDir, revisionName), { recursive: true }); + await checkout(revisionName, revision); + await createBaselineAppMaps(rl, command); + process.stdout.write(`Processing AppMaps in ${appmapDir}...`); + const result = new Set([ + ...(await processAppMapDir(appmapDir, outputDir, revisionName, appmapReferences)), ]); - process.stdout.write(`done (${baseAppMapFileNames.size})\n`); - } + process.stdout.write(`done (${result.size})\n`); + return result; + }; - { - await checkout('head', headRevision); - await updateAppMaps(appmapDir, rl, touchOutDateTestFiles, headCommand); - process.stdout.write(`Processing AppMaps...`); - headAppMapFileNames = new Set([ - ...(await processAppMaps(appmapDir, outputDir, RevisionName.Head, appmapReferences)), + const restoreAppMaps = async (revisionName: RevisionName): Promise> => { + process.stdout.write( + `Loading existing AppMap and diagram data from ${join(outputDir, revisionName)}...` + ); + const result = new Set([ + ...(await restoreAppMapDir(outputDir, revisionName, appmapReferences)), ]); - process.stdout.write(`done (${headAppMapFileNames.size})\n`); - } + process.stdout.write(`done (${result.size})\n`); + return result; + }; + + const processOrRestoreAppMaps = async ( + revisionName: RevisionName, + revision: string, + command?: string + ): Promise> => { + if (await exists(join(outputDir, revisionName))) { + return restoreAppMaps(revisionName); + } else { + return processAppMaps(revisionName, revision, command); + } + }; + + baseAppMapFileNames = await processOrRestoreAppMaps(RevisionName.Base, baseRevision, baseCommand); + headAppMapFileNames = await processOrRestoreAppMaps(RevisionName.Head, headRevision, headCommand); // unstashAll() @@ -477,7 +506,7 @@ function makeTempFile(fileName: string): string { return join(tmpdir(), fileName); } -async function processAppMaps( +async function processAppMapDir( appmapDir: string, outputDir: string, revisionName: RevisionName, @@ -493,6 +522,10 @@ async function processAppMaps( appmapReference.sequenceDiagramFilePath(revisionName, FormatType.JSON, true), format(FormatType.JSON, diagram, appmapFileName).diagram ); + await writeFile( + appmapReference.sequenceDiagramFilePath(revisionName, FormatType.Text, true), + format(FormatType.Text, diagram, appmapFileName).diagram + ); if (!appmapReferences.get(appmapFileName)) { appmapReferences.set(appmapFileName, appmapReference); } @@ -500,6 +533,26 @@ async function processAppMaps( return appmapFileNames; } +async function restoreAppMapDir( + outputDir: string, + revisionName: RevisionName, + appmapReferences: Map +): Promise { + const baseDir = join(outputDir, revisionName); + const appmapFileNames = await promisify(glob)(`${baseDir}/*.appmap.json`); + for (let i = 0; i < appmapFileNames.length; i++) { + const appmapFileName = appmapFileNames[i]; + const appmapReference = new AppMapReference(outputDir, relative(baseDir, appmapFileName)); + const appmapData = JSON.parse(await readFile(appmapFileName, 'utf-8')); + const appmap = buildAppMap().source(appmapData).build(); + appmapReference.populateMetadata(appmap); + if (!appmapReferences.get(appmapReference.appmapFileName)) { + appmapReferences.set(appmapReference.appmapFileName, appmapReference); + } + } + return appmapFileNames.map((fileName) => relative(baseDir, fileName)); +} + async function confirm(prompt: string, rl: readline.Interface): Promise { return (await ask(rl, `${prompt} (y/n) `)) === 'y'; } From b06a28f7ddd7fe77ff921dd712d69c4031a3f832 Mon Sep 17 00:00:00 2001 From: Kevin Gilpin Date: Thu, 19 Jan 2023 15:14:02 -0500 Subject: [PATCH 14/24] feat: Improve the describe-change workflow --- packages/cli/src/cmds/describeChange.ts | 98 ++++++++++++++++++++----- 1 file changed, 78 insertions(+), 20 deletions(-) diff --git a/packages/cli/src/cmds/describeChange.ts b/packages/cli/src/cmds/describeChange.ts index 75467ffc68..d2b27a76f4 100644 --- a/packages/cli/src/cmds/describeChange.ts +++ b/packages/cli/src/cmds/describeChange.ts @@ -32,6 +32,8 @@ import assert from 'assert'; import { touch } from '../touchFile'; import chalk from 'chalk'; +export class ValidationError extends Error {} + export const command = 'describe-change'; export const describe = 'Compare base and head revisions'; @@ -95,12 +97,28 @@ enum RevisionName { } class AppMapReference { + // Set of all source files that were used by the app. public sourcePaths = new Set(); + // Name of the test case that produced these AppMaps. public sourceLocation: string | undefined; public appmapName: string | undefined; constructor(public outputDir: string, public appmapFileName: string) {} + async sourceDiff(baseRevision: string): Promise { + if (this.sourcePaths.size === 0) return; + + const existingSourcePaths = [...this.sourcePaths].filter(existsSync).sort(); + return ( + await executeCommand( + `git diff ${baseRevision} -- ${existingSourcePaths.join(' ')}`, + false, + false, + false + ) + ).trim(); + } + sequenceDiagramFileName(format: string): string { return [basename(this.appmapFileName, '.appmap.json'), `sequence.${format}`].join('.'); } @@ -178,11 +196,8 @@ export const handler = async (argv: any) => { let { outputDir } = argv; if (!(await exists(plantumlJar))) { - yargs.exit( - 1, - new Error( - `PlantUML JAR file ${plantumlJar} does not exist. Use --plantuml-jar option to provide the JAR file location.` - ) + throw new ValidationError( + `PlantUML JAR file ${plantumlJar} does not exist. Use --plantuml-jar option to provide the JAR file location.` ); } @@ -204,7 +219,7 @@ export const handler = async (argv: any) => { const headRevision = argv.headRevision || (currentBranch || 'HEAD').trim(); if (baseRevision === headRevision) { - yargs.exit(1, new Error(`Base and head revisions are the same: ${baseRevision}`)); + throw new ValidationError(`Base and head revisions are the same: ${baseRevision}`); } if (!outputDir) { @@ -216,8 +231,16 @@ export const handler = async (argv: any) => { if (await exists(outputDir)) { if ( argv.clobberOutputDir || - (await confirm(`Remove existing output directory ${outputDir}?`, rl)) + !(await confirm(`Use existing data directory ${outputDir}?`, rl)) ) { + if ( + !argv.clobberOutputDir && + !(await confirm(`Delete existing data directory ${outputDir}?`, rl)) + ) { + const msg = `The data directory ${outputDir} exists but you don't want to use it or delete it. Aborting...`; + console.warn(msg); + yargs.exit(1, new Error(msg)); + } await rm(outputDir, { recursive: true, force: true }); // Rapid rm and then mkdir will silently fail in practice. await new Promise((resolve) => setTimeout(resolve, 100)); @@ -288,6 +311,7 @@ export const handler = async (argv: any) => { if (!baseAppMapFileNames.has(appmapFileName)) { console.log(`${appmapFileName} is new in the head revision`); + await printSourceDiff(baseRevision, appmapReference, rl); await reportDiagram( RevisionName.Head, outputDir, @@ -380,13 +404,20 @@ export const handler = async (argv: any) => { for (let i = 0; i < changedAppMaps.length; i++) { const appmapReference = changedAppMaps[i]; const diagramText = await appmapReference.loadSequenceDiagramText(RevisionName.Diff); + + const diagramTextLines = diagramText + .split('\n') + .filter((line) => !/^\d+ times:$/.test(line.trim())); + if (diagramTextLines.length === 0) { + continue; + } + const diffDiagram = await appmapReference.loadSequenceDiagramJSON(RevisionName.Diff); console.log(prominentStyle(`${appmapReference.appmapFileName} changed:`)); - console.log(diagramText); - console.log(); + printDiagramText(diagramTextLines); + await printSourceDiff(baseRevision, appmapReference, rl); let diagramFile = await showDiagram(rl, plantumlJar, diffDiagram); - await reportDiagram( RevisionName.Diff, outputDir, @@ -491,7 +522,7 @@ async function enumerateOutOfDateTestFiles( async function checkout(revisionName: string, revision: string): Promise { console.log(); console.log(actionStyle(`Switching to ${revisionName} revision: ${revision}`)); - await executeCommand(`git checkout ${revision}`, false, false); + await executeCommand(`git checkout ${revision}`, true, false, false); console.log(); } @@ -554,7 +585,11 @@ async function restoreAppMapDir( } async function confirm(prompt: string, rl: readline.Interface): Promise { - return (await ask(rl, `${prompt} (y/n) `)) === 'y'; + let response = ''; + while (!['y', 'n'].includes(response)) { + response = await ask(rl, `${prompt} (y/n) `); + } + return response === 'y'; } async function showDiagram( @@ -570,7 +605,6 @@ async function showDiagram( const fileNameSVG = await renderDiagram(plantumlJar, diagram); await executeCommand(`open ${fileNameSVG}`); - await waitForEnter(rl); return fileNameSVG; } @@ -590,8 +624,13 @@ async function unstashAll(): Promise { } */ -function executeCommand(cmd: string, printStdout = true, printStderr = true): Promise { - console.log(commandStyle(cmd)); +function executeCommand( + cmd: string, + printCommand = true, + printStdout = true, + printStderr = true +): Promise { + if (printCommand) console.log(commandStyle(cmd)); const command = exec(cmd); const result: string[] = []; if (command.stdout) { @@ -684,11 +723,8 @@ async function reportDiagram( reportLines.push(''); } - if (appmapReference.sourcePaths.size > 0) { - const existingSourcePaths = [...appmapReference.sourcePaths].filter(existsSync).sort(); - const sourceDiff = await executeCommand( - `git diff ${baseRevision} -- ${existingSourcePaths.join(' ')}` - ); + const sourceDiff = await appmapReference.sourceDiff(baseRevision); + if (sourceDiff) { reportLines.push('```'); reportLines.push(sourceDiff); reportLines.push('```'); @@ -709,3 +745,25 @@ async function reportDiagram( function sanitizeRevision(revision: string): string { return revision.replace(/[^a-zA-Z0-9_]/g, '_'); } + +async function printSourceDiff( + baseRevision: any, + appmapReference: AppMapReference, + rl: readline.Interface +): Promise { + const sourceDiff = await appmapReference.sourceDiff(baseRevision); + if (sourceDiff) { + if (await confirm(`View ${sourceDiff.split('\n').length} line source diff?`, rl)) { + console.log(); + console.log(sourceDiff); + console.log(); + } + } +} + +function printDiagramText(diagramTextLines: string[]): void { + console.log(`${diagramTextLines.length} line sequence diagram diff:`); + console.log(); + diagramTextLines.forEach((line) => console.log(line)); + console.log(); +} From 3417aac91a61525ae94338b4c777584b71328707 Mon Sep 17 00:00:00 2001 From: Kevin Gilpin Date: Mon, 27 Feb 2023 17:15:30 -0500 Subject: [PATCH 15/24] feat: Generate png of sequence diagram --- packages/cli/src/lib/serveAndOpen.ts | 112 +++++++++------------------ 1 file changed, 36 insertions(+), 76 deletions(-) diff --git a/packages/cli/src/lib/serveAndOpen.ts b/packages/cli/src/lib/serveAndOpen.ts index df86885f82..66e769411d 100644 --- a/packages/cli/src/lib/serveAndOpen.ts +++ b/packages/cli/src/lib/serveAndOpen.ts @@ -1,8 +1,8 @@ import { createReadStream } from 'fs'; -import { createServer } from 'http'; -import { AddressInfo } from 'net'; +import { createServer, ServerResponse } from 'http'; +import { AddressInfo, Server } from 'net'; import open from 'open'; -import { extname, isAbsolute, join } from 'path'; +import { extname, join } from 'path'; import { parse } from 'url'; import { exists, verbose } from '../utils'; import UI from '../cmds/userInteraction'; @@ -12,50 +12,34 @@ function mimeTypeOfName(filename: string): string { { js: 'application/javascript', css: 'text/css', - json: 'application/json', map: 'application/json', }[extname(filename)] || 'application/octet-stream' ); } -export default async function serveAndOpen( - file: string, - resources: Record, - verifyInSubdir: boolean, - onListen: (url: string) => void -) { +export default async function serveAndOpen(file: string, resources: Record) { UI.progress(`Opening ${file}`); const baseDir = join(__dirname, '..', '..', 'built', 'html'); - if (!(await exists(join(baseDir, file)))) throw new Error(`File ${file} does not exist`); const server = createServer(async (req, res) => { - const send404 = () => { - res.writeHead(404); - res.end(); - }; - - const serveStaticFile = (dir: string, fileName: string, contentType?: string) => { - const path = isAbsolute(fileName) ? fileName : join(dir, fileName); - if (verifyInSubdir && !path.startsWith(dir)) return send404(); - - if (!contentType) contentType = mimeTypeOfName(fileName); - + const serveStaticFile = (fileName: string, contentType: string) => { res.writeHead(200, 'OK', { 'Content-Type': contentType }); - const fileStream = createReadStream(path); + const fileStream = createReadStream(fileName); fileStream.pipe(res); fileStream.on('open', function () { if (verbose()) { - console.log(`${path}: 200`); + console.log(`${fileName}: 200`); } res.writeHead(200); }); fileStream.on('error', function (e) { if (verbose()) { - console.log(`${path}: 404 (${e})`); + console.log(`${fileName}: 404`); } - send404(); + res.writeHead(404); + res.end(); }); }; @@ -64,18 +48,27 @@ export default async function serveAndOpen( console.log(req.url); } - const requestUrl = parse(req.url!); + var requestUrl = parse(req.url!); const pathname = requestUrl.pathname; + if (pathname === '/') { - return serveStaticFile(baseDir, file, 'text/html'); - } else if (pathname === '/resource') { - const pathname = requestUrl.query; - if (pathname) serveStaticFile(process.cwd(), decodeURIComponent(pathname)); - else send404(); - } else { - serveStaticFile(baseDir, (pathname || '/').slice(1)); + return serveStaticFile(join(baseDir, file), 'text/html'); + } else if (pathname?.startsWith('/resources/')) { + const resourceName = pathname.substring('/resources/'.length); + const fileName = resources[resourceName]; + if (fileName) { + return serveStaticFile(fileName, 'application/json'); + } + } else if (pathname && !pathname.startsWith('.')) { + const path = join(baseDir, pathname); + return serveStaticFile(path, mimeTypeOfName(path)); } + if (verbose()) { + console.log(`${pathname}: 404`); + } + res.writeHead(404); + res.end(); } catch (e: any) { console.log(e.stack); res.writeHead(500); @@ -84,63 +77,30 @@ export default async function serveAndOpen( }) .listen(0, '127.0.0.1', () => { const port = (server!.address() as AddressInfo).port; - - const params = new URLSearchParams(); - for (const [key, value] of Object.entries(resources)) { - params.append(key, value); - } - const url = new URL(`http://localhost:${port}/?${params.toString()}`); - onListen(url.toString()); + tryOpen(`http://localhost:${port}/`); }) .on('connection', function (socket) { // Don't let the open socket keep the process alive. socket.unref(); }) - .unref(); +// .unref(); } -export async function serveAndOpenSequenceDiagram( - diagramFile: string, - verifyInSubdir: boolean, - onListen: (url: string) => void -): Promise { - return new Promise((resolve) => { - serveAndOpen( - 'sequenceDiagram.html', - { - diagram: diagramFile, - }, - verifyInSubdir, - async (url) => { - onListen(url); - resolve(url); - } - ); +export async function serveAndOpenSequenceDiagram(diagramFile: string) { + return serveAndOpen('sequenceDiagram.html', { + diagram: diagramFile, }); } -export async function serveAndOpenAppMap( - appMapFile: string, - verifyInSubdir: boolean -): Promise { - return new Promise((resolve) => { - serveAndOpen( - 'appmap.html', - { - appmap: appMapFile, - }, - verifyInSubdir, - async (url) => { - await tryOpen(url); - resolve(url); - } - ); +export async function serveAndOpenAppMap(appMapFile: string) { + return serveAndOpen('appmap.html', { + appmap: appMapFile, }); } async function tryOpen(url: string) { const showMessage = () => - UI.warn(`\nWe could not open the browser automatically.\nOpen ${url} to view the content.\n`); + UI.warn(`\nWe could not open the browser automatically.\nOpen ${url} to see the AppMap.\n`); const cp = await open(url); cp.once('error', showMessage); cp.once('exit', (code, signal) => (code || signal) && showMessage()); From f13778c052ef69c563ff7d9adfcfce8a6401cf9c Mon Sep 17 00:00:00 2001 From: Kevin Gilpin Date: Wed, 1 Mar 2023 15:30:42 -0500 Subject: [PATCH 16/24] feat: Render sequence diagram natively as PNG Part 2 --- packages/cli/src/lib/serveAndOpen.ts | 112 ++++++++++++++++++--------- yarn.lock | 94 ++++------------------ 2 files changed, 93 insertions(+), 113 deletions(-) diff --git a/packages/cli/src/lib/serveAndOpen.ts b/packages/cli/src/lib/serveAndOpen.ts index 66e769411d..df86885f82 100644 --- a/packages/cli/src/lib/serveAndOpen.ts +++ b/packages/cli/src/lib/serveAndOpen.ts @@ -1,8 +1,8 @@ import { createReadStream } from 'fs'; -import { createServer, ServerResponse } from 'http'; -import { AddressInfo, Server } from 'net'; +import { createServer } from 'http'; +import { AddressInfo } from 'net'; import open from 'open'; -import { extname, join } from 'path'; +import { extname, isAbsolute, join } from 'path'; import { parse } from 'url'; import { exists, verbose } from '../utils'; import UI from '../cmds/userInteraction'; @@ -12,34 +12,50 @@ function mimeTypeOfName(filename: string): string { { js: 'application/javascript', css: 'text/css', + json: 'application/json', map: 'application/json', }[extname(filename)] || 'application/octet-stream' ); } -export default async function serveAndOpen(file: string, resources: Record) { +export default async function serveAndOpen( + file: string, + resources: Record, + verifyInSubdir: boolean, + onListen: (url: string) => void +) { UI.progress(`Opening ${file}`); const baseDir = join(__dirname, '..', '..', 'built', 'html'); + if (!(await exists(join(baseDir, file)))) throw new Error(`File ${file} does not exist`); const server = createServer(async (req, res) => { - const serveStaticFile = (fileName: string, contentType: string) => { + const send404 = () => { + res.writeHead(404); + res.end(); + }; + + const serveStaticFile = (dir: string, fileName: string, contentType?: string) => { + const path = isAbsolute(fileName) ? fileName : join(dir, fileName); + if (verifyInSubdir && !path.startsWith(dir)) return send404(); + + if (!contentType) contentType = mimeTypeOfName(fileName); + res.writeHead(200, 'OK', { 'Content-Type': contentType }); - const fileStream = createReadStream(fileName); + const fileStream = createReadStream(path); fileStream.pipe(res); fileStream.on('open', function () { if (verbose()) { - console.log(`${fileName}: 200`); + console.log(`${path}: 200`); } res.writeHead(200); }); fileStream.on('error', function (e) { if (verbose()) { - console.log(`${fileName}: 404`); + console.log(`${path}: 404 (${e})`); } - res.writeHead(404); - res.end(); + send404(); }); }; @@ -48,27 +64,18 @@ export default async function serveAndOpen(file: string, resources: Record { const port = (server!.address() as AddressInfo).port; - tryOpen(`http://localhost:${port}/`); + + const params = new URLSearchParams(); + for (const [key, value] of Object.entries(resources)) { + params.append(key, value); + } + const url = new URL(`http://localhost:${port}/?${params.toString()}`); + onListen(url.toString()); }) .on('connection', function (socket) { // Don't let the open socket keep the process alive. socket.unref(); }) -// .unref(); + .unref(); } -export async function serveAndOpenSequenceDiagram(diagramFile: string) { - return serveAndOpen('sequenceDiagram.html', { - diagram: diagramFile, +export async function serveAndOpenSequenceDiagram( + diagramFile: string, + verifyInSubdir: boolean, + onListen: (url: string) => void +): Promise { + return new Promise((resolve) => { + serveAndOpen( + 'sequenceDiagram.html', + { + diagram: diagramFile, + }, + verifyInSubdir, + async (url) => { + onListen(url); + resolve(url); + } + ); }); } -export async function serveAndOpenAppMap(appMapFile: string) { - return serveAndOpen('appmap.html', { - appmap: appMapFile, +export async function serveAndOpenAppMap( + appMapFile: string, + verifyInSubdir: boolean +): Promise { + return new Promise((resolve) => { + serveAndOpen( + 'appmap.html', + { + appmap: appMapFile, + }, + verifyInSubdir, + async (url) => { + await tryOpen(url); + resolve(url); + } + ); }); } async function tryOpen(url: string) { const showMessage = () => - UI.warn(`\nWe could not open the browser automatically.\nOpen ${url} to see the AppMap.\n`); + UI.warn(`\nWe could not open the browser automatically.\nOpen ${url} to view the content.\n`); const cp = await open(url); cp.once('error', showMessage); cp.once('exit', (code, signal) => (code || signal) && showMessage()); diff --git a/yarn.lock b/yarn.lock index edc8c5ef1e..ece3a8e230 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11086,17 +11086,6 @@ __metadata: languageName: node linkType: hard -"chromium-bidi@npm:0.4.4": - version: 0.4.4 - resolution: "chromium-bidi@npm:0.4.4" - dependencies: - mitt: 3.0.0 - peerDependencies: - devtools-protocol: "*" - checksum: 153a276fcce8eb934c9e6a6f8bde5bc37c454d76e82994ce3eca16ab32415a44c553cba3e2ab7ab48ef65f7423f4be35e0ff5b57c7eb378f65fd9b2042df7d03 - languageName: node - linkType: hard - "chromium-bidi@npm:0.4.5": version: 0.4.5 resolution: "chromium-bidi@npm:0.4.5" @@ -12083,18 +12072,6 @@ __metadata: languageName: node linkType: hard -"cosmiconfig@npm:8.0.0": - version: 8.0.0 - resolution: "cosmiconfig@npm:8.0.0" - dependencies: - import-fresh: ^3.2.1 - js-yaml: ^4.1.0 - parse-json: ^5.0.0 - path-type: ^4.0.0 - checksum: ff4cdf89ac1ae52e7520816622c21a9e04380d04b82d653f5139ec581aa4f7f29e096d46770bc76c4a63c225367e88a1dfa233ea791669a35101f5f9b972c7d1 - languageName: node - linkType: hard - "cosmiconfig@npm:8.1.0": version: 8.1.0 resolution: "cosmiconfig@npm:8.1.0" @@ -26519,30 +26496,6 @@ __metadata: languageName: node linkType: hard -"puppeteer-core@npm:19.7.2": - version: 19.7.2 - resolution: "puppeteer-core@npm:19.7.2" - dependencies: - chromium-bidi: 0.4.4 - cross-fetch: 3.1.5 - debug: 4.3.4 - devtools-protocol: 0.0.1094867 - extract-zip: 2.0.1 - https-proxy-agent: 5.0.1 - proxy-from-env: 1.1.0 - rimraf: 3.0.2 - tar-fs: 2.1.1 - unbzip2-stream: 1.4.3 - ws: 8.11.0 - peerDependencies: - typescript: ">= 4.7.4" - peerDependenciesMeta: - typescript: - optional: true - checksum: dafb2f63b74e3270426011ddcd1f4a06e74ca44322137d4240bfaf6090d8450fbf3607336109286861aa96e1f4f2ab9c13cef66c2fc4ec1a690d92d48e8fe773 - languageName: node - linkType: hard - "puppeteer-core@npm:19.7.5": version: 19.7.5 resolution: "puppeteer-core@npm:19.7.5" @@ -26567,20 +26520,7 @@ __metadata: languageName: node linkType: hard -"puppeteer@npm:^19.7.2": - version: 19.7.2 - resolution: "puppeteer@npm:19.7.2" - dependencies: - cosmiconfig: 8.0.0 - https-proxy-agent: 5.0.1 - progress: 2.0.3 - proxy-from-env: 1.1.0 - puppeteer-core: 19.7.2 - checksum: 10a77ee40173e56ddce605c07296b7e538a7e18b0a517e55d570fb2944c7d983d1b19581e31e0f3626e21544c63d20bbac1583307a7a3afbfc80f83fa695f1ae - languageName: node - linkType: hard - -"puppeteer@npm:^19.7.5": +"puppeteer@npm:^19.7.2, puppeteer@npm:^19.7.5": version: 19.7.5 resolution: "puppeteer@npm:19.7.5" dependencies: @@ -27660,7 +27600,7 @@ resolve@1.1.7: languageName: node linkType: hard -"rimraf@npm:*, rimraf@npm:3.0.2, rimraf@npm:^3.0.0, rimraf@npm:^3.0.2": +"rimraf@npm:*, rimraf@npm:^3.0.0, rimraf@npm:^3.0.2": version: 3.0.2 resolution: "rimraf@npm:3.0.2" dependencies: @@ -32704,21 +32644,6 @@ typescript@~4.4.3: languageName: node linkType: hard -"ws@npm:8.11.0, ws@npm:^8.0.0": - version: 8.11.0 - resolution: "ws@npm:8.11.0" - peerDependencies: - bufferutil: ^4.0.1 - utf-8-validate: ^5.0.2 - peerDependenciesMeta: - bufferutil: - optional: true - utf-8-validate: - optional: true - checksum: 316b33aba32f317cd217df66dbfc5b281a2f09ff36815de222bc859e3424d83766d9eb2bd4d667de658b6ab7be151f258318fb1da812416b30be13103e5b5c67 - languageName: node - linkType: hard - "ws@npm:8.12.1": version: 8.12.1 resolution: "ws@npm:8.12.1" @@ -32767,6 +32692,21 @@ typescript@~4.4.3: languageName: node linkType: hard +"ws@npm:^8.0.0": + version: 8.13.0 + resolution: "ws@npm:8.13.0" + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: ">=5.0.2" + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + checksum: 53e991bbf928faf5dc6efac9b8eb9ab6497c69feeb94f963d648b7a3530a720b19ec2e0ec037344257e05a4f35bd9ad04d9de6f289615ffb133282031b18c61c + languageName: node + linkType: hard + "ws@npm:^8.2.3": version: 8.5.0 resolution: "ws@npm:8.5.0" From 6b011775e46e22016819ff7ef0c8d77a86ce0880 Mon Sep 17 00:00:00 2001 From: Kevin Gilpin Date: Fri, 3 Mar 2023 14:42:55 -0500 Subject: [PATCH 17/24] wip: Remove report prompts; link to live diagrams rather than images --- packages/cli/package.json | 1 + packages/cli/src/cmds/describeChange.ts | 396 +++++------------- .../cli/src/describeChange/AppMapReference.ts | 98 +++++ .../cli/src/describeChange/RevisionName.ts | 5 + packages/cli/src/html/appmap.js | 3 +- packages/cli/src/html/sequenceDiagram.js | 62 ++- packages/cli/src/lib/serveAndOpen.ts | 5 +- yarn.lock | 8 + 8 files changed, 273 insertions(+), 305 deletions(-) create mode 100644 packages/cli/src/describeChange/AppMapReference.ts create mode 100644 packages/cli/src/describeChange/RevisionName.ts diff --git a/packages/cli/package.json b/packages/cli/package.json index 8d48178aaa..64df351d3f 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -79,6 +79,7 @@ "@appland/sequence-diagram": "workspace:^1", "@sidvind/better-ajv-errors": "^0.9.1", "@types/moo": "^0.5.5", + "@zip.js/zip.js": "^2.6.75", "JSONStream": "^1.3.5", "ajv": "^8.6.3", "applicationinsights": "^2.1.4", diff --git a/packages/cli/src/cmds/describeChange.ts b/packages/cli/src/cmds/describeChange.ts index d2b27a76f4..5f93833c55 100644 --- a/packages/cli/src/cmds/describeChange.ts +++ b/packages/cli/src/cmds/describeChange.ts @@ -4,33 +4,20 @@ import readline from 'readline'; import { handleWorkingDirectory } from '../lib/handleWorkingDirectory'; import { locateAppMapDir } from '../lib/locateAppMapDir'; -import { fingerprintDirectory } from '../fingerprint'; -import Depends from '../depends'; import { exists, verbose } from '../utils'; import { exec } from 'child_process'; -import { cwd, nextTick, rawListeners } from 'process'; -import { queue } from 'async'; -import { basename, isAbsolute, join, relative } from 'path'; +import { isAbsolute, join, relative } from 'path'; import { existsSync } from 'fs'; import { tmpdir } from 'os'; -import { - Action, - actionActors, - buildDiagram, - Diagram, - format, - FormatType, - SequenceDiagramOptions, - Specification, -} from '@appland/sequence-diagram'; -import { AppMap, CodeObject, buildAppMap } from '@appland/models'; +import { Action, actionActors, Diagram, format, FormatType } from '@appland/sequence-diagram'; +import { buildAppMap } from '@appland/models'; import { glob } from 'glob'; import { promisify } from 'util'; import { DiffDiagrams } from '../sequenceDiagramDiff/DiffDiagrams'; -import { readDiagramFile } from './sequenceDiagram/readDiagramFile'; import assert from 'assert'; -import { touch } from '../touchFile'; import chalk from 'chalk'; +import { AppMapReference } from '../describeChange/AppMapReference'; +import { RevisionName } from '../describeChange/RevisionName'; export class ValidationError extends Error {} @@ -69,17 +56,6 @@ export const builder = (args: yargs.Argv) => { describe: 'command to use to run the head tests', }); - args.option('touch', { - describe: 'touch out-of-date test files when switching to a new revision', - type: 'boolean', - default: false, - }); - - args.option('plantuml-jar', { - describe: 'location of PlantUML JAR file', - default: 'plantuml.jar', - }); - args.option('include', { describe: 'code objects to include in the comparison (inclusive of descendants)', }); @@ -90,94 +66,6 @@ export const builder = (args: yargs.Argv) => { return args.strict(); }; -enum RevisionName { - Base = 'base', - Head = 'head', - Diff = 'diff', -} - -class AppMapReference { - // Set of all source files that were used by the app. - public sourcePaths = new Set(); - // Name of the test case that produced these AppMaps. - public sourceLocation: string | undefined; - public appmapName: string | undefined; - - constructor(public outputDir: string, public appmapFileName: string) {} - - async sourceDiff(baseRevision: string): Promise { - if (this.sourcePaths.size === 0) return; - - const existingSourcePaths = [...this.sourcePaths].filter(existsSync).sort(); - return ( - await executeCommand( - `git diff ${baseRevision} -- ${existingSourcePaths.join(' ')}`, - false, - false, - false - ) - ).trim(); - } - - sequenceDiagramFileName(format: string): string { - return [basename(this.appmapFileName, '.appmap.json'), `sequence.${format}`].join('.'); - } - - sequenceDiagramFilePath( - revisionName: RevisionName, - format: FormatType | string, - includeOutputDir: boolean - ): string { - const tokens = [revisionName, this.sequenceDiagramFileName(format)]; - if (includeOutputDir) tokens.unshift(this.outputDir); - return join(...tokens); - } - - archivedAppMapFilePath(revisionName: RevisionName): string { - return join(this.outputDir, revisionName, basename(this.appmapFileName)); - } - - async loadSequenceDiagramJSON(revisionName: RevisionName): Promise { - return readDiagramFile(this.sequenceDiagramFilePath(revisionName, FormatType.JSON, true)); - } - - async loadSequenceDiagramText(revisionName: RevisionName): Promise { - return await readFile( - this.sequenceDiagramFilePath(revisionName, FormatType.Text, true), - 'utf-8' - ); - } - - async buildSequenceDiagram(): Promise { - const specOptions = { loops: false } as SequenceDiagramOptions; - const appmap = await this.buildAppMap(); - const specification = Specification.build(appmap, specOptions); - return buildDiagram(this.appmapFileName, appmap, specification); - } - - async buildAppMap(): Promise { - const appmapData = JSON.parse(await readFile(this.appmapFileName, 'utf-8')); - const appmap = buildAppMap().source(appmapData).build(); - this.populateMetadata(appmap); - return appmap; - } - - populateMetadata(appmap: AppMap): void { - if (appmap.metadata) { - if (!this.sourceLocation) this.sourceLocation = (appmap.metadata as any).source_location; - if (!this.appmapName) this.appmapName = appmap.metadata.name; - } - const collectSourcePath = (codeObject: CodeObject) => { - const location = codeObject.location; - if (location) { - const path = location.split(':')[0]; - if (!isAbsolute(path)) this.sourcePaths.add(path); - } - }; - appmap.classMap.visit(collectSourcePath); - } -} - export const handler = async (argv: any) => { verbose(argv.verbose); @@ -192,15 +80,9 @@ export const handler = async (argv: any) => { handleWorkingDirectory(argv.directory); const appmapDir = await locateAppMapDir(); - const { touch: touchOutDateTestFiles, plantumlJar, baseCommand, headCommand } = argv; + const { baseCommand, headCommand } = argv; let { outputDir } = argv; - if (!(await exists(plantumlJar))) { - throw new ValidationError( - `PlantUML JAR file ${plantumlJar} does not exist. Use --plantuml-jar option to provide the JAR file location.` - ); - } - let currentBranch = (await executeCommand(`git branch --show-current`)).trim(); if (currentBranch === '') { currentBranch = (await executeCommand(`git show --format=oneline --abbrev-commit`)).split( @@ -316,12 +198,9 @@ export const handler = async (argv: any) => { RevisionName.Head, outputDir, baseRevision, + headRevision, appmapReference, - reportLines, - 'is new in the head revision', - rl, - plantumlJar, - undefined + reportLines ); continue; } @@ -391,6 +270,7 @@ export const handler = async (argv: any) => { ); if (diffSnippets.has(diffText)) continue; + diffSnippets.add(diffText); changedAppMaps.push(appmapReference); @@ -412,27 +292,40 @@ export const handler = async (argv: any) => { continue; } - const diffDiagram = await appmapReference.loadSequenceDiagramJSON(RevisionName.Diff); - - console.log(prominentStyle(`${appmapReference.appmapFileName} changed:`)); - printDiagramText(diagramTextLines); - await printSourceDiff(baseRevision, appmapReference, rl); - let diagramFile = await showDiagram(rl, plantumlJar, diffDiagram); await reportDiagram( RevisionName.Diff, outputDir, baseRevision, + headRevision, appmapReference, - reportLines, - 'has changed', - rl, - plantumlJar, - diagramFile + reportLines ); } if (outputDir) { - await writeFile(join(outputDir, 'report.md'), reportLines.join('\n')); + const report = ` + AppMap code change report + + + ${reportLines.join('\n')} +`; + const diagramHTML = ` + Sequence diagram + + +
+ +`; + const appmapHTML = ` + AppMap + + +
+ +`; + await writeFile(join(outputDir, 'report.html'), report); + await writeFile(join(outputDir, 'diagram.html'), diagramHTML); + await writeFile(join(outputDir, 'appmap.html'), appmapHTML); } rl.close(); @@ -451,72 +344,11 @@ async function createBaselineAppMaps(rl: readline.Interface, testCommand?: strin } console.log( - prominentStyle(`Run the tests for this revision in a separate terminal. For example:`) + prominentStyle( + `Generate the AppMaps for this revision in a separate terminal - for example, by running the tests.` + ) ); - console.log(` rails test`); - await waitForEnter(rl); -} - -async function updateAppMaps( - appmapDir: string, - rl: readline.Interface, - touchOutDateTestFiles: boolean, - testCommand?: string -): Promise { - await indexAppMaps(appmapDir); - const testFileNames = await enumerateOutOfDateTestFiles(appmapDir, rl); - if (testFileNames.length === 0) return 0; - - console.log(prominentStyle(`${testFileNames.length} tests are out of date.`)); - - const fileName = makeTempFile('outOfDateTests.txt'); - writeFile(fileName, testFileNames.sort().join('\n')); - if (touchOutDateTestFiles) { - testFileNames.forEach(touch); - } - - if (testCommand) { - await executeCommand(testCommand); - return testFileNames.length; - } - - console.log(`A list of the test cases that are out-of-date has been written to a temp file:`); - console.log(`\t${fileName}`); - if (touchOutDateTestFiles) { - console.log(`Each file has also been "touched".`); - } - - console.log(`Re-run these tests in a separate terminal. For example:`); - console.log(` cat ${fileName} | xargs rails test`); - console.log(); - await waitForEnter(rl); - return testFileNames.length; -} - -async function enumerateOutOfDateTestFiles( - appmapDir: string, - rl: readline.Interface -): Promise { - const depends = new Depends(appmapDir); - const outOfDateAppMapNames = await depends.depends((appmapName: string) => {}); - const testFileNames = new Set(); - if (outOfDateAppMapNames.length > 0) { - const q = queue(async (appMapBaseName: string) => { - const data = await readFile(join(appMapBaseName, 'metadata.json'), 'utf-8'); - const metadata = JSON.parse(data); - const value = metadata['source_location'] as string; - if (value) { - const tokens = value.split(':'); - testFileNames.add(tokens[0]); - } else { - console.warn(warningStyle(`No source_location in ${appMapBaseName}`)); - } - }, 5); - outOfDateAppMapNames.forEach((name) => q.push(name)); - await q.drain(); - } - - return [...testFileNames].sort(); + await waitForEnter(rl, `Press Enter when the AppMaps are ready`); } async function checkout(revisionName: string, revision: string): Promise { @@ -526,17 +358,6 @@ async function checkout(revisionName: string, revision: string): Promise { console.log(); } -async function indexAppMaps(appmapDir: string): Promise { - console.log(mutedStyle(`Indexing AppMaps in ${cwd()}`)); - await fingerprintDirectory(appmapDir); -} - -function makeTempFile(fileName: string): string { - if (existsSync('tmp')) return join('tmp', fileName); - - return join(tmpdir(), fileName); -} - async function processAppMapDir( appmapDir: string, outputDir: string, @@ -548,7 +369,7 @@ async function processAppMapDir( const appmapFileName = appmapFileNames[i]; const appmapReference = new AppMapReference(outputDir, appmapFileName); const diagram = await appmapReference.buildSequenceDiagram(); - await copyFile(appmapFileName, appmapReference.archivedAppMapFilePath(revisionName)); + await copyFile(appmapFileName, appmapReference.archivedAppMapFilePath(revisionName, true)); await writeFile( appmapReference.sequenceDiagramFilePath(revisionName, FormatType.JSON, true), format(FormatType.JSON, diagram, appmapFileName).diagram @@ -592,39 +413,7 @@ async function confirm(prompt: string, rl: readline.Interface): Promise return response === 'y'; } -async function showDiagram( - rl: readline.Interface, - plantumlJar: string, - diagram: Diagram | undefined -): Promise { - if (!diagram) return; - - if (!(await confirm('Open diagram?', rl))) { - return; - } - - const fileNameSVG = await renderDiagram(plantumlJar, diagram); - await executeCommand(`open ${fileNameSVG}`); - return fileNameSVG; -} - -/* -async function stashAll(): Promise { - console.log(`Stashing all modified and untracked files`); - await executeCommand(`git stash --include-untracked`); -} - -async function unstashAll(): Promise { - console.log(`Restoring modified and untracked files`); - try { - await executeCommand(`git stash pop`); - } catch { - console.log(warningStyle(`Command failed. Continuing optimistically...`)); - } -} -*/ - -function executeCommand( +export function executeCommand( cmd: string, printCommand = true, printStdout = true, @@ -651,9 +440,6 @@ function executeCommand( }); } -function warningStyle(message: string): string { - return chalk.yellow(message); -} function actionStyle(message: string): string { return chalk.bold(chalk.green(message)); } @@ -666,8 +452,11 @@ function mutedStyle(message: string): string { function commandStyle(message: string): string { return chalk.gray(`$ ${message}`); } -async function waitForEnter(rl: readline.Interface): Promise { - console.log(prominentStyle(`Press Enter to continue...`)); +async function waitForEnter( + rl: readline.Interface, + prompt = 'Press Enter to continue' +): Promise { + console.log(prominentStyle(`${prompt}...`)); await new Promise((resolve) => { const listener = () => { rl.removeListener('line', listener); @@ -677,41 +466,43 @@ async function waitForEnter(rl: readline.Interface): Promise { }); } -async function renderDiagram(plantumlJar: string, diagram: Diagram): Promise { - const plantUML = format(FormatType.PlantUML, diagram, `Diagram`); - const fileNameUML = makeTempFile(`sequence${plantUML.extension}`); - const fileNameSVG = makeTempFile(`sequence.svg`); - await writeFile(fileNameUML, plantUML.diagram); - await executeCommand(`java -jar ${plantumlJar} -tsvg ${fileNameUML}`); - return fileNameSVG; -} - async function reportDiagram( revisionName: RevisionName, outputDir: string, baseRevision: string, + headRevision: string, appmapReference: AppMapReference, - reportLines: string[], - message: string, - rl: readline.Interface, - plantumlJar: any, - diagramFile?: string + reportLines: string[] ): Promise { - if (await confirm(`Include in the change report?`, rl)) { - const diagram = await appmapReference.loadSequenceDiagramJSON(revisionName); + if (true /* await confirm(`Include in the change report?`, rl)) */) { const diagramText = await appmapReference.loadSequenceDiagramText(revisionName); - if (!diagramFile) diagramFile = await renderDiagram(plantumlJar, diagram); const appmapName = appmapReference.appmapName || appmapReference.appmapFileName; - await rename(diagramFile, appmapReference.sequenceDiagramFilePath(revisionName, 'svg', true)); - reportLines.push( - `## [${appmapName}](${appmapReference.sequenceDiagramFilePath( - revisionName, - 'svg', - false - )}) ${message}.` - ); + if (revisionName === RevisionName.Head) { + reportLines.push( + `

+ ${appmapName} is new in ${headRevision} +

` + ); + } else { + assert( + revisionName === RevisionName.Diff, + `Expecting revisionName to be '${RevisionName.Diff}', got '${revisionName}'` + ); + reportLines.push( + `

+ ${appmapName} has changed +

` + ); + } if (appmapReference.sourceLocation) { const sourcePath = appmapReference.sourceLocation.split(':')[0]; @@ -719,29 +510,38 @@ async function reportDiagram( ? relative(outputDir, sourcePath) : relative(outputDir, join(process.cwd(), sourcePath)); reportLines.push(''); - reportLines.push(`[${relative(process.cwd(), appmapReference.sourceLocation)}](${fileURL})`); + reportLines.push( + `${relative(process.cwd(), appmapReference.sourceLocation)}` + ); reportLines.push(''); } const sourceDiff = await appmapReference.sourceDiff(baseRevision); if (sourceDiff) { - reportLines.push('```'); + reportLines.push('
'); + reportLines.push(''); + reportLines.push('
');
       reportLines.push(sourceDiff);
-      reportLines.push('```');
+      reportLines.push('
'); + reportLines.push(''); + reportLines.push('
'); } - reportLines.push(''); - reportLines.push(''); - reportLines.push('```'); + reportLines.push('
'); + reportLines.push('
'); + reportLines.push(''); + reportLines.push('
');
     reportLines.push(diagramText);
-    reportLines.push('```');
-    reportLines.push('');
-    reportLines.push(
-      `![${appmapName}](${appmapReference.sequenceDiagramFilePath(revisionName, 'svg', false)})`
-    );
-    reportLines.push('');
+    reportLines.push('
'); + reportLines.push(''); + // reportLines.push(''); + // reportLines.push( + // `![${appmapName}](${appmapReference.sequenceDiagramFilePath(revisionName, 'svg', false)})` + // ); + // reportLines.push(''); } } + function sanitizeRevision(revision: string): string { return revision.replace(/[^a-zA-Z0-9_]/g, '_'); } @@ -753,11 +553,11 @@ async function printSourceDiff( ): Promise { const sourceDiff = await appmapReference.sourceDiff(baseRevision); if (sourceDiff) { - if (await confirm(`View ${sourceDiff.split('\n').length} line source diff?`, rl)) { - console.log(); - console.log(sourceDiff); - console.log(); - } + // if (await confirm(`View ${sourceDiff.split('\n').length} line source diff?`, rl)) { + console.log(); + console.log(sourceDiff); + console.log(); + // } } } diff --git a/packages/cli/src/describeChange/AppMapReference.ts b/packages/cli/src/describeChange/AppMapReference.ts new file mode 100644 index 0000000000..99e6e4a27f --- /dev/null +++ b/packages/cli/src/describeChange/AppMapReference.ts @@ -0,0 +1,98 @@ +import { readFile } from 'fs/promises'; +import { basename, isAbsolute, join } from 'path'; +import { existsSync } from 'fs'; +import { + buildDiagram, + Diagram, + FormatType, + SequenceDiagramOptions, + Specification, +} from '@appland/sequence-diagram'; +import { AppMap, CodeObject, buildAppMap } from '@appland/models'; +import { readDiagramFile } from '../cmds/sequenceDiagram/readDiagramFile'; +import { executeCommand } from '../cmds/describeChange'; +import { RevisionName } from './RevisionName'; + +export class AppMapReference { + // Set of all source files that were used by the app. + public sourcePaths = new Set(); + // Name of the test case that produced these AppMaps. + public sourceLocation: string | undefined; + public appmapName: string | undefined; + + constructor(public outputDir: string, public appmapFileName: string) {} + + async sourceDiff(baseRevision: string): Promise { + if (this.sourcePaths.size === 0) return; + + const existingSourcePaths = [...this.sourcePaths].filter(existsSync).sort(); + return ( + await executeCommand( + `git diff ${baseRevision} -- ${existingSourcePaths.join(' ')}`, + false, + false, + false + ) + ).trim(); + } + + sequenceDiagramFileName(format: string): string { + return [basename(this.appmapFileName, '.appmap.json'), `sequence.${format}`].join('.'); + } + + sequenceDiagramFilePath( + revisionName: RevisionName, + format: FormatType | string, + includeOutputDir: boolean + ): string { + const tokens = [revisionName, this.sequenceDiagramFileName(format)]; + if (includeOutputDir) tokens.unshift(this.outputDir); + return join(...tokens); + } + + archivedAppMapFilePath(revisionName: RevisionName, includeOutputDir: boolean): string { + const tokens = [revisionName, basename(this.appmapFileName)]; + if (includeOutputDir) tokens.unshift(this.outputDir); + return join(...tokens); + } + + async loadSequenceDiagramJSON(revisionName: RevisionName): Promise { + return readDiagramFile(this.sequenceDiagramFilePath(revisionName, FormatType.JSON, true)); + } + + async loadSequenceDiagramText(revisionName: RevisionName): Promise { + return await readFile( + this.sequenceDiagramFilePath(revisionName, FormatType.Text, true), + 'utf-8' + ); + } + + async buildSequenceDiagram(): Promise { + const specOptions = { loops: false } as SequenceDiagramOptions; + const appmap = await this.buildAppMap(); + const specification = Specification.build(appmap, specOptions); + return buildDiagram(this.appmapFileName, appmap, specification); + } + + async buildAppMap(): Promise { + const appmapData = JSON.parse(await readFile(this.appmapFileName, 'utf-8')); + const appmap = buildAppMap().source(appmapData).build(); + this.populateMetadata(appmap); + return appmap; + } + + populateMetadata(appmap: AppMap): void { + if (appmap.metadata) { + if (!this.sourceLocation) this.sourceLocation = (appmap.metadata as any).source_location; + if (!this.appmapName) this.appmapName = appmap.metadata.name; + } + const collectSourcePath = (codeObject: CodeObject) => { + const location = codeObject.location; + if (location) { + const path = location.split(':')[0]; + if (!isAbsolute(path)) this.sourcePaths.add(path); + } + }; + appmap.classMap.visit(collectSourcePath); + } +} diff --git a/packages/cli/src/describeChange/RevisionName.ts b/packages/cli/src/describeChange/RevisionName.ts new file mode 100644 index 0000000000..9d3005a251 --- /dev/null +++ b/packages/cli/src/describeChange/RevisionName.ts @@ -0,0 +1,5 @@ +export enum RevisionName { + Base = 'base', + Head = 'head', + Diff = 'diff', +} diff --git a/packages/cli/src/html/appmap.js b/packages/cli/src/html/appmap.js index 58add8f659..c8fcdfd1a4 100644 --- a/packages/cli/src/html/appmap.js +++ b/packages/cli/src/html/appmap.js @@ -20,7 +20,8 @@ async function initializeApp() { async mounted() { const params = new URL(document.location).searchParams; const appmap = params.get('appmap'); - const res = await fetch(`/resource?${encodeURIComponent(appmap)}`); + const res = await fetch(appmap); + const { ui } = this.$refs; ui.loadData((await res.json()) || {}); diff --git a/packages/cli/src/html/sequenceDiagram.js b/packages/cli/src/html/sequenceDiagram.js index 39585b6d6a..acd99a1582 100644 --- a/packages/cli/src/html/sequenceDiagram.js +++ b/packages/cli/src/html/sequenceDiagram.js @@ -1,5 +1,6 @@ import Vue from 'vue'; import Vuex from 'vuex'; +import { BlobReader, ZipReader } from '@zip.js/zip.js'; import plugin, { VDiagramSequence } from '@appland/components'; import '@appland/diagrams/dist/style.css'; @@ -7,6 +8,20 @@ import '@appland/diagrams/dist/style.css'; Vue.use(Vuex); Vue.use(plugin); +// https://stackoverflow.com/a/18650249/953770 +async function blobToBase64(blob) { + return new Promise((resolve, _) => { + const reader = new FileReader(); + reader.onloadend = () => resolve(reader.result); + reader.readAsDataURL(blob); + }); +} + +// https://stackoverflow.com/a/36183085/953770 +async function base64UrltoBlob(base64Url) { + return fetch(base64Url).then((res) => res.blob()); +} + async function initializeApp() { return new Vue({ el: '#app', @@ -19,11 +34,52 @@ async function initializeApp() { }), async mounted() { const params = new URL(document.location).searchParams; - const diagram = params.get('diagram'); - const res = await fetch(`/resource?${encodeURIComponent(diagram)}`); + const diagramUrl = params.get('diagram'); + let diagramData; + if (diagramUrl.endsWith('.zip')) { + const resourceId = params.get('resourceId'); + if (!resourceId) throw new Error(`resourceId is required with diagram resource ZIP file`); + + const localStorageKey = ['appmap', 'diagram', diagramUrl].join('-'); + let diagramDataEncoded; + if (!['false', 'no'].includes(params.get('cache'))) + diagramDataEncoded = localStorage.getItem(localStorageKey); + if (diagramDataEncoded) { + diagramData = await base64UrltoBlob(diagramDataEncoded); + } else { + diagramData = await (await fetch(diagramUrl)).blob(); + diagramDataEncoded = await blobToBase64(diagramData); + localStorage.setItem(localStorageKey, diagramDataEncoded); + } + + // Creates a BlobReader object used to read `zipFileBlob`. + const zipFileReader = new BlobReader(diagramData); + // Creates a TransformStream object, the content of the first entry in the zip + // will be written in the `writable` property. + const dataStream = new TransformStream(); + // Creates a Promise object resolved to the content of the first entry returned + // as text from `helloWorldStream.readable`. + const dataStreamPromise = new Response(dataStream.readable).text(); + + // Creates a ZipReader object reading the zip content via `zipFileReader`, + // retrieves metadata (name, dates, etc.) of the first entry, retrieves its + // content into `helloWorldStream.writable`, and closes the reader. + const zipReader = new ZipReader(zipFileReader); + const entries = await zipReader.getEntries(); + const requestedEntry = entries.find((entry) => entry.filename === resourceId); + if (!requestedEntry) throw new Error(`Resource ${resourceId} not found in ${diagramUrl}`); + + await requestedEntry.getData(dataStream.writable); + await zipReader.close(); + const diagramDataRaw = await dataStreamPromise; + diagramData = JSON.parse(diagramDataRaw); + } else { + diagramData = (await fetch(diagramUrl)).json(); + } + const { ui } = this.$refs; - ui.loadData((await res.json()) || {}); + ui.loadData((await diagramData) || {}); }, }); } diff --git a/packages/cli/src/lib/serveAndOpen.ts b/packages/cli/src/lib/serveAndOpen.ts index df86885f82..dc83db15a4 100644 --- a/packages/cli/src/lib/serveAndOpen.ts +++ b/packages/cli/src/lib/serveAndOpen.ts @@ -75,7 +75,6 @@ export default async function serveAndOpen( } else { serveStaticFile(baseDir, (pathname || '/').slice(1)); } - } catch (e: any) { console.log(e.stack); res.writeHead(500); @@ -108,7 +107,7 @@ export async function serveAndOpenSequenceDiagram( serveAndOpen( 'sequenceDiagram.html', { - diagram: diagramFile, + diagram: `/resource?${encodeURIComponent(diagramFile)}`, }, verifyInSubdir, async (url) => { @@ -127,7 +126,7 @@ export async function serveAndOpenAppMap( serveAndOpen( 'appmap.html', { - appmap: appMapFile, + appmap: `/resource?${encodeURIComponent(appMapFile)}`, }, verifyInSubdir, async (url) => { diff --git a/yarn.lock b/yarn.lock index ece3a8e230..d2f49f28e8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -108,6 +108,7 @@ __metadata: "@types/tmp": ^0.2.3 "@types/validator": ^13.7.10 "@types/w3c-xmlserializer": ^2.0.2 + "@zip.js/zip.js": ^2.6.75 JSONStream: ^1.3.5 ajv: ^8.6.3 applicationinsights: ^2.1.4 @@ -8432,6 +8433,13 @@ __metadata: languageName: node linkType: hard +"@zip.js/zip.js@npm:^2.6.75": + version: 2.6.75 + resolution: "@zip.js/zip.js@npm:2.6.75" + checksum: bf152ca599350be3cc87bcfa06467aa4209226c7772243ba57f41ef0eb71e2db2848e9df1c37d7b1c4a717158db33e76b0485ccae1a9cd3b20d03f5a772627cb + languageName: node + linkType: hard + "JSONStream@npm:^1.0.4, JSONStream@npm:^1.3.5": version: 1.3.5 resolution: "JSONStream@npm:1.3.5" From 750269a5c5eedf2a77644cbf5f295ac192fe608a Mon Sep 17 00:00:00 2001 From: Kevin Gilpin Date: Fri, 3 Mar 2023 16:38:16 -0500 Subject: [PATCH 18/24] wip: Organize changes by API operation --- packages/cli/src/cmds/describeChange.ts | 80 +++++++------ packages/cli/src/cmds/openapi.ts | 94 +-------------- .../cli/src/describeChange/AppMapReference.ts | 41 ++++++- .../src/describeChange/OperationReference.ts | 8 ++ .../src/describeChange/buildChangeReport.ts | 110 ++++++++++++++++++ .../cli/src/describeChange/executeCommand.ts | 29 +++++ packages/cli/src/describeChange/types.ts | 23 ++++ packages/cli/src/openapi/OpenAPICommand.ts | 94 +++++++++++++++ packages/cli/src/openapi/fileSizeFilter.ts | 25 ++++ 9 files changed, 373 insertions(+), 131 deletions(-) create mode 100644 packages/cli/src/describeChange/OperationReference.ts create mode 100644 packages/cli/src/describeChange/buildChangeReport.ts create mode 100644 packages/cli/src/describeChange/executeCommand.ts create mode 100644 packages/cli/src/describeChange/types.ts create mode 100644 packages/cli/src/openapi/OpenAPICommand.ts create mode 100644 packages/cli/src/openapi/fileSizeFilter.ts diff --git a/packages/cli/src/cmds/describeChange.ts b/packages/cli/src/cmds/describeChange.ts index 5f93833c55..b0dc9cc69f 100644 --- a/packages/cli/src/cmds/describeChange.ts +++ b/packages/cli/src/cmds/describeChange.ts @@ -1,14 +1,12 @@ import { copyFile, mkdir, readFile, rename, rm, writeFile } from 'fs/promises'; import yargs from 'yargs'; import readline from 'readline'; +import { OpenAPIV3 } from 'openapi-types'; import { handleWorkingDirectory } from '../lib/handleWorkingDirectory'; import { locateAppMapDir } from '../lib/locateAppMapDir'; import { exists, verbose } from '../utils'; -import { exec } from 'child_process'; import { isAbsolute, join, relative } from 'path'; -import { existsSync } from 'fs'; -import { tmpdir } from 'os'; import { Action, actionActors, Diagram, format, FormatType } from '@appland/sequence-diagram'; import { buildAppMap } from '@appland/models'; import { glob } from 'glob'; @@ -18,6 +16,11 @@ import assert from 'assert'; import chalk from 'chalk'; import { AppMapReference } from '../describeChange/AppMapReference'; import { RevisionName } from '../describeChange/RevisionName'; +import { OpenAPICommand } from '../openapi/OpenAPICommand'; +import { DefaultMaxAppMapSizeInMB, fileSizeFilter } from '../openapi/fileSizeFilter'; +import buildChangeReport from '../describeChange/buildChangeReport'; +import { executeCommand } from '../describeChange/executeCommand'; +import { OperationReference } from '../describeChange/OperationReference'; export class ValidationError extends Error {} @@ -134,6 +137,7 @@ export const handler = async (argv: any) => { await mkdir(outputDir, { recursive: true }); await mkdir(join(outputDir, RevisionName.Diff), { recursive: true }); + const operationReference = new OperationReference(); const appmapReferences = new Map(); let baseAppMapFileNames: Set; let headAppMapFileNames: Set; @@ -148,7 +152,13 @@ export const handler = async (argv: any) => { await createBaselineAppMaps(rl, command); process.stdout.write(`Processing AppMaps in ${appmapDir}...`); const result = new Set([ - ...(await processAppMapDir(appmapDir, outputDir, revisionName, appmapReferences)), + ...(await processAppMapDir( + appmapDir, + outputDir, + revisionName, + operationReference, + appmapReferences + )), ]); process.stdout.write(`done (${result.size})\n`); return result; @@ -159,7 +169,7 @@ export const handler = async (argv: any) => { `Loading existing AppMap and diagram data from ${join(outputDir, revisionName)}...` ); const result = new Set([ - ...(await restoreAppMapDir(outputDir, revisionName, appmapReferences)), + ...(await restoreAppMapDir(outputDir, revisionName, operationReference, appmapReferences)), ]); process.stdout.write(`done (${result.size})\n`); return result; @@ -177,11 +187,32 @@ export const handler = async (argv: any) => { } }; + const buildOpenAPI = async (revisionName: RevisionName): Promise => { + const cmd = new OpenAPICommand(AppMapReference.outputPath(outputDir, revisionName)); + cmd.filter = fileSizeFilter(DefaultMaxAppMapSizeInMB * 1024 * 1024); + const [openapi] = await cmd.execute(); + return openapi.paths; + }; + baseAppMapFileNames = await processOrRestoreAppMaps(RevisionName.Base, baseRevision, baseCommand); headAppMapFileNames = await processOrRestoreAppMaps(RevisionName.Head, headRevision, headCommand); + const basePaths = await buildOpenAPI(RevisionName.Base); + const headPaths = await buildOpenAPI(RevisionName.Head); + // unstashAll() + const changeReport = await buildChangeReport( + baseRevision, + basePaths, + headPaths, + baseAppMapFileNames, + headAppMapFileNames, + operationReference, + appmapReferences + ); + console.log(JSON.stringify(changeReport, null, 2)); + const headAppMapFileNameArray = [...headAppMapFileNames].sort(); const changedAppMaps = new Array(); const diffSnippets = new Set(); @@ -362,12 +393,13 @@ async function processAppMapDir( appmapDir: string, outputDir: string, revisionName: RevisionName, + operationReference: OperationReference, appmapReferences: Map ): Promise { const appmapFileNames = await promisify(glob)(`${appmapDir}/**/*.appmap.json`); for (let i = 0; i < appmapFileNames.length; i++) { const appmapFileName = appmapFileNames[i]; - const appmapReference = new AppMapReference(outputDir, appmapFileName); + const appmapReference = new AppMapReference(operationReference, outputDir, appmapFileName); const diagram = await appmapReference.buildSequenceDiagram(); await copyFile(appmapFileName, appmapReference.archivedAppMapFilePath(revisionName, true)); await writeFile( @@ -388,13 +420,18 @@ async function processAppMapDir( async function restoreAppMapDir( outputDir: string, revisionName: RevisionName, + operationReference: OperationReference, appmapReferences: Map ): Promise { const baseDir = join(outputDir, revisionName); const appmapFileNames = await promisify(glob)(`${baseDir}/*.appmap.json`); for (let i = 0; i < appmapFileNames.length; i++) { const appmapFileName = appmapFileNames[i]; - const appmapReference = new AppMapReference(outputDir, relative(baseDir, appmapFileName)); + const appmapReference = new AppMapReference( + operationReference, + outputDir, + relative(baseDir, appmapFileName) + ); const appmapData = JSON.parse(await readFile(appmapFileName, 'utf-8')); const appmap = buildAppMap().source(appmapData).build(); appmapReference.populateMetadata(appmap); @@ -413,33 +450,6 @@ async function confirm(prompt: string, rl: readline.Interface): Promise return response === 'y'; } -export function executeCommand( - cmd: string, - printCommand = true, - printStdout = true, - printStderr = true -): Promise { - if (printCommand) console.log(commandStyle(cmd)); - const command = exec(cmd); - const result: string[] = []; - if (command.stdout) { - command.stdout.addListener('data', (data) => { - if (printStdout) process.stdout.write(data); - result.push(data); - }); - } - if (printStderr && command.stderr) command.stderr.pipe(process.stdout); - return new Promise((resolve, reject) => { - command.addListener('exit', (code) => { - if (code === 0) { - resolve(result.join('')); - } else { - reject(new Error(`Command failed with code ${code}`)); - } - }); - }); -} - function actionStyle(message: string): string { return chalk.bold(chalk.green(message)); } @@ -449,7 +459,7 @@ function prominentStyle(message: string): string { function mutedStyle(message: string): string { return chalk.dim(message); } -function commandStyle(message: string): string { +export function commandStyle(message: string): string { return chalk.gray(`$ ${message}`); } async function waitForEnter( diff --git a/packages/cli/src/cmds/openapi.ts b/packages/cli/src/cmds/openapi.ts index 14d3549190..7f9c96fe41 100644 --- a/packages/cli/src/cmds/openapi.ts +++ b/packages/cli/src/cmds/openapi.ts @@ -1,21 +1,11 @@ import { join } from 'path'; -import { existsSync, promises as fsp } from 'fs'; +import { promises as fsp } from 'fs'; import { readFile } from 'fs/promises'; -import { queue } from 'async'; -import { glob } from 'glob'; import yaml, { load } from 'js-yaml'; import { OpenAPIV3 } from 'openapi-types'; -import { - Model, - parseHTTPServerRequests, - rpcRequestForEvent, - SecuritySchemes, - verbose, -} from '@appland/openapi'; -import { Event } from '@appland/models'; +import { verbose } from '@appland/openapi'; import { Arguments, Argv, number } from 'yargs'; -import { inspect } from 'util'; import { locateAppMapDir } from '../lib/locateAppMapDir'; import { handleWorkingDirectory } from '../lib/handleWorkingDirectory'; @@ -26,86 +16,6 @@ import { DefaultMaxAppMapSizeInMB, fileSizeFilter } from '../lib/fileSizeFilter' export type FilterFunction = (file: string) => Promise<{ enable: boolean; message?: string }>; -class OpenAPICommand { - private readonly model = new Model(); - private readonly securitySchemes = new SecuritySchemes(); - - public errors: string[] = []; - public filter: FilterFunction = async (file: string) => ({ enable: true }); - - constructor(private readonly appmapDir: string) {} - - async execute(): Promise< - [ - { - paths: OpenAPIV3.PathsObject; - securitySchemes: Record; - }, - number - ] - > { - const q = queue(this.collectAppMap.bind(this), 5); - q.pause(); - - // Make sure the directory exists -- if it doesn't, the glob below just returns nothing. - if (!existsSync(this.appmapDir)) { - throw new Error(`AppMap directory ${this.appmapDir} does not exist`); - } - - const files = glob.sync(join(this.appmapDir, '**', '*.appmap.json')); - for (let index = 0; index < files.length; index++) { - const file = files[index]; - const filterResult = await this.filter(file); - if (!filterResult.enable) { - if (filterResult.message) console.warn(filterResult.message); - continue; - } - - q.push(file); - } - - if (q.length() > 0) { - await new Promise((resolve, reject) => { - q.drain(resolve); - q.error(reject); - q.resume(); - }); - } - - return [ - { - paths: this.model.openapi(), - securitySchemes: this.securitySchemes.openapi(), - }, - files.length, - ]; - } - - async collectAppMap(file: string): Promise { - try { - const data = await fsp.readFile(file, 'utf-8'); - parseHTTPServerRequests(JSON.parse(data), (e: Event) => { - const request = rpcRequestForEvent(e); - if (request) { - this.model.addRpcRequest(request); - this.securitySchemes.addRpcRequest(request); - } - }); - } catch (e) { - // Re-throwing this error crashes the whole process. - // So if there is a malformed AppMap, indicate it here but don't blow everything up. - // Do not write to stdout! - let errorString: string; - try { - errorString = inspect(e); - } catch (x) { - errorString = ((e as any) || '').toString(); - } - this.errors.push(errorString); - } - } -} - async function loadTemplate(fileName: string): Promise { if (!fileName) { // eslint-disable-next-line no-param-reassign diff --git a/packages/cli/src/describeChange/AppMapReference.ts b/packages/cli/src/describeChange/AppMapReference.ts index 99e6e4a27f..9252cc23cb 100644 --- a/packages/cli/src/describeChange/AppMapReference.ts +++ b/packages/cli/src/describeChange/AppMapReference.ts @@ -10,8 +10,9 @@ import { } from '@appland/sequence-diagram'; import { AppMap, CodeObject, buildAppMap } from '@appland/models'; import { readDiagramFile } from '../cmds/sequenceDiagram/readDiagramFile'; -import { executeCommand } from '../cmds/describeChange'; +import { executeCommand } from './executeCommand'; import { RevisionName } from './RevisionName'; +import { OperationReference } from './OperationReference'; export class AppMapReference { // Set of all source files that were used by the app. @@ -20,7 +21,15 @@ export class AppMapReference { public sourceLocation: string | undefined; public appmapName: string | undefined; - constructor(public outputDir: string, public appmapFileName: string) {} + constructor( + private operationReference: OperationReference, + public outputDir: string, + public appmapFileName: string + ) {} + + static outputPath(outputDir: string, revisionName: RevisionName): string { + return join(outputDir, revisionName); + } async sourceDiff(baseRevision: string): Promise { if (this.sourcePaths.size === 0) return; @@ -86,13 +95,37 @@ export class AppMapReference { if (!this.sourceLocation) this.sourceLocation = (appmap.metadata as any).source_location; if (!this.appmapName) this.appmapName = appmap.metadata.name; } - const collectSourcePath = (codeObject: CodeObject) => { + const collectPath = (codeObject: CodeObject, fn: (path: string) => void) => { const location = codeObject.location; if (location) { const path = location.split(':')[0]; - if (!isAbsolute(path)) this.sourcePaths.add(path); + if (!isAbsolute(path)) fn(path); } }; + + const collectSourcePath = (codeObject: CodeObject) => { + collectPath(codeObject, (path) => this.sourcePaths.add(path)); + }; + appmap.classMap.visit(collectSourcePath); + for (let eventId = 0; eventId < appmap.events.length; eventId++) { + const event = appmap.events[eventId]; + if (event.httpServerRequest && event.httpServerResponse) { + const method = event.httpServerRequest.request_method; + const path = + event.httpServerRequest.normalized_path_info || event.httpServerRequest.path_info; + const status = event.httpServerResponse.status; + const key = OperationReference.operationKey(method, path, status); + while (eventId < event.returnEvent.id) { + const event = appmap.events[eventId]; + collectPath(event.codeObject, (path) => { + if (!this.operationReference.sourcePathsByOperation.get(key)) + this.operationReference.sourcePathsByOperation.set(key, new Set()); + this.operationReference.sourcePathsByOperation.get(key)!.add(path); + }); + eventId += 1; + } + } + } } } diff --git a/packages/cli/src/describeChange/OperationReference.ts b/packages/cli/src/describeChange/OperationReference.ts new file mode 100644 index 0000000000..c439fa4cc3 --- /dev/null +++ b/packages/cli/src/describeChange/OperationReference.ts @@ -0,0 +1,8 @@ +export class OperationReference { + // Set of all source files, indexed by route ([ ()]). + public sourcePathsByOperation = new Map>(); + + static operationKey(method: string, path: string, status: number): string { + return `${method.toUpperCase()} ${path} (${status})`; + } +} diff --git a/packages/cli/src/describeChange/buildChangeReport.ts b/packages/cli/src/describeChange/buildChangeReport.ts new file mode 100644 index 0000000000..637b2be23d --- /dev/null +++ b/packages/cli/src/describeChange/buildChangeReport.ts @@ -0,0 +1,110 @@ +import { AppMapReference } from './AppMapReference'; +import { OpenAPIV3 } from 'openapi-types'; +import { Changes, Operation, OperationChange, RouteChanges } from './types'; +import assert from 'assert'; +import { executeCommand } from './executeCommand'; +import { exists } from '../utils'; +import { OperationReference } from './OperationReference'; + +export default async function buildChangeReport( + baseRevision: string, + basePaths: OpenAPIV3.PathsObject, + headPaths: OpenAPIV3.PathsObject, + baseAppMapFileNames: Set, + headAppMapFileNames: Set, + operationReference: OperationReference, + appmapReferences: Map +): Promise { + const buildOperationAdded = async (operation: Operation): Promise => { + return { operation /* sourceDiff, sequenceDiagrams */ }; + }; + const buildOperationChanged = async ( + operation: Operation + ): Promise => { + console.log( + OperationReference.operationKey(operation.method, operation.path, operation.status) + ); + + const candidateSourcePaths = operationReference.sourcePathsByOperation.get( + OperationReference.operationKey(operation.method, operation.path, operation.status) + ); + if (!candidateSourcePaths) return; + + const existingSourcePaths = new Set(); + await Promise.all( + [...candidateSourcePaths].map( + async (path) => (await exists(path)) && existingSourcePaths.add(path) + ) + ); + const sourcePaths = [...candidateSourcePaths].filter((path) => existingSourcePaths.has(path)); + + let sourceDiff: string | undefined; + if (sourcePaths && sourcePaths.length > 0) { + sourceDiff = await executeCommand( + `git diff ${baseRevision} -- ${[...sourcePaths].sort().join(' ')}`, + true, + true, + true + ); + } + + if (!sourceDiff) return; + + return { operation, sourceDiff /* sequenceDiagrams */ }; + }; + const buildOperationRemoved = (operation: Operation): OperationChange => { + return { operation }; + }; + + const routeChanges = { + added: [], + removed: [], + changed: [], + } as RouteChanges; + + for (const pattern of Object.keys(basePaths)) { + const pathInfo = basePaths[pattern]; + assert(pathInfo); + + for (const method of Object.keys(pathInfo)) { + const operation = pathInfo[method]; + assert(operation); + + for (const status of Object.keys(operation.responses)) { + const operation: Operation = { method, path: pattern, status: parseInt(status, 10) }; + + const headResponse = headPaths[pattern]?.[method]?.responses[status]; + if (!headResponse) { + routeChanges.removed.push(buildOperationRemoved(operation)); + break; + } + + const changed = await buildOperationChanged(operation); + if (changed) routeChanges.changed.push(changed); + } + } + } + + for (const pattern of Object.keys(headPaths)) { + const pathInfo = headPaths[pattern]; + assert(pathInfo); + + for (const method of Object.keys(pathInfo)) { + const operation = pathInfo[method]; + assert(operation); + + for (const status of Object.keys(operation.responses)) { + const operation: Operation = { method, path: pattern, status: parseInt(status, 10) }; + + const baseResponse = basePaths[pattern]?.[method]?.responses[status]; + if (!baseResponse) { + routeChanges.added.push(await buildOperationAdded(operation)); + } + } + } + } + + return { + routeChanges, + }; +} diff --git a/packages/cli/src/describeChange/executeCommand.ts b/packages/cli/src/describeChange/executeCommand.ts new file mode 100644 index 0000000000..e846649e03 --- /dev/null +++ b/packages/cli/src/describeChange/executeCommand.ts @@ -0,0 +1,29 @@ +import { exec } from 'child_process'; +import { commandStyle } from '../cmds/describeChange'; + +export function executeCommand( + cmd: string, + printCommand = true, + printStdout = true, + printStderr = true +): Promise { + if (printCommand) console.log(commandStyle(cmd)); + const command = exec(cmd); + const result: string[] = []; + if (command.stdout) { + command.stdout.addListener('data', (data) => { + if (printStdout) process.stdout.write(data); + result.push(data); + }); + } + if (printStderr && command.stderr) command.stderr.pipe(process.stdout); + return new Promise((resolve, reject) => { + command.addListener('exit', (code) => { + if (code === 0) { + resolve(result.join('')); + } else { + reject(new Error(`Command failed with code ${code}`)); + } + }); + }); +} diff --git a/packages/cli/src/describeChange/types.ts b/packages/cli/src/describeChange/types.ts new file mode 100644 index 0000000000..9d6ad99d1f --- /dev/null +++ b/packages/cli/src/describeChange/types.ts @@ -0,0 +1,23 @@ +import { Diagram as SequenceDiagram } from '@appland/sequence-diagram'; + +export type Operation = { + method: string; + path: string; + status: number; +}; + +export type OperationChange = { + operation: Operation; + sourceDiff?: string | undefined; + sequenceDiagrams?: SequenceDiagram[]; +}; + +export type RouteChanges = { + added: OperationChange[]; + removed: OperationChange[]; + changed: OperationChange[]; +}; + +export type Changes = { + routeChanges: RouteChanges; +}; diff --git a/packages/cli/src/openapi/OpenAPICommand.ts b/packages/cli/src/openapi/OpenAPICommand.ts new file mode 100644 index 0000000000..db684d36f3 --- /dev/null +++ b/packages/cli/src/openapi/OpenAPICommand.ts @@ -0,0 +1,94 @@ +import { join } from 'path'; +import { existsSync, promises as fsp } from 'fs'; +import { queue } from 'async'; +import { glob } from 'glob'; +import { OpenAPIV3 } from 'openapi-types'; +import { + Model, + parseHTTPServerRequests, + rpcRequestForEvent, + SecuritySchemes, +} from '@appland/openapi'; +import { Event } from '@appland/models'; +import { inspect } from 'util'; +import { FilterFunction } from '../cmds/openapi'; + +export class OpenAPICommand { + private readonly model = new Model(); + private readonly securitySchemes = new SecuritySchemes(); + + public errors: string[] = []; + public filter: FilterFunction = async (file: string) => ({ enable: true }); + + constructor(private readonly appmapDir: string) {} + + async execute(): Promise< + [ + { + paths: OpenAPIV3.PathsObject; + securitySchemes: Record; + }, + number + ] + > { + const q = queue(this.collectAppMap.bind(this), 5); + q.pause(); + + // Make sure the directory exists -- if it doesn't, the glob below just returns nothing. + if (!existsSync(this.appmapDir)) { + throw new Error(`AppMap directory ${this.appmapDir} does not exist`); + } + + const files = glob.sync(join(this.appmapDir, '**', '*.appmap.json')); + for (let index = 0; index < files.length; index++) { + const file = files[index]; + const filterResult = await this.filter(file); + if (!filterResult.enable) { + if (filterResult.message) console.warn(filterResult.message); + continue; + } + + q.push(file); + } + + if (q.length() > 0) { + await new Promise((resolve, reject) => { + q.drain(resolve); + q.error(reject); + q.resume(); + }); + } + + return [ + { + paths: this.model.openapi(), + securitySchemes: this.securitySchemes.openapi(), + }, + files.length, + ]; + } + + async collectAppMap(file: string): Promise { + try { + const data = await fsp.readFile(file, 'utf-8'); + parseHTTPServerRequests(JSON.parse(data), (e: Event) => { + const request = rpcRequestForEvent(e); + if (request) { + this.model.addRpcRequest(request); + this.securitySchemes.addRpcRequest(request); + } + }); + } catch (e) { + // Re-throwing this error crashes the whole process. + // So if there is a malformed AppMap, indicate it here but don't blow everything up. + // Do not write to stdout! + let errorString: string; + try { + errorString = inspect(e); + } catch (x) { + errorString = ((e as any) || '').toString(); + } + this.errors.push(errorString); + } + } +} diff --git a/packages/cli/src/openapi/fileSizeFilter.ts b/packages/cli/src/openapi/fileSizeFilter.ts new file mode 100644 index 0000000000..b2a2e33508 --- /dev/null +++ b/packages/cli/src/openapi/fileSizeFilter.ts @@ -0,0 +1,25 @@ +import { Stats } from 'fs'; +import { stat } from 'fs/promises'; +import { FilterFunction } from '../cmds/openapi'; + +// Skip files that are larger than a specified max size. + +export const DefaultMaxAppMapSizeInMB = 50; + +export function fileSizeFilter(maxFileSize: number): FilterFunction { + return async (file: string): Promise<{ enable: boolean; message?: string }> => { + let fileStat: Stats; + try { + fileStat = await stat(file); + } catch { + return { enable: false, message: `File ${file} not found` }; + } + + if (fileStat.size <= maxFileSize) return { enable: true }; + else + return { + enable: false, + message: `Skipping ${file} as its file size of ${fileStat.size} bytes is larger than the maximum configured file size of ${maxFileSize} bytes`, + }; + }; +} From 607e140c278e78c895ff523bcf12d048633ace56 Mon Sep 17 00:00:00 2001 From: Kevin Gilpin Date: Sat, 4 Mar 2023 15:42:30 -0500 Subject: [PATCH 19/24] wip: Normalize diagrams to contain only user code --- packages/cli/src/cmds/describeChange.ts | 39 ++- .../cli/src/describeChange/AppMapReference.ts | 146 +++++++++--- .../src/describeChange/OperationReference.ts | 225 ++++++++++++++++++ .../src/describeChange/buildChangeReport.ts | 107 ++++++++- 4 files changed, 455 insertions(+), 62 deletions(-) diff --git a/packages/cli/src/cmds/describeChange.ts b/packages/cli/src/cmds/describeChange.ts index b0dc9cc69f..b50faa44e6 100644 --- a/packages/cli/src/cmds/describeChange.ts +++ b/packages/cli/src/cmds/describeChange.ts @@ -1,4 +1,4 @@ -import { copyFile, mkdir, readFile, rename, rm, writeFile } from 'fs/promises'; +import { mkdir, rm, writeFile } from 'fs/promises'; import yargs from 'yargs'; import readline from 'readline'; import { OpenAPIV3 } from 'openapi-types'; @@ -8,7 +8,6 @@ import { locateAppMapDir } from '../lib/locateAppMapDir'; import { exists, verbose } from '../utils'; import { isAbsolute, join, relative } from 'path'; import { Action, actionActors, Diagram, format, FormatType } from '@appland/sequence-diagram'; -import { buildAppMap } from '@appland/models'; import { glob } from 'glob'; import { promisify } from 'util'; import { DiffDiagrams } from '../sequenceDiagramDiff/DiffDiagrams'; @@ -137,7 +136,7 @@ export const handler = async (argv: any) => { await mkdir(outputDir, { recursive: true }); await mkdir(join(outputDir, RevisionName.Diff), { recursive: true }); - const operationReference = new OperationReference(); + const operationReference = new OperationReference(outputDir); const appmapReferences = new Map(); let baseAppMapFileNames: Set; let headAppMapFileNames: Set; @@ -148,6 +147,10 @@ export const handler = async (argv: any) => { command?: string ): Promise> => { await mkdir(join(outputDir, revisionName), { recursive: true }); + await mkdir(join(outputDir, revisionName, 'operations', 'sequence-diagrams'), { + recursive: true, + }); + await checkout(revisionName, revision); await createBaselineAppMaps(rl, command); process.stdout.write(`Processing AppMaps in ${appmapDir}...`); @@ -181,9 +184,16 @@ export const handler = async (argv: any) => { command?: string ): Promise> => { if (await exists(join(outputDir, revisionName))) { - return restoreAppMaps(revisionName); + // TODO: This is temporary; operation reference indexing is not needed here. + operationReference.startIndexing(); + const result = await restoreAppMaps(revisionName); + await operationReference.finishIndexing(); + return result; } else { - return processAppMaps(revisionName, revision, command); + operationReference.startIndexing(); + const result = await processAppMaps(revisionName, revision, command); + await operationReference.finishIndexing(); + return result; } }; @@ -211,7 +221,6 @@ export const handler = async (argv: any) => { operationReference, appmapReferences ); - console.log(JSON.stringify(changeReport, null, 2)); const headAppMapFileNameArray = [...headAppMapFileNames].sort(); const changedAppMaps = new Array(); @@ -400,16 +409,7 @@ async function processAppMapDir( for (let i = 0; i < appmapFileNames.length; i++) { const appmapFileName = appmapFileNames[i]; const appmapReference = new AppMapReference(operationReference, outputDir, appmapFileName); - const diagram = await appmapReference.buildSequenceDiagram(); - await copyFile(appmapFileName, appmapReference.archivedAppMapFilePath(revisionName, true)); - await writeFile( - appmapReference.sequenceDiagramFilePath(revisionName, FormatType.JSON, true), - format(FormatType.JSON, diagram, appmapFileName).diagram - ); - await writeFile( - appmapReference.sequenceDiagramFilePath(revisionName, FormatType.Text, true), - format(FormatType.Text, diagram, appmapFileName).diagram - ); + await appmapReference.processAppMap(revisionName); if (!appmapReferences.get(appmapFileName)) { appmapReferences.set(appmapFileName, appmapReference); } @@ -432,12 +432,9 @@ async function restoreAppMapDir( outputDir, relative(baseDir, appmapFileName) ); - const appmapData = JSON.parse(await readFile(appmapFileName, 'utf-8')); - const appmap = buildAppMap().source(appmapData).build(); - appmapReference.populateMetadata(appmap); - if (!appmapReferences.get(appmapReference.appmapFileName)) { + await appmapReference.restoreMetadata(revisionName); + if (!appmapReferences.get(appmapReference.appmapFileName)) appmapReferences.set(appmapReference.appmapFileName, appmapReference); - } } return appmapFileNames.map((fileName) => relative(baseDir, fileName)); } diff --git a/packages/cli/src/describeChange/AppMapReference.ts b/packages/cli/src/describeChange/AppMapReference.ts index 9252cc23cb..57adf32110 100644 --- a/packages/cli/src/describeChange/AppMapReference.ts +++ b/packages/cli/src/describeChange/AppMapReference.ts @@ -1,11 +1,15 @@ -import { readFile } from 'fs/promises'; +import { copyFile, readFile, writeFile } from 'fs/promises'; import { basename, isAbsolute, join } from 'path'; import { existsSync } from 'fs'; import { + Action, buildDiagram, Diagram, + format, FormatType, + isServerRPC, SequenceDiagramOptions, + ServerRPC, Specification, } from '@appland/sequence-diagram'; import { AppMap, CodeObject, buildAppMap } from '@appland/models'; @@ -14,6 +18,12 @@ import { executeCommand } from './executeCommand'; import { RevisionName } from './RevisionName'; import { OperationReference } from './OperationReference'; +type Metadata = { + sourceLocation: string | undefined; + appmapName: string | undefined; + sourcePaths: string[]; +}; + export class AppMapReference { // Set of all source files that were used by the app. public sourcePaths = new Set(); @@ -49,6 +59,10 @@ export class AppMapReference { return [basename(this.appmapFileName, '.appmap.json'), `sequence.${format}`].join('.'); } + metadataFileName(): string { + return [basename(this.appmapFileName, '.appmap.json'), `metadata.json`].join('.'); + } + sequenceDiagramFilePath( revisionName: RevisionName, format: FormatType | string, @@ -76,38 +90,70 @@ export class AppMapReference { ); } - async buildSequenceDiagram(): Promise { + async processAppMap(revisionName: RevisionName) { + const appmap = await this.loadAppMap(); + + this.collectAppMapOperationData(appmap); + const metadata = AppMapReference.collectMetadata(appmap); + const diagram = await this.buildSequenceDiagram(appmap); + this.collectSequenceDiagramOperationData(revisionName, diagram); + + await copyFile(this.appmapFileName, this.archivedAppMapFilePath(revisionName, true)); + await writeFile( + this.archivedMetadataFilePath(revisionName, true), + JSON.stringify(metadata, null, 2) + ); + await writeFile( + this.sequenceDiagramFilePath(revisionName, FormatType.JSON, true), + format(FormatType.JSON, diagram, this.appmapFileName).diagram + ); + await writeFile( + this.sequenceDiagramFilePath(revisionName, FormatType.JSON, true), + format(FormatType.JSON, diagram, this.appmapFileName).diagram + ); + await writeFile( + this.sequenceDiagramFilePath(revisionName, FormatType.Text, true), + format(FormatType.Text, diagram, this.appmapFileName).diagram + ); + } + + public async restoreMetadata(revisionName: RevisionName) { + const appmapData = JSON.parse( + await readFile(this.archivedAppMapFilePath(revisionName, true), 'utf-8') + ); + const appmap = buildAppMap().source(appmapData).build(); + + this.collectAppMapOperationData(appmap); + const diagram = await this.buildSequenceDiagram(appmap); + this.collectSequenceDiagramOperationData(revisionName, diagram); + + const metadata = JSON.parse( + await readFile(this.archivedMetadataFilePath(revisionName, true), 'utf-8') + ); + + this.sourceLocation = metadata.sourceLocation; + this.appmapName = metadata.appmapName; + this.sourcePaths = new Set(metadata.sourcePaths); + } + + private async buildSequenceDiagram(appmap: AppMap): Promise { const specOptions = { loops: false } as SequenceDiagramOptions; - const appmap = await this.buildAppMap(); const specification = Specification.build(appmap, specOptions); return buildDiagram(this.appmapFileName, appmap, specification); } - async buildAppMap(): Promise { + private async loadAppMap(): Promise { const appmapData = JSON.parse(await readFile(this.appmapFileName, 'utf-8')); - const appmap = buildAppMap().source(appmapData).build(); - this.populateMetadata(appmap); - return appmap; + return buildAppMap().source(appmapData).build(); } - populateMetadata(appmap: AppMap): void { - if (appmap.metadata) { - if (!this.sourceLocation) this.sourceLocation = (appmap.metadata as any).source_location; - if (!this.appmapName) this.appmapName = appmap.metadata.name; - } - const collectPath = (codeObject: CodeObject, fn: (path: string) => void) => { - const location = codeObject.location; - if (location) { - const path = location.split(':')[0]; - if (!isAbsolute(path)) fn(path); - } - }; - - const collectSourcePath = (codeObject: CodeObject) => { - collectPath(codeObject, (path) => this.sourcePaths.add(path)); - }; + private archivedMetadataFilePath(revisionName: RevisionName, includeOutputDir: boolean): string { + const tokens = [revisionName, this.metadataFileName()]; + if (includeOutputDir) tokens.unshift(this.outputDir); + return join(...tokens); + } - appmap.classMap.visit(collectSourcePath); + private collectAppMapOperationData(appmap: AppMap) { for (let eventId = 0; eventId < appmap.events.length; eventId++) { const event = appmap.events[eventId]; if (event.httpServerRequest && event.httpServerResponse) { @@ -118,14 +164,56 @@ export class AppMapReference { const key = OperationReference.operationKey(method, path, status); while (eventId < event.returnEvent.id) { const event = appmap.events[eventId]; - collectPath(event.codeObject, (path) => { - if (!this.operationReference.sourcePathsByOperation.get(key)) - this.operationReference.sourcePathsByOperation.set(key, new Set()); - this.operationReference.sourcePathsByOperation.get(key)!.add(path); - }); + AppMapReference.collectPath(event.codeObject, (path) => + this.operationReference.addSourcePath(key, path, event.codeObject) + ); eventId += 1; } } } } + + private collectSequenceDiagramOperationData(revisionName: RevisionName, diagram: Diagram) { + const serverRPCActions: ServerRPC[] = []; + + const collectServerRPCActions = (action: Action) => { + if (isServerRPC(action)) { + serverRPCActions.push(action); + } else { + action.children.forEach(collectServerRPCActions); + } + }; + + diagram.rootActions.forEach(collectServerRPCActions); + + serverRPCActions.forEach((action) => { + this.operationReference.addServerRPC(revisionName, action); + }); + } + + private static collectPath(codeObject: CodeObject, fn: (path: string) => void) { + const location = codeObject.location; + // If there's no line number, it's not considered a proper source location. + // It may be native code, or some kind of pseudo-path. + if (location && location.includes(':')) { + const path = location.split(':')[0]; + if (path.match(/\.\w+$/) && !isAbsolute(path)) fn(path); + } + } + + private static collectMetadata(appmap: AppMap): Metadata { + const sourcePaths = new Set(); + + const collectSourcePath = (codeObject: CodeObject) => { + AppMapReference.collectPath(codeObject, (path) => sourcePaths.add(path)); + }; + + appmap.classMap.visit(collectSourcePath); + + return { + sourceLocation: (appmap.metadata as any).source_location, + appmapName: appmap.metadata.name, + sourcePaths: [...sourcePaths].sort(), + }; + } } diff --git a/packages/cli/src/describeChange/OperationReference.ts b/packages/cli/src/describeChange/OperationReference.ts index c439fa4cc3..3cf2392784 100644 --- a/packages/cli/src/describeChange/OperationReference.ts +++ b/packages/cli/src/describeChange/OperationReference.ts @@ -1,8 +1,233 @@ +import { CodeObject } from '@appland/models'; +import { + Action, + actionActors, + Actor, + Diagram as SequenceDiagram, + format, + FormatType, + isFunction, + ServerRPC, +} from '@appland/sequence-diagram'; +import assert from 'assert'; +import * as async from 'async'; +import { link, mkdir, writeFile } from 'fs/promises'; +import { glob } from 'glob'; +import { basename, join } from 'path'; +import { promisify } from 'util'; +import { readDiagramFile } from '../cmds/sequenceDiagram/readDiagramFile'; +import { exists } from '../utils'; +import { RevisionName } from './RevisionName'; + +type Route = { + method: string; + path: string; + status: number; +}; + +type QueueJob = { revisionName: RevisionName; action: ServerRPC }; + export class OperationReference { // Set of all source files, indexed by route ([ ()]). public sourcePathsByOperation = new Map>(); + private codeObjectIds = new Set(); + + // The queue has concurrency of 1, so it's used to ensure that only one route + // is processed / updated at a time. It's essential to begin indexing with startIndexing, + // and to wait for finishIndexing() to complete before continuing. + private queue: async.QueueObject | undefined; + + constructor(public outputDir: string) {} + static operationKey(method: string, path: string, status: number): string { return `${method.toUpperCase()} ${path} (${status})`; } + + startIndexing() { + this.queue = async.queue(async (job: QueueJob) => { + try { + const { revisionName, action } = job; + const { subtreeDigest } = action; + const [baseMethod, path] = action.route.split(' '); + const method = baseMethod.toUpperCase(); + + await this.findOrCreateDiagram(revisionName, subtreeDigest, action); + + await this.saveDiagramLink( + revisionName, + { method, path, status: action.status }, + subtreeDigest + ); + } catch (err) { + console.warn(err); + } + }, 1); + } + + async findOrCreateDiagram( + revisionName: RevisionName, + subtreeDigest: string, + action: ServerRPC + ): Promise { + if (await exists(this.diagramPath(revisionName, subtreeDigest))) { + return readDiagramFile(this.diagramPath(revisionName, subtreeDigest)); + } + + const diagram = this.buildDiagram(action); + await writeFile( + this.diagramPath(revisionName, subtreeDigest), + format(FormatType.JSON, diagram, subtreeDigest).diagram + ); + return diagram; + } + + async listSequenceDiagrams(revisionName: RevisionName, route: Route): Promise { + const files = await promisify(glob)( + `${this.operationPath(revisionName, route)}/*.sequence.json` + ); + return files.map((file) => basename(file, '.sequence.json')); + } + + async loadSequenceDiagram( + revisionName: RevisionName, + subtreeDigest: string + ): Promise { + const diagramPath = this.diagramPath(revisionName, subtreeDigest); + return readDiagramFile(diagramPath); + } + + async finishIndexing() { + if (!this.queue) return; + + await this.queue.drain(); + this.queue = undefined; + } + + addSourcePath(key: string, path: string, codeObject: CodeObject): void { + if (!this.sourcePathsByOperation.get(key)) + this.sourcePathsByOperation.set(key, new Set()); + this.sourcePathsByOperation.get(key)!.add(path); + + let co: CodeObject | undefined = codeObject; + while (co) { + this.codeObjectIds.add(co.fqid); + co = co.parent; + } + } + + addServerRPC(revisionName: RevisionName, action: ServerRPC) { + assert(this.queue, 'OperationReference is not in processing mode'); + + this.queue.push({ revisionName, action }); + } + + private buildDiagram(rootAction: Action): SequenceDiagram { + const validSourceLocation = (action: Action) => { + if (!isFunction(action)) return true; + + return this.codeObjectIds.has(action.callee.id); + }; + + const filterAction = (action: Action, parent: Action | undefined) => { + let actionClone: Action | undefined; + if (validSourceLocation(action)) { + actionClone = cloneAction(action); + if (parent) { + actionClone.parent = parent; + parent.children.push(actionClone); + } + parent = actionClone; + } + + action.children.forEach((child) => filterAction(child, parent)); + + return actionClone; + }; + + const rootActionClone = filterAction(rootAction, undefined); + assert(rootActionClone); + + const actors = new Array(); + const actorIds = new Set(); + + const collectActors = (action: Action) => { + actionActors(action).forEach((actor) => { + if (!actor) return; + + if (!actorIds.has(actor.id)) { + actorIds.add(actor.id); + actors.push(actor); + } + }); + + action.children.forEach(collectActors); + }; + collectActors(rootActionClone); + + return { + actors, + rootActions: [rootActionClone], + }; + } + + private async saveDiagramLink(revisionName: RevisionName, route: Route, subtreeDigest: string) { + await mkdir(this.operationPath(revisionName, route), { + recursive: true, + }); + + const targetPath = join( + this.operationPath(revisionName, route), + [subtreeDigest, 'sequence.json'].join('.') + ); + if (await exists(targetPath)) return; + + await link(this.diagramPath(revisionName, subtreeDigest), targetPath); + } + + private operationPath(revisionName: RevisionName, route: Route): string { + let { path } = route; + // To avoid a conflict with a real route, use a character that's invalid in a URL. + // The mneumonic for '^' is that it's used in regexp to start a line. + if (path === '/') path = '^'; + + return join( + this.outputDir, + revisionName, + 'operations', + route.method.toUpperCase(), + path, + route.status.toString() + ); + } + + private diagramPath(revisionName: RevisionName, subtreeDigest: string): string { + return join( + this.outputDir, + revisionName, + 'operations', + 'sequence-diagrams', + [subtreeDigest, 'sequence.json'].join('.') + ); + } +} + +function cloneAction(action: Action): Action { + const { children, caller, callee } = action as any; + + let { parent } = action as any; + if (caller) (action as any).caller = undefined; + if (callee) (action as any).callee = undefined; + action.parent = undefined; + action.children = []; + const actionClone = JSON.parse(JSON.stringify(action)); + assert(actionClone); + action.children = children; + action.parent = parent; + if (caller) (action as any).caller = caller; + if (callee) (action as any).callee = callee; + if (caller) (actionClone as any).caller = caller; + if (callee) (actionClone as any).callee = callee; + + return actionClone; } diff --git a/packages/cli/src/describeChange/buildChangeReport.ts b/packages/cli/src/describeChange/buildChangeReport.ts index 637b2be23d..4b9aa348e5 100644 --- a/packages/cli/src/describeChange/buildChangeReport.ts +++ b/packages/cli/src/describeChange/buildChangeReport.ts @@ -5,6 +5,9 @@ import assert from 'assert'; import { executeCommand } from './executeCommand'; import { exists } from '../utils'; import { OperationReference } from './OperationReference'; +import { RevisionName } from './RevisionName'; +import { Action, Diagram as SequenceDiagram } from '@appland/sequence-diagram'; +import { DiffDiagrams } from '../sequenceDiagramDiff/DiffDiagrams'; export default async function buildChangeReport( baseRevision: string, @@ -25,20 +28,102 @@ export default async function buildChangeReport( OperationReference.operationKey(operation.method, operation.path, operation.status) ); - const candidateSourcePaths = operationReference.sourcePathsByOperation.get( - OperationReference.operationKey(operation.method, operation.path, operation.status) + const baseDiagrams = new Set( + await operationReference.listSequenceDiagrams(RevisionName.Base, { + method: operation.method, + path: operation.path, + status: operation.status, + }) + ); + const headDiagrams = new Set( + await operationReference.listSequenceDiagrams(RevisionName.Head, { + method: operation.method, + path: operation.path, + status: operation.status, + }) + ); + const baseOnlyDiagrams = new Array(); + const headOnlyDiagrams = new Array(); + baseDiagrams.forEach((diagram) => + headDiagrams.has(diagram) ? undefined : baseOnlyDiagrams.push(diagram) + ); + headDiagrams.forEach((diagram) => + baseDiagrams.has(diagram) ? undefined : headOnlyDiagrams.push(diagram) ); - if (!candidateSourcePaths) return; - const existingSourcePaths = new Set(); - await Promise.all( - [...candidateSourcePaths].map( - async (path) => (await exists(path)) && existingSourcePaths.add(path) + if (baseOnlyDiagrams.length === 0 && headOnlyDiagrams.length === 0) return; + + const diffAlgorithm = new DiffDiagrams(); + // TODO: Apply includes and excludes to reduce noise and false positives. + + function countDiffActions(diagram: SequenceDiagram | undefined): number { + if (!diagram) return 0; + + let count = 0; + const countAction = (action: Action) => { + if (action.diffMode) count += 1; + action.children.forEach(countAction); + }; + diagram.rootActions.forEach(countAction); + return count; + } + + const minimalDiff = async ( + headDiagram: SequenceDiagram, + baseDiagrams: string[] + ): Promise => { + return ( + await Promise.all( + baseDiagrams.map(async (baseDiagramId) => { + const baseDiagram = await operationReference.loadSequenceDiagram( + RevisionName.Base, + baseDiagramId + ); + return diffAlgorithm.diff(baseDiagram, headDiagram); + }) + ) ) - ); - const sourcePaths = [...candidateSourcePaths].filter((path) => existingSourcePaths.has(path)); + .filter(Boolean) + .sort((a, b) => countDiffActions(a) - countDiffActions(b))[0]; + }; + + const sequenceDiagrams = ( + await Promise.all( + headOnlyDiagrams.map(async (headDiagramId) => { + const headDiagram = await operationReference.loadSequenceDiagram( + RevisionName.Head, + headDiagramId + ); + // Choose three random base diagrams. Compute the diff between the head diagram and each of + // the randomly selected base diagrams. Report the diff with the smallest number of changes. + const baseDiagramIds = [ + ...new Set( + [0, 1, 2].map( + () => baseOnlyDiagrams[Math.floor(Math.random() * baseOnlyDiagrams.length)] + ) + ), + ]; + return await minimalDiff(headDiagram, baseDiagramIds); + }) + ) + ).filter(Boolean) as SequenceDiagram[]; + let sourcePaths: string[] | undefined; let sourceDiff: string | undefined; + + const candidateSourcePaths = operationReference.sourcePathsByOperation.get( + OperationReference.operationKey(operation.method, operation.path, operation.status) + ); + if (candidateSourcePaths) { + const existingSourcePaths = new Set(); + await Promise.all( + [...candidateSourcePaths].map( + async (path) => (await exists(path)) && existingSourcePaths.add(path) + ) + ); + sourcePaths = [...candidateSourcePaths].filter((path) => existingSourcePaths.has(path)); + } + if (sourcePaths && sourcePaths.length > 0) { sourceDiff = await executeCommand( `git diff ${baseRevision} -- ${[...sourcePaths].sort().join(' ')}`, @@ -48,9 +133,7 @@ export default async function buildChangeReport( ); } - if (!sourceDiff) return; - - return { operation, sourceDiff /* sequenceDiagrams */ }; + return { operation, sourceDiff, sequenceDiagrams }; }; const buildOperationRemoved = (operation: Operation): OperationChange => { return { operation }; From e112bfef883ab64431b3eafc93afc4aa9e0e6cfd Mon Sep 17 00:00:00 2001 From: Kevin Gilpin Date: Mon, 6 Mar 2023 10:15:26 -0500 Subject: [PATCH 20/24] wip: Filter sequence diagrams I'm discarding this to filter AppMaps instead --- packages/cli/src/describeChange/buildChangeReport.ts | 10 ++-------- packages/cli/src/utils.ts | 10 ++++++++++ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/packages/cli/src/describeChange/buildChangeReport.ts b/packages/cli/src/describeChange/buildChangeReport.ts index 4b9aa348e5..44636e9eeb 100644 --- a/packages/cli/src/describeChange/buildChangeReport.ts +++ b/packages/cli/src/describeChange/buildChangeReport.ts @@ -3,7 +3,7 @@ import { OpenAPIV3 } from 'openapi-types'; import { Changes, Operation, OperationChange, RouteChanges } from './types'; import assert from 'assert'; import { executeCommand } from './executeCommand'; -import { exists } from '../utils'; +import { exists, shuffleArray } from '../utils'; import { OperationReference } from './OperationReference'; import { RevisionName } from './RevisionName'; import { Action, Diagram as SequenceDiagram } from '@appland/sequence-diagram'; @@ -96,13 +96,7 @@ export default async function buildChangeReport( ); // Choose three random base diagrams. Compute the diff between the head diagram and each of // the randomly selected base diagrams. Report the diff with the smallest number of changes. - const baseDiagramIds = [ - ...new Set( - [0, 1, 2].map( - () => baseOnlyDiagrams[Math.floor(Math.random() * baseOnlyDiagrams.length)] - ) - ), - ]; + const baseDiagramIds = shuffleArray([...baseOnlyDiagrams]).slice(0, 3); return await minimalDiff(headDiagram, baseDiagramIds); }) ) diff --git a/packages/cli/src/utils.ts b/packages/cli/src/utils.ts index 93145ad447..ffef829fb4 100644 --- a/packages/cli/src/utils.ts +++ b/packages/cli/src/utils.ts @@ -136,6 +136,16 @@ export function exists(path: PathLike): Promise { }); } +// Shuffles an array in place, returning it for convenience. +// https://stackoverflow.com/a/12646864/953770 +export function shuffleArray(array: Array): Array { + for (let i = array.length - 1; i > 0; i--) { + const j = Math.floor(Math.random() * (i + 1)); + [array[i], array[j]] = [array[j], array[i]]; + } + return array; +} + /** * Append a prefix to each line in a string * @param str the string to be prefixed From a7320ffba4e15f523a9089893a68560c61e5d29f Mon Sep 17 00:00:00 2001 From: Kevin Gilpin Date: Mon, 6 Mar 2023 17:58:32 -0500 Subject: [PATCH 21/24] wip: Compute changes on per-operation basis --- packages/cli/src/cmds/describeChange.ts | 106 +++++++++- packages/cli/src/cmds/sequenceDiagram.ts | 22 +- .../cli/src/describeChange/AppMapReference.ts | 77 ++++--- .../src/describeChange/OperationReference.ts | 200 +++++++----------- .../src/describeChange/buildChangeReport.ts | 24 ++- .../renderSequenceDiagramPNG.ts | 22 ++ 6 files changed, 261 insertions(+), 190 deletions(-) create mode 100644 packages/cli/src/sequenceDiagram/renderSequenceDiagramPNG.ts diff --git a/packages/cli/src/cmds/describeChange.ts b/packages/cli/src/cmds/describeChange.ts index b50faa44e6..2aa0f5e82d 100644 --- a/packages/cli/src/cmds/describeChange.ts +++ b/packages/cli/src/cmds/describeChange.ts @@ -7,7 +7,14 @@ import { handleWorkingDirectory } from '../lib/handleWorkingDirectory'; import { locateAppMapDir } from '../lib/locateAppMapDir'; import { exists, verbose } from '../utils'; import { isAbsolute, join, relative } from 'path'; -import { Action, actionActors, Diagram, format, FormatType } from '@appland/sequence-diagram'; +import { + Action, + actionActors, + Diagram, + format, + FormatType, + ServerRPC, +} from '@appland/sequence-diagram'; import { glob } from 'glob'; import { promisify } from 'util'; import { DiffDiagrams } from '../sequenceDiagramDiff/DiffDiagrams'; @@ -20,6 +27,9 @@ import { DefaultMaxAppMapSizeInMB, fileSizeFilter } from '../openapi/fileSizeFil import buildChangeReport from '../describeChange/buildChangeReport'; import { executeCommand } from '../describeChange/executeCommand'; import { OperationReference } from '../describeChange/OperationReference'; +import { Operation } from '../describeChange/types'; +import puppeteer from 'puppeteer'; +import { renderSequenceDiagramPNG } from '../sequenceDiagram/renderSequenceDiagramPNG'; export class ValidationError extends Error {} @@ -212,16 +222,101 @@ export const handler = async (argv: any) => { // unstashAll() + const diffDiagrams = new DiffDiagrams(); + + if (argv.include) + (Array.isArray(argv.include) ? argv.include : [argv.include]).forEach((expr: string) => + diffDiagrams.include(expr) + ); + if (argv.exclude) + (Array.isArray(argv.exclude) ? argv.exclude : [argv.exclude]).forEach((expr: string) => + diffDiagrams.exclude(expr) + ); + const changeReport = await buildChangeReport( + diffDiagrams, baseRevision, basePaths, headPaths, - baseAppMapFileNames, - headAppMapFileNames, - operationReference, - appmapReferences + operationReference ); + const operationUrl = (operation: Operation): string => + [operation.method.toUpperCase(), operation.path, `(${operation.status})`].join(' '); + + if (verbose()) console.warn(`Preparing browser for PNG rendering`); + const browser = await puppeteer.launch({ timeout: 120 * 1000, headless: !argv.showBrowser }); + + async function saveSequenceDiagram(subdir: string, diagram: Diagram, name?: string) { + if (!name) { + const { subtreeDigest } = diagram.rootActions[0]; + name = subtreeDigest; + } + const serverRPC = diagram.rootActions[0] as ServerRPC; + + let { route, status } = serverRPC; + const [method, path] = route.split(' '); + + const operationDir = join( + outputDir, + RevisionName.Diff, + subdir, + method.toUpperCase(), + path, + status.toString() + ); + await mkdir(operationDir, { recursive: true }); + await writeFile( + join(operationDir, [name, 'sequence.json'].join('.')), + format(FormatType.JSON, diagram, name).diagram + ); + + const diagramAsText = format(FormatType.Text, diagram, ''); + if (diagramAsText.diagram.trim().length > 0) + await writeFile(join(operationDir, [name, 'sequence.txt'].join('.')), diagramAsText.diagram); + + await renderSequenceDiagramPNG( + join(operationDir, [name, 'sequence.png'].join('.')), + join(operationDir, [name, 'sequence.json'].join('.')), + browser + ); + } + + await Promise.all( + changeReport.routeChanges.added.map(async (change) => { + console.log(`Added route: ${operationUrl(change.operation)}`); + if (change.sourceDiff) console.log(`Source diff: ${change.sourceDiff}`); + console.log(`${change.sequenceDiagrams?.length} sequence diagrams`); + for (const diagram of change.sequenceDiagrams || []) { + assert(diagram.rootActions.length === 1, 'Expecting a single root action'); + + await saveSequenceDiagram('added', diagram); + } + }) + ); + changeReport.routeChanges.removed.forEach((change) => { + console.log(`Removed route: ${operationUrl(change.operation)}`); + if (change.sourceDiff) console.log(`Source diff: ${change.sourceDiff}`); + }); + await Promise.all( + changeReport.routeChanges.changed.map(async (change) => { + if (!change.sourceDiff && (!change.sequenceDiagrams || change.sequenceDiagrams.length === 0)) + return; + + assert(change.sequenceDiagrams); + console.log(`Changed route: ${operationUrl(change.operation)}`); + if (change.sourceDiff) console.log(`Source diff: ${change.sourceDiff}`); + console.log(`${change.sequenceDiagrams?.length} sequence diagrams`); + for (let index = 0; index < change.sequenceDiagrams.length; index++) { + const diagram = change.sequenceDiagrams[index]; + assert(diagram.rootActions.length === 1, 'Expecting a single root action'); + await saveSequenceDiagram('changed', diagram, `diagram_${index + 1}`); + } + }) + ); + + /* + const headAppMapFileNameArray = [...headAppMapFileNames].sort(); const changedAppMaps = new Array(); const diffSnippets = new Set(); @@ -368,6 +463,7 @@ export const handler = async (argv: any) => { await writeFile(join(outputDir, 'appmap.html'), appmapHTML); } + */ rl.close(); }; diff --git a/packages/cli/src/cmds/sequenceDiagram.ts b/packages/cli/src/cmds/sequenceDiagram.ts index b36925820a..28739b33d4 100644 --- a/packages/cli/src/cmds/sequenceDiagram.ts +++ b/packages/cli/src/cmds/sequenceDiagram.ts @@ -12,7 +12,7 @@ import { Formatters, FormatType, } from '@appland/sequence-diagram'; -import { serveAndOpenSequenceDiagram } from '../lib/serveAndOpen'; +import { renderSequenceDiagramPNG } from '../sequenceDiagram/renderSequenceDiagramPNG'; import assert from 'assert'; import BrowserRenderer from './sequenceDiagram/browserRenderer'; @@ -112,22 +112,14 @@ export const handler = async (argv: any) => { if (argv.format === 'png') { // PNG rendering is performed by loading the sequence // diagram in a browser and taking a screenshot. + assert(browserRender); const diagramPath = await printDiagram(FormatType.JSON); - const outputPath = await new Promise((resolve) => - serveAndOpenSequenceDiagram(diagramPath, false, async (url) => { - if (verbose()) console.warn(`Rendering PNG`); - assert(browserRender, 'Browser not initialized'); - - const outputPath = join( - dirname(diagramPath), - [basename(diagramPath, '.json'), '.png'].join('') - ); - - await browserRender.screenshot(url, outputPath); - - resolve(outputPath); - }) + const outputPath = join( + dirname(diagramPath), + [basename(diagramPath, '.json'), '.png'].join('') ); + await renderSequenceDiagramPNG(outputPath, diagramPath, browserRender); + console.warn(`Printed diagram ${outputPath}`); } else { // Other forms of output are produced directly by the diff --git a/packages/cli/src/describeChange/AppMapReference.ts b/packages/cli/src/describeChange/AppMapReference.ts index 57adf32110..922d102033 100644 --- a/packages/cli/src/describeChange/AppMapReference.ts +++ b/packages/cli/src/describeChange/AppMapReference.ts @@ -12,7 +12,7 @@ import { ServerRPC, Specification, } from '@appland/sequence-diagram'; -import { AppMap, CodeObject, buildAppMap } from '@appland/models'; +import { AppMap, CodeObject, buildAppMap, Event } from '@appland/models'; import { readDiagramFile } from '../cmds/sequenceDiagram/readDiagramFile'; import { executeCommand } from './executeCommand'; import { RevisionName } from './RevisionName'; @@ -93,10 +93,9 @@ export class AppMapReference { async processAppMap(revisionName: RevisionName) { const appmap = await this.loadAppMap(); - this.collectAppMapOperationData(appmap); + this.collectAppMapOperationData(revisionName, appmap); const metadata = AppMapReference.collectMetadata(appmap); const diagram = await this.buildSequenceDiagram(appmap); - this.collectSequenceDiagramOperationData(revisionName, diagram); await copyFile(this.appmapFileName, this.archivedAppMapFilePath(revisionName, true)); await writeFile( @@ -123,10 +122,7 @@ export class AppMapReference { ); const appmap = buildAppMap().source(appmapData).build(); - this.collectAppMapOperationData(appmap); - const diagram = await this.buildSequenceDiagram(appmap); - this.collectSequenceDiagramOperationData(revisionName, diagram); - + this.collectAppMapOperationData(revisionName, appmap); const metadata = JSON.parse( await readFile(this.archivedMetadataFilePath(revisionName, true), 'utf-8') ); @@ -153,52 +149,51 @@ export class AppMapReference { return join(...tokens); } - private collectAppMapOperationData(appmap: AppMap) { + private collectAppMapOperationData(revisionName: RevisionName, appmap: AppMap) { for (let eventId = 0; eventId < appmap.events.length; eventId++) { const event = appmap.events[eventId]; if (event.httpServerRequest && event.httpServerResponse) { - const method = event.httpServerRequest.request_method; - const path = - event.httpServerRequest.normalized_path_info || event.httpServerRequest.path_info; - const status = event.httpServerResponse.status; - const key = OperationReference.operationKey(method, path, status); - while (eventId < event.returnEvent.id) { - const event = appmap.events[eventId]; - AppMapReference.collectPath(event.codeObject, (path) => - this.operationReference.addSourcePath(key, path, event.codeObject) - ); - eventId += 1; - } + const requestAppMap = AppMapReference.buildServerRPCAppMap(appmap, event); + this.operationReference.addServerRPC(revisionName, requestAppMap); + eventId = event.returnEvent.id; } } } - private collectSequenceDiagramOperationData(revisionName: RevisionName, diagram: Diagram) { - const serverRPCActions: ServerRPC[] = []; + private static buildServerRPCAppMap(appmap: AppMap, event: Event) { + const startId = event.id; + const endId = event.returnEvent.id; + const events = appmap.events.filter((e) => { + if (e.id < startId || e.id > endId) return false; - const collectServerRPCActions = (action: Action) => { - if (isServerRPC(action)) { - serverRPCActions.push(action); - } else { - action.children.forEach(collectServerRPCActions); - } - }; + if (e.codeObject.type !== 'function') return true; - diagram.rootActions.forEach(collectServerRPCActions); - - serverRPCActions.forEach((action) => { - this.operationReference.addServerRPC(revisionName, action); + const { isLocal } = AppMapReference.isLocalPath(e.codeObject.location); + return isLocal; }); + + return buildAppMap({ + events, + classMap: appmap.classMap.roots.map((c) => ({ ...c.data })), + metadata: appmap.metadata, + }).build(); } - private static collectPath(codeObject: CodeObject, fn: (path: string) => void) { - const location = codeObject.location; - // If there's no line number, it's not considered a proper source location. - // It may be native code, or some kind of pseudo-path. - if (location && location.includes(':')) { - const path = location.split(':')[0]; - if (path.match(/\.\w+$/) && !isAbsolute(path)) fn(path); - } + static isLocalPath(location?: string): { isLocal: boolean; path?: string } { + if (!location) return { isLocal: false }; + + if (!location.includes(':')) return { isLocal: false }; + + const path = location.split(':')[0]; + if (path.match(/\.\w+$/) && !isAbsolute(path)) return { isLocal: true, path }; + + return { isLocal: false }; + } + + static collectPath(codeObject: CodeObject, fn: (path: string) => void) { + const { isLocal, path } = AppMapReference.isLocalPath(codeObject.location); + + if (isLocal && path) fn(path); } private static collectMetadata(appmap: AppMap): Metadata { diff --git a/packages/cli/src/describeChange/OperationReference.ts b/packages/cli/src/describeChange/OperationReference.ts index 3cf2392784..445571735d 100644 --- a/packages/cli/src/describeChange/OperationReference.ts +++ b/packages/cli/src/describeChange/OperationReference.ts @@ -1,22 +1,22 @@ -import { CodeObject } from '@appland/models'; +import { AppMap, CodeObject } from '@appland/models'; import { - Action, - actionActors, - Actor, + buildDiagram, + Diagram, Diagram as SequenceDiagram, format, FormatType, - isFunction, - ServerRPC, + SequenceDiagramOptions, + Specification, } from '@appland/sequence-diagram'; import assert from 'assert'; import * as async from 'async'; -import { link, mkdir, writeFile } from 'fs/promises'; +import { link, mkdir, unlink, writeFile } from 'fs/promises'; import { glob } from 'glob'; import { basename, join } from 'path'; import { promisify } from 'util'; import { readDiagramFile } from '../cmds/sequenceDiagram/readDiagramFile'; import { exists } from '../utils'; +import { AppMapReference } from './AppMapReference'; import { RevisionName } from './RevisionName'; type Route = { @@ -25,14 +25,12 @@ type Route = { status: number; }; -type QueueJob = { revisionName: RevisionName; action: ServerRPC }; +type QueueJob = { revisionName: RevisionName; appmap: AppMap; diagram: Diagram }; export class OperationReference { // Set of all source files, indexed by route ([ ()]). public sourcePathsByOperation = new Map>(); - private codeObjectIds = new Set(); - // The queue has concurrency of 1, so it's used to ensure that only one route // is processed / updated at a time. It's essential to begin indexing with startIndexing, // and to wait for finishIndexing() to complete before continuing. @@ -47,39 +45,56 @@ export class OperationReference { startIndexing() { this.queue = async.queue(async (job: QueueJob) => { try { - const { revisionName, action } = job; - const { subtreeDigest } = action; - const [baseMethod, path] = action.route.split(' '); - const method = baseMethod.toUpperCase(); - - await this.findOrCreateDiagram(revisionName, subtreeDigest, action); - - await this.saveDiagramLink( - revisionName, - { method, path, status: action.status }, - subtreeDigest - ); + const { revisionName, appmap, diagram } = job; + assert(appmap.events[0].httpServerRequest, 'Expecting an HTTP server request (only)'); + const rootEvent = appmap.events[0]; + assert(rootEvent.httpServerRequest); + assert(rootEvent.httpServerResponse); + assert(diagram.rootActions.length === 1, 'Expecting a single root action'); + const { subtreeDigest } = diagram.rootActions[0]; + + const method = rootEvent.httpServerRequest?.request_method.toUpperCase(); + const path = + rootEvent.httpServerRequest.normalized_path_info || rootEvent.httpServerRequest.path_info; + const status = rootEvent.httpServerResponse.status; + + const key = OperationReference.operationKey(method, path, status); + appmap.classMap.visit((codeObject) => { + AppMapReference.collectPath(codeObject, (path) => + this.addSourcePath(key, path, codeObject) + ); + }); + + await this.findOrCreateAppMap(revisionName, appmap, diagram); + await this.saveLinks(revisionName, { method, path, status: status }, subtreeDigest); } catch (err) { console.warn(err); } }, 1); } - async findOrCreateDiagram( + // Returns true if the appmap was created, false if it already existed. + async findOrCreateAppMap( revisionName: RevisionName, - subtreeDigest: string, - action: ServerRPC - ): Promise { - if (await exists(this.diagramPath(revisionName, subtreeDigest))) { - return readDiagramFile(this.diagramPath(revisionName, subtreeDigest)); + appmap: AppMap, + diagram: Diagram + ): Promise { + assert(diagram.rootActions.length === 1, 'Expecting a single root action'); + const { subtreeDigest } = diagram.rootActions[0]; + + if ( + (await exists(this.diagramPath(revisionName, subtreeDigest))) && + (await exists(this.appmapPath(revisionName, subtreeDigest))) + ) { + return false; } - const diagram = this.buildDiagram(action); + await writeFile(this.appmapPath(revisionName, subtreeDigest), JSON.stringify(appmap, null, 2)); await writeFile( this.diagramPath(revisionName, subtreeDigest), format(FormatType.JSON, diagram, subtreeDigest).diagram ); - return diagram; + return true; } async listSequenceDiagrams(revisionName: RevisionName, route: Route): Promise { @@ -104,85 +119,42 @@ export class OperationReference { this.queue = undefined; } - addSourcePath(key: string, path: string, codeObject: CodeObject): void { - if (!this.sourcePathsByOperation.get(key)) - this.sourcePathsByOperation.set(key, new Set()); - this.sourcePathsByOperation.get(key)!.add(path); - - let co: CodeObject | undefined = codeObject; - while (co) { - this.codeObjectIds.add(co.fqid); - co = co.parent; - } - } - - addServerRPC(revisionName: RevisionName, action: ServerRPC) { + addServerRPC(revisionName: RevisionName, appmap: AppMap) { assert(this.queue, 'OperationReference is not in processing mode'); - this.queue.push({ revisionName, action }); - } - - private buildDiagram(rootAction: Action): SequenceDiagram { - const validSourceLocation = (action: Action) => { - if (!isFunction(action)) return true; - - return this.codeObjectIds.has(action.callee.id); - }; - - const filterAction = (action: Action, parent: Action | undefined) => { - let actionClone: Action | undefined; - if (validSourceLocation(action)) { - actionClone = cloneAction(action); - if (parent) { - actionClone.parent = parent; - parent.children.push(actionClone); - } - parent = actionClone; - } - - action.children.forEach((child) => filterAction(child, parent)); - - return actionClone; - }; - - const rootActionClone = filterAction(rootAction, undefined); - assert(rootActionClone); - - const actors = new Array(); - const actorIds = new Set(); - - const collectActors = (action: Action) => { - actionActors(action).forEach((actor) => { - if (!actor) return; - - if (!actorIds.has(actor.id)) { - actorIds.add(actor.id); - actors.push(actor); - } - }); + const specOptions = { loops: false } as SequenceDiagramOptions; + const specification = Specification.build(appmap, specOptions); + const diagram = buildDiagram('', appmap, specification); - action.children.forEach(collectActors); - }; - collectActors(rootActionClone); + this.queue.push({ revisionName, appmap, diagram }); + } - return { - actors, - rootActions: [rootActionClone], - }; + private addSourcePath(key: string, path: string, codeObject: CodeObject): void { + if (!this.sourcePathsByOperation.get(key)) + this.sourcePathsByOperation.set(key, new Set()); + this.sourcePathsByOperation.get(key)!.add(path); } - private async saveDiagramLink(revisionName: RevisionName, route: Route, subtreeDigest: string) { + private async saveLinks(revisionName: RevisionName, route: Route, subtreeDigest: string) { await mkdir(this.operationPath(revisionName, route), { recursive: true, }); - const targetPath = join( - this.operationPath(revisionName, route), - [subtreeDigest, 'sequence.json'].join('.') - ); - if (await exists(targetPath)) return; - - await link(this.diagramPath(revisionName, subtreeDigest), targetPath); + const paths = [ + [ + this.diagramPath(revisionName, subtreeDigest), + join(this.operationPath(revisionName, route), [subtreeDigest, 'sequence.json'].join('.')), + ], + [ + this.appmapPath(revisionName, subtreeDigest), + join(this.operationPath(revisionName, route), [subtreeDigest, 'appmap.json'].join('.')), + ], + ]; + for (const [src, dest] of paths) { + if (await exists(dest)) continue; + + await link(src, dest); + } } private operationPath(revisionName: RevisionName, route: Route): string { @@ -201,33 +173,21 @@ export class OperationReference { ); } + private appmapPath(revisionName: RevisionName, subtreeDigest: string): string { + return join( + this.outputDir, + revisionName, + 'operations', + [subtreeDigest, 'appmap.json'].join('.') + ); + } + private diagramPath(revisionName: RevisionName, subtreeDigest: string): string { return join( this.outputDir, revisionName, 'operations', - 'sequence-diagrams', [subtreeDigest, 'sequence.json'].join('.') ); } } - -function cloneAction(action: Action): Action { - const { children, caller, callee } = action as any; - - let { parent } = action as any; - if (caller) (action as any).caller = undefined; - if (callee) (action as any).callee = undefined; - action.parent = undefined; - action.children = []; - const actionClone = JSON.parse(JSON.stringify(action)); - assert(actionClone); - action.children = children; - action.parent = parent; - if (caller) (action as any).caller = caller; - if (callee) (action as any).callee = callee; - if (caller) (actionClone as any).caller = caller; - if (callee) (actionClone as any).callee = callee; - - return actionClone; -} diff --git a/packages/cli/src/describeChange/buildChangeReport.ts b/packages/cli/src/describeChange/buildChangeReport.ts index 44636e9eeb..e50271809d 100644 --- a/packages/cli/src/describeChange/buildChangeReport.ts +++ b/packages/cli/src/describeChange/buildChangeReport.ts @@ -10,16 +10,25 @@ import { Action, Diagram as SequenceDiagram } from '@appland/sequence-diagram'; import { DiffDiagrams } from '../sequenceDiagramDiff/DiffDiagrams'; export default async function buildChangeReport( + diffDiagrams: DiffDiagrams, baseRevision: string, basePaths: OpenAPIV3.PathsObject, headPaths: OpenAPIV3.PathsObject, - baseAppMapFileNames: Set, - headAppMapFileNames: Set, - operationReference: OperationReference, - appmapReferences: Map + operationReference: OperationReference ): Promise { const buildOperationAdded = async (operation: Operation): Promise => { - return { operation /* sourceDiff, sequenceDiagrams */ }; + const diagramIds = await operationReference.listSequenceDiagrams(RevisionName.Head, { + method: operation.method, + path: operation.path, + status: operation.status, + }); + const sequenceDiagrams = new Array(); + for (const diagramId of diagramIds) + sequenceDiagrams.push( + await operationReference.loadSequenceDiagram(RevisionName.Head, diagramId) + ); + + return { operation, sequenceDiagrams }; }; const buildOperationChanged = async ( operation: Operation @@ -53,9 +62,6 @@ export default async function buildChangeReport( if (baseOnlyDiagrams.length === 0 && headOnlyDiagrams.length === 0) return; - const diffAlgorithm = new DiffDiagrams(); - // TODO: Apply includes and excludes to reduce noise and false positives. - function countDiffActions(diagram: SequenceDiagram | undefined): number { if (!diagram) return 0; @@ -79,7 +85,7 @@ export default async function buildChangeReport( RevisionName.Base, baseDiagramId ); - return diffAlgorithm.diff(baseDiagram, headDiagram); + return diffDiagrams.diff(baseDiagram, headDiagram); }) ) ) diff --git a/packages/cli/src/sequenceDiagram/renderSequenceDiagramPNG.ts b/packages/cli/src/sequenceDiagram/renderSequenceDiagramPNG.ts new file mode 100644 index 0000000000..a851f7d719 --- /dev/null +++ b/packages/cli/src/sequenceDiagram/renderSequenceDiagramPNG.ts @@ -0,0 +1,22 @@ +import { Browser } from 'puppeteer'; +import { verbose } from '../utils'; +import { serveAndOpenSequenceDiagram } from '../lib/serveAndOpen'; +import assert from 'assert'; +import BrowserRenderer from '../cmds/sequenceDiagram/browserRenderer'; + +export async function renderSequenceDiagramPNG( + outputPath: string, + diagramPath: string, + browser: BrowserRenderer +): Promise { + return new Promise((resolve) => + serveAndOpenSequenceDiagram(diagramPath, false, async (url) => { + if (verbose()) console.warn(`Rendering PNG`); + assert(browser, 'Browser not initialized'); + + await browser.screenshot(url, outputPath); + + resolve(); + }) + ); +} From a43610f2375a04f8bea93ef6bdc6c0ddde78514b Mon Sep 17 00:00:00 2001 From: Kevin Gilpin Date: Mon, 6 Mar 2023 19:24:20 -0500 Subject: [PATCH 22/24] fixup! wip: Compute changes on per-operation basis --- packages/cli/src/cmds/describeChange.ts | 2 +- packages/cli/src/describeChange/OperationReference.ts | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/cmds/describeChange.ts b/packages/cli/src/cmds/describeChange.ts index 2aa0f5e82d..46c7612f16 100644 --- a/packages/cli/src/cmds/describeChange.ts +++ b/packages/cli/src/cmds/describeChange.ts @@ -157,7 +157,7 @@ export const handler = async (argv: any) => { command?: string ): Promise> => { await mkdir(join(outputDir, revisionName), { recursive: true }); - await mkdir(join(outputDir, revisionName, 'operations', 'sequence-diagrams'), { + await mkdir(join(outputDir, revisionName, 'operations'), { recursive: true, }); diff --git a/packages/cli/src/describeChange/OperationReference.ts b/packages/cli/src/describeChange/OperationReference.ts index 445571735d..46194d0a8a 100644 --- a/packages/cli/src/describeChange/OperationReference.ts +++ b/packages/cli/src/describeChange/OperationReference.ts @@ -115,7 +115,8 @@ export class OperationReference { async finishIndexing() { if (!this.queue) return; - await this.queue.drain(); + this.queue.length() === 0 || (await this.queue.drain()); + this.queue = undefined; } From 7f5f35e5f569628badc07d53247918d4d5de5f01 Mon Sep 17 00:00:00 2001 From: Kevin Gilpin Date: Tue, 7 Mar 2023 11:51:53 -0500 Subject: [PATCH 23/24] feat: List failed tests and print their logs --- packages/cli/src/cmds/describeChange.ts | 29 +++++++++++--- .../cli/src/describeChange/AppMapReference.ts | 31 +++++++++++--- .../src/describeChange/buildChangeReport.ts | 40 ++++++++++++++++++- packages/cli/src/describeChange/types.ts | 18 +++++++++ .../renderSequenceDiagramPNG.ts | 14 ++++--- 5 files changed, 116 insertions(+), 16 deletions(-) diff --git a/packages/cli/src/cmds/describeChange.ts b/packages/cli/src/cmds/describeChange.ts index 46c7612f16..8cac72a5e9 100644 --- a/packages/cli/src/cmds/describeChange.ts +++ b/packages/cli/src/cmds/describeChange.ts @@ -275,11 +275,15 @@ export const handler = async (argv: any) => { if (diagramAsText.diagram.trim().length > 0) await writeFile(join(operationDir, [name, 'sequence.txt'].join('.')), diagramAsText.diagram); - await renderSequenceDiagramPNG( - join(operationDir, [name, 'sequence.png'].join('.')), - join(operationDir, [name, 'sequence.json'].join('.')), - browser - ); + try { + await renderSequenceDiagramPNG( + join(operationDir, [name, 'sequence.png'].join('.')), + join(operationDir, [name, 'sequence.json'].join('.')), + browser + ); + } catch (e) { + console.warn(`Failed to render sequence diagram for ${operationDir}: ${e}`); + } } await Promise.all( @@ -315,6 +319,21 @@ export const handler = async (argv: any) => { }) ); + if (changeReport.failedTests) { + console.log(`${changeReport.failedTests.length} tests failed`); + for (const test of changeReport.failedTests) { + console.log(`${test.testLocation} (${test.name})`); + console.log(`${test.appmapFile}`); + console.log(`Log messages:`); + for (const logEntry of test.logEntries) { + process.stdout.write(`${logEntry.message}`); + if (verbose()) console.log(` ${logEntry.stack.join('\n ')}`); + } + } + } else { + console.log(`All tests passed`); + } + /* const headAppMapFileNameArray = [...headAppMapFileNames].sort(); diff --git a/packages/cli/src/describeChange/AppMapReference.ts b/packages/cli/src/describeChange/AppMapReference.ts index 922d102033..5df9941093 100644 --- a/packages/cli/src/describeChange/AppMapReference.ts +++ b/packages/cli/src/describeChange/AppMapReference.ts @@ -1,15 +1,12 @@ import { copyFile, readFile, writeFile } from 'fs/promises'; -import { basename, isAbsolute, join } from 'path'; +import { basename, dirname, isAbsolute, join } from 'path'; import { existsSync } from 'fs'; import { - Action, buildDiagram, Diagram, format, FormatType, - isServerRPC, SequenceDiagramOptions, - ServerRPC, Specification, } from '@appland/sequence-diagram'; import { AppMap, CodeObject, buildAppMap, Event } from '@appland/models'; @@ -17,11 +14,14 @@ import { readDiagramFile } from '../cmds/sequenceDiagram/readDiagramFile'; import { executeCommand } from './executeCommand'; import { RevisionName } from './RevisionName'; import { OperationReference } from './OperationReference'; +import { promisify } from 'util'; +import { glob } from 'glob'; type Metadata = { sourceLocation: string | undefined; appmapName: string | undefined; sourcePaths: string[]; + testStatus?: string; }; export class AppMapReference { @@ -79,6 +79,26 @@ export class AppMapReference { return join(...tokens); } + /** + * Gets the AppMap file names of failed test cases. + */ + static async listFailedTests(outputDir: string, revisionName: RevisionName): Promise { + const metadataFiles = await promisify(glob)(`${outputDir}/${revisionName}/*.metadata.json`); + const result = new Array(); + for (const metadataFile of metadataFiles) { + const metadata = JSON.parse(await readFile(metadataFile, 'utf-8')) as Metadata; + if (metadata.testStatus === 'failed') { + result.push( + join( + dirname(metadataFile), + [basename(metadataFile, '.metadata.json'), 'appmap.json'].join('.') + ) + ); + } + } + return result; + } + async loadSequenceDiagramJSON(revisionName: RevisionName): Promise { return readDiagramFile(this.sequenceDiagramFilePath(revisionName, FormatType.JSON, true)); } @@ -116,7 +136,7 @@ export class AppMapReference { ); } - public async restoreMetadata(revisionName: RevisionName) { + async restoreMetadata(revisionName: RevisionName) { const appmapData = JSON.parse( await readFile(this.archivedAppMapFilePath(revisionName, true), 'utf-8') ); @@ -208,6 +228,7 @@ export class AppMapReference { return { sourceLocation: (appmap.metadata as any).source_location, appmapName: appmap.metadata.name, + testStatus: appmap.metadata.test_status, sourcePaths: [...sourcePaths].sort(), }; } diff --git a/packages/cli/src/describeChange/buildChangeReport.ts b/packages/cli/src/describeChange/buildChangeReport.ts index e50271809d..312e23452c 100644 --- a/packages/cli/src/describeChange/buildChangeReport.ts +++ b/packages/cli/src/describeChange/buildChangeReport.ts @@ -1,6 +1,6 @@ import { AppMapReference } from './AppMapReference'; import { OpenAPIV3 } from 'openapi-types'; -import { Changes, Operation, OperationChange, RouteChanges } from './types'; +import { Changes, LogEntry, Operation, OperationChange, RouteChanges, TestFailure } from './types'; import assert from 'assert'; import { executeCommand } from './executeCommand'; import { exists, shuffleArray } from '../utils'; @@ -8,6 +8,8 @@ import { OperationReference } from './OperationReference'; import { RevisionName } from './RevisionName'; import { Action, Diagram as SequenceDiagram } from '@appland/sequence-diagram'; import { DiffDiagrams } from '../sequenceDiagramDiff/DiffDiagrams'; +import { readFile } from 'fs/promises'; +import { buildAppMap } from '@appland/models'; export default async function buildChangeReport( diffDiagrams: DiffDiagrams, @@ -187,7 +189,43 @@ export default async function buildChangeReport( } } + const appmapFileNamesOfFailedTests = await AppMapReference.listFailedTests( + operationReference.outputDir, + RevisionName.Head + ); + + const buildFailedTest = async (appmapFileName: string): Promise => { + const appmapData = await readFile(appmapFileName, 'utf-8'); + const appmap = buildAppMap().source(appmapData).build(); + + const logEntries = appmap.events + .filter((event) => event.isCall() && event.labels.has('log')) + .map((log) => { + const message = (log.parameters || []).map((param) => param.value).join(''); + if (message) { + return { + message, + stack: log + .callStack() + .map((event) => event.codeObject.location || event.codeObject.fqid), + }; + } + }) + .filter(Boolean) as LogEntry[]; + + return { + appmapFile: appmapFileName, + name: appmap.metadata.name, + testLocation: appmap.metadata.source_location, + logEntries, + }; + }; + + const failedTests = await Promise.all(appmapFileNamesOfFailedTests.map(buildFailedTest)); + return { routeChanges, + findings: [], + failedTests, }; } diff --git a/packages/cli/src/describeChange/types.ts b/packages/cli/src/describeChange/types.ts index 9d6ad99d1f..d8af015517 100644 --- a/packages/cli/src/describeChange/types.ts +++ b/packages/cli/src/describeChange/types.ts @@ -18,6 +18,24 @@ export type RouteChanges = { changed: OperationChange[]; }; +export type Finding = { + ruleId: string; +}; + +export type LogEntry = { + stack: string[]; + message: string; +}; + +export type TestFailure = { + appmapFile: string; + name: string; + testLocation?: string; + logEntries: LogEntry[]; +}; + export type Changes = { routeChanges: RouteChanges; + findings: Finding[]; + failedTests: TestFailure[]; }; diff --git a/packages/cli/src/sequenceDiagram/renderSequenceDiagramPNG.ts b/packages/cli/src/sequenceDiagram/renderSequenceDiagramPNG.ts index a851f7d719..276d7bbab8 100644 --- a/packages/cli/src/sequenceDiagram/renderSequenceDiagramPNG.ts +++ b/packages/cli/src/sequenceDiagram/renderSequenceDiagramPNG.ts @@ -9,14 +9,18 @@ export async function renderSequenceDiagramPNG( diagramPath: string, browser: BrowserRenderer ): Promise { - return new Promise((resolve) => + return new Promise((resolve, reject) => serveAndOpenSequenceDiagram(diagramPath, false, async (url) => { - if (verbose()) console.warn(`Rendering PNG`); - assert(browser, 'Browser not initialized'); + try { + if (verbose()) console.warn(`Rendering PNG`); + assert(browser, 'Browser not initialized'); - await browser.screenshot(url, outputPath); + await browser.screenshot(url, outputPath); - resolve(); + resolve(); + } catch (e) { + reject(e); + } }) ); } From 795a8048e9bfefc959f0891a14367a1b41edf011 Mon Sep 17 00:00:00 2001 From: Kevin Gilpin Date: Sat, 18 Mar 2023 17:25:19 -0400 Subject: [PATCH 24/24] fix: Fix some merge conflicts --- packages/cli/src/cli.ts | 1 + packages/cli/src/cmds/describeChange.ts | 17 +++++++---------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 540e76a352..36e5f2d0eb 100755 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -32,6 +32,7 @@ const InspectCommand = require('./cmds/inspect/inspect'); const SequenceDiagramCommand = require('./cmds/sequenceDiagram'); const SequenceDiagramDiffCommand = require('./cmds/sequenceDiagramDiff'); const StatsCommand = require('./cmds/stats/stats'); +import UploadCommand from './cmds/upload'; const BuildArchive = require('./cmds/archive/archive'); const RestoreArchive = require('./cmds/archive/restore'); const UpdateAppMaps = require('./cmds/update'); diff --git a/packages/cli/src/cmds/describeChange.ts b/packages/cli/src/cmds/describeChange.ts index 8cac72a5e9..be0605ddaf 100644 --- a/packages/cli/src/cmds/describeChange.ts +++ b/packages/cli/src/cmds/describeChange.ts @@ -7,14 +7,7 @@ import { handleWorkingDirectory } from '../lib/handleWorkingDirectory'; import { locateAppMapDir } from '../lib/locateAppMapDir'; import { exists, verbose } from '../utils'; import { isAbsolute, join, relative } from 'path'; -import { - Action, - actionActors, - Diagram, - format, - FormatType, - ServerRPC, -} from '@appland/sequence-diagram'; +import { Diagram, format, FormatType, ServerRPC } from '@appland/sequence-diagram'; import { glob } from 'glob'; import { promisify } from 'util'; import { DiffDiagrams } from '../sequenceDiagramDiff/DiffDiagrams'; @@ -30,6 +23,7 @@ import { OperationReference } from '../describeChange/OperationReference'; import { Operation } from '../describeChange/types'; import puppeteer from 'puppeteer'; import { renderSequenceDiagramPNG } from '../sequenceDiagram/renderSequenceDiagramPNG'; +import BrowserRenderer from './sequenceDiagram/browserRenderer'; export class ValidationError extends Error {} @@ -245,7 +239,7 @@ export const handler = async (argv: any) => { [operation.method.toUpperCase(), operation.path, `(${operation.status})`].join(' '); if (verbose()) console.warn(`Preparing browser for PNG rendering`); - const browser = await puppeteer.launch({ timeout: 120 * 1000, headless: !argv.showBrowser }); + const browserRender = new BrowserRenderer(argv.showBrowser); async function saveSequenceDiagram(subdir: string, diagram: Diagram, name?: string) { if (!name) { @@ -279,7 +273,7 @@ export const handler = async (argv: any) => { await renderSequenceDiagramPNG( join(operationDir, [name, 'sequence.png'].join('.')), join(operationDir, [name, 'sequence.json'].join('.')), - browser + browserRender ); } catch (e) { console.warn(`Failed to render sequence diagram for ${operationDir}: ${e}`); @@ -298,6 +292,9 @@ export const handler = async (argv: any) => { } }) ); + + await browserRender.close(); + changeReport.routeChanges.removed.forEach((change) => { console.log(`Removed route: ${operationUrl(change.operation)}`); if (change.sourceDiff) console.log(`Source diff: ${change.sourceDiff}`);