diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 6eead7d969..6aa01290f9 100755 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -24,6 +24,7 @@ const InventoryCommand = require('./inventoryCommand'); const OpenCommand = require('./cmds/open/open'); const InspectCommand = require('./cmds/inspect/inspect'); const RecordCommand = require('./cmds/record/record'); +const SearchCommand = require('./cmds/search/search'); import InstallCommand from './cmds/agentInstaller/install-agent'; import StatusCommand from './cmds/agentInstaller/status'; import PruneCommand from './cmds/prune/prune'; @@ -451,6 +452,7 @@ yargs(process.argv.slice(2)) .command(RecordCommand) .command(StatusCommand) .command(InspectCommand) + .command(SearchCommand) .command(PruneCommand) .strict() .demandCommand() diff --git a/packages/cli/src/cmds/search/sanitizeStack.ts b/packages/cli/src/cmds/search/sanitizeStack.ts new file mode 100644 index 0000000000..f6fd552a52 --- /dev/null +++ b/packages/cli/src/cmds/search/sanitizeStack.ts @@ -0,0 +1,25 @@ +import isNumeric from "../../lib/isNumeric"; + +/** + * Process and sanitize a raw user-input stack trace. The output is an array of strings, each of the + * form `path/to/file.ext(:)?`, where lineno is an optional line number. The first (0th) + * entry in the input stack is expected to be the deepest function. The output stack lines are in + * the reverse order. + * + * @param stack raw stack trace input from the user. + */ +export function sanitizeStack(stack: string): string[] { + const sanitize = (line: string): string => { + const [path, lineno] = line.split(':', 2); + const result = [path]; + if (isNumeric(lineno)) result.push(lineno); + return result.join(':'); + }; + + return stack + .split('\n') + .map((line) => line.trim()) + .filter((line) => line !== '') + .map(sanitize) + .reverse(); +} diff --git a/packages/cli/src/cmds/search/search.ts b/packages/cli/src/cmds/search/search.ts new file mode 100644 index 0000000000..35ccd1512e --- /dev/null +++ b/packages/cli/src/cmds/search/search.ts @@ -0,0 +1,106 @@ +import yargs, { number } from 'yargs'; +import readline from 'readline'; +import { handleWorkingDirectory } from '../../lib/handleWorkingDirectory'; +import { locateAppMapDir } from '../../lib/locateAppMapDir'; +import FindCodeObjects from '../../search/findCodeObjects'; +import FindEvents from '../../search/findEvents'; +import { verbose } from '../../utils'; +import FindStack, { FindStackMatch } from '../../search/findStack'; +import { sanitizeStack } from './sanitizeStack'; + +export const command = 'search'; +export const describe = + 'Search AppMaps for references to a code objects (package, function, line, class, query, route, etc)'; + +export const builder = (args) => { + args.option('directory', { + describe: 'program working directory', + type: 'string', + alias: 'd', + }); + args.option('appmap-dir', { + describe: 'directory to recursively inspect for AppMaps', + }); + args.option('route', { + describe: 'a route which all matches must contain', + }); + args.option('limit', { + describe: 'number of top matches to print', + type: number, + default: 20, + }); + return args.strict(); +}; + +export const handler = async (argv) => { + verbose(argv.verbose); + handleWorkingDirectory(argv.directory); + const appmapDir = await locateAppMapDir(argv.appmapDir); + const route = argv.route; + const limit = argv.limit; + + if (!route) yargs.exit(1, new Error(`No route was provided`)); + + const routeParam = `route:${route}`; + let stack: string; + + if (process.stdin.isTTY) { + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, + }); + stack = await new Promise((resolve) => { + rl.question(`Enter a stack trace to search for: `, resolve); + }); + } else { + const result: Buffer[] = []; + let length = 0; + for await (const chunk of process.stdin) { + result.push(chunk); + length += chunk.length; + } + stack = Buffer.concat(result, length).toString('utf8'); + } + + if (!stack) yargs.exit(1, new Error(`No stack was provided`)); + const stackLines = sanitizeStack(stack); + + const finder = new FindCodeObjects(appmapDir, routeParam); + const codeObjectMatches = await finder.find( + (count) => {}, + () => {} + ); + + if (codeObjectMatches?.length === 0) { + return yargs.exit(1, new Error(`Code object '${routeParam}' not found`)); + } + + const result: FindStackMatch[] = []; + await Promise.all( + codeObjectMatches.map(async (codeObjectMatch) => { + const findStack = new FindStack(codeObjectMatch.appmap, stackLines); + const matches = await findStack.match(); + result.push(...matches); + }) + ); + + let duplicateCount = 0; + const hashes = new Set(); + result + .sort((a, b) => b.score - a.score) + .slice(0, limit) + .forEach((match) => { + if (hashes.has(match.hash_v2)) { + duplicateCount += 1; + return; + } + hashes.add(match.hash_v2); + console.log( + `${match.appmap}.appmap.json:${ + match.eventIds[match.eventIds.length - 1] + } (score=${match.score})` + ); + }); + console.log(); + console.log(`Suppressed printing of ${duplicateCount} duplicates`); +}; diff --git a/packages/cli/src/lib/isNumeric.ts b/packages/cli/src/lib/isNumeric.ts new file mode 100644 index 0000000000..de7ee68510 --- /dev/null +++ b/packages/cli/src/lib/isNumeric.ts @@ -0,0 +1,3 @@ +export default function isNumeric(n: string): boolean { + return !isNaN(parseFloat(n)) && isFinite(parseFloat(n)); +} diff --git a/packages/cli/src/search/findStack.ts b/packages/cli/src/search/findStack.ts new file mode 100644 index 0000000000..b5c15c3787 --- /dev/null +++ b/packages/cli/src/search/findStack.ts @@ -0,0 +1,139 @@ +import { buildAppMap, CodeObject, Event } from '@appland/models'; +import { readFile } from 'fs/promises'; +import { inspect } from 'util'; +import { verbose } from '../utils'; +import LocationMap from './locationMap'; +import SearchResultHashV2 from './searchResultHashV2'; + +export type FindStackMatch = { + appmap: string; + eventIds: number[]; + score: number; + hash_v2: string; +}; + +export default class FindStack { + constructor(public appMapName: string, public stackLines: string[]) {} + + async match(): Promise { + const appmapFile = [this.appMapName, 'appmap.json'].join('.'); + + let appmapData: string; + try { + appmapData = JSON.parse(await readFile(appmapFile, 'utf-8')); + } catch (e) { + console.log((e as any).code); + console.warn(`Error loading ${appmapFile}: ${e}`); + return []; + } + + const appmap = buildAppMap(appmapData).build(); + const locationMap = new LocationMap(appmap.classMap); + + const locationStack = [...this.stackLines]; + if (verbose()) + console.log(`Searching for stack: ${inspect(locationStack)}`); + const result: FindStackMatch[] = []; + let score: number[] = []; + let stack: Event[] = []; + + const enter = (event: Event): boolean | undefined => { + let matchIndex: number | undefined; + if (event.path && event.lineno) { + if (verbose()) + console.log( + `${stack.map((_) => ' ').join('')}${event.path}:${event.lineno}` + ); + for ( + let i = 0; + matchIndex === undefined && i < locationStack.length; + i++ + ) { + const [stackLinePath, stackLineLineno] = locationStack[i].split( + ':', + 2 + ); + + if (stackLinePath === event.path) + locationMap.functionContainsLine( + stackLinePath, + event.lineno, + parseFloat(stackLineLineno) + ); + + if ( + stackLinePath === event.path && + locationMap.functionContainsLine( + stackLinePath, + event.lineno, + parseFloat(stackLineLineno) + ) + ) { + matchIndex = i; + } + } + } + + stack.push(event); + if (matchIndex !== undefined) { + if (verbose()) + console.log( + `${stack.map((_) => ' ').join('')}Matched ${ + locationStack[matchIndex] + } at event ${event.id} (${event.codeObject.fqid})` + ); + locationStack.splice(0, matchIndex + 1); + if (verbose()) + console.log(`Now matching stack: ${inspect(locationStack)}`); + + score.push(1); + if (locationStack.length === 0) { + return true; + } + } else { + score.push(0); + } + }; + + const leave = () => { + stack.pop(); + score.pop(); + }; + + const eventsEmitted = new Set(); + for (let i = 0; i < appmap.events.length; ) { + const event = appmap.events[i]; + if (event.isCall()) { + const isFullMatch = enter(event); + const isLeaf = event.children.length === 0; + if (isFullMatch || isLeaf) { + const total = score.reduce((sum, n) => (n ? sum + n : sum)); + if (total > 0) { + const matchStack = stack.filter((_, index) => score[index]); + if (!eventsEmitted.has(matchStack[matchStack.length - 1].id)) { + eventsEmitted.add(matchStack[matchStack.length - 1].id); + const hash = new SearchResultHashV2(matchStack); + if (verbose()) console.log(`Match hash: ${hash.canonicalString}`); + result.push({ + appmap: this.appMapName, + eventIds: matchStack.map((e) => e.id), + hash_v2: hash.digest(), + score: total, + }); + } + } + } + if (isFullMatch) { + for (++i; appmap.events[i] !== event.returnEvent; ++i) {} + } else { + ++i; + } + } else { + ++i; + leave(); + } + } + + return result; + } +} diff --git a/packages/cli/src/search/locationMap.ts b/packages/cli/src/search/locationMap.ts new file mode 100644 index 0000000000..f702402c95 --- /dev/null +++ b/packages/cli/src/search/locationMap.ts @@ -0,0 +1,76 @@ +import { AppMap, ClassMap, CodeObject } from '@appland/models'; +import isNumeric from '../lib/isNumeric'; + +export const Threshold = 20; + +export default class LocationMap { + lineNumbers: Map; + + constructor(public classMap: ClassMap) { + const lineNumbers: Map = new Map(); + classMap.visit((co: CodeObject) => { + if (!co.location) return; + + const [path, lineno] = co.location.split(':', 2); + if (!path || !isNumeric(lineno)) return; + + if (!lineNumbers.get(path)) lineNumbers.set(path, []); + + lineNumbers.get(path)!.push(parseFloat(lineno)); + }); + + this.lineNumbers = new Map(); + for (const entry of lineNumbers) { + this.lineNumbers.set(entry[0], [...entry[1].sort((a, b) => a - b)]); + } + } + + /** + * Tests whether a test line number is contained within the function that starts at a given + * line number. + * + * If the test line number is greater than the given line number and less than the + * next known function line, returns true. + * + * If the test line number is greater than the given line number and greater than the + * next known function line, returns false. + * + * If the test line number is greater than the given line number, and there is no greater + * known function line, returns true if the test line number is within a threshold of + * the given line number. + * + * @param path file path containing the function to test + * @param lineNumber start line of the function + * @param testLineNumber line number to test + * @param threshold used if the testLineNumber is greater than the lineNumber, and there is + * no known larger function line number in the code file (path). + */ + functionContainsLine( + path: string, + lineNumber: number, + testLineNumber: number, + threshold = Threshold + ): boolean { + if (testLineNumber < lineNumber) return false; + + const lineNumbers = this.lineNumbers.get(path); + if (!lineNumbers) return false; + + const lastLineNumber = lineNumbers[lineNumbers.length - 1]; + + let containingFunctionLine: number | undefined; + for (let i = 0; i < lineNumbers.length; i++) { + const line = lineNumbers[i]; + if (line <= testLineNumber) containingFunctionLine = line; + else break; + } + + if (lineNumber === containingFunctionLine) { + if (containingFunctionLine === lastLineNumber) + return testLineNumber - lastLineNumber < threshold; + else return true; + } + + return false; + } +} diff --git a/packages/cli/src/search/searchResultHashV2.ts b/packages/cli/src/search/searchResultHashV2.ts new file mode 100644 index 0000000000..e1ea83acdf --- /dev/null +++ b/packages/cli/src/search/searchResultHashV2.ts @@ -0,0 +1,58 @@ +import { Event } from '@appland/models'; +import { createHash, Hash } from 'crypto'; + +function hashEvent(entries: string[], prefix: string, event: Event): void { + Object.keys(event.stableProperties) + .sort() + .forEach((key) => + entries.push( + [[prefix, key].join('.'), event.stableProperties[key].toString()].join( + '=' + ) + ) + ); +} + +/** + * Builds a hash (digest) of a search result. The digest is constructed by first building a canonical + * string of the search result, of the form: + * + * ``` + * [ + * algorithmVersion=2 + * stack[1].=value1 + * ... + * stack[1].=valueN + * ... + * stack[3].=value1 + * ... + * stack[3].=valueN + * ] + * ``` + */ +export default class SearchResultHashV2 { + private hashEntries: string[] = []; + private hash: Hash; + + constructor(stack: Event[]) { + this.hash = createHash('sha256'); + + // Algorithm version is 2 because it closely matches the Findings hash algorithm v2 + const hashEntries = [['algorithmVersion', '2']].map((e) => e.join('=')); + this.hashEntries = hashEntries; + + stack.forEach((event, index) => + hashEvent(hashEntries, `stack[${index + 1}]`, event) + ); + + hashEntries.forEach((e) => this.hash.update(e)); + } + + get canonicalString(): string { + return this.hashEntries.join('\n'); + } + + digest(): string { + return this.hash.digest('hex'); + } +}