Skip to content

perf: stl remote indexing, no node_modules in local tracking W-21986538 #847

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mshanemc merged 3 commits into from
Apr 11, 2026
Merged

perf: stl remote indexing, no node_modules in local tracking W-21986538 #847

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
3aea8b5
fix: stl remote indexing perf
mshanemc Apr 11, 2026
15277aa
chore: bump deps for xnuts
mshanemc Apr 11, 2026
c6dc6aa
fix: ignore node_modules because ui-bundle/webapp/react stuff
mshanemc Apr 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,9 +50,9 @@
"node": ">=18.0.0"
},
"dependencies": {
"@salesforce/core": "^8.27.1",
"@salesforce/kit": "^3.2.4",
"@salesforce/source-deploy-retrieve": "^12.32.1",
"@salesforce/core": "^8.28.1",
"@salesforce/kit": "^3.2.6",
"@salesforce/source-deploy-retrieve": "^12.32.3",
"@salesforce/ts-types": "^2.0.12",
"fast-xml-parser": "^5.5.7",
"graceful-fs": "^4.2.11",
Expand Down
2 changes: 2 additions & 0 deletions src/shared/local/localShadowRepo.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -356,6 +356,8 @@ const fileFilter =
(f: string): boolean =>
// no hidden files
!f.includes(`${path.sep}.`) &&
// no node_modules (e.g. uiBundle packages inside force-app)
!f.split(path.sep).includes('node_modules') &&
// no lwc tests
excludeLwcLocalOnlyTest(f) &&
// no gitignore files
Expand Down
51 changes: 22 additions & 29 deletions src/shared/remote/remoteSourceTrackingService.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,9 @@ export class RemoteSourceTrackingService {
private queryCache = new Map<number, SourceMember[]>();
private userQueryCache = new Map<string, string>();

/** Lazily-built reverse index: decoded key → stored key, for O(1) URI-encoded key resolution */
private decodedKeyIndex: Map<string, string> | undefined;

/**
* Initializes the service with existing remote source tracking data, or sets
* the state to begin source tracking of metadata changes in the org.
Expand Down Expand Up @@ -482,20 +485,22 @@ ${formatSourceMemberWarnings(outstandingSourceMembers)}`

/** Return a tracked element as MemberRevision data.*/
private getSourceMember(key: string): MemberRevision | undefined {
return (
this.sourceMembers.get(key) ??
this.sourceMembers.get(
getDecodedKeyIfSourceMembersHas({ sourceMembers: this.sourceMembers, key, logger: this.logger })
)
);
return this.sourceMembers.get(key) ?? this.sourceMembers.get(this.resolveDecodedKey(key));
}

private setMemberRevision(key: string, sourceMember: MemberRevision): void {
const sourceMembers = this.sourceMembers;
const matchingKey = sourceMembers.get(key)
? key
: getDecodedKeyIfSourceMembersHas({ sourceMembers, key, logger: this.logger });
const matchingKey = this.sourceMembers.has(key) ? key : this.resolveDecodedKey(key);
this.sourceMembers.set(matchingKey, { ...sourceMember, MemberName: decodeURIComponent(sourceMember.MemberName) });
// incrementally update the index instead of invalidating (avoids O(n) rebuild per insert)
if (this.decodedKeyIndex) {
this.decodedKeyIndex.set(decodeURIComponent(matchingKey), matchingKey);
}
}

/** O(1) decoded-key lookup via lazily-built reverse index */
private resolveDecodedKey(key: string): string {
this.decodedKeyIndex ??= buildDecodedKeyIndex(this.sourceMembers);
return resolveDecodedKey(key, this.decodedKeyIndex, this.logger);
}
}

Expand All @@ -521,31 +526,19 @@ export const remoteChangeElementToChangeResult = (
});
};

/**
*
* iterate SourceMember keys and compare their decoded value with the decoded key.
* if there's a match, return the matching decoded key, otherwise, return the original key
*/
const getDecodedKeyIfSourceMembersHas = ({
key,
sourceMembers,
logger,
}: {
sourceMembers: Map<string, MemberRevision>;
key: string;
logger: PinoLogger;
}): string => {
/** Build a reverse index: decoded key → stored key for O(1) URI-encoded key resolution */
const buildDecodedKeyIndex = (sourceMembers: Map<string, MemberRevision>): Map<string, string> =>
new Map(Array.from(sourceMembers.keys()).map((k) => [decodeURIComponent(k), k]));

/** Resolve a key against the decoded-key index, returning the stored key if found */
const resolveDecodedKey = (key: string, index: Map<string, string>, logger: PinoLogger): string => {
try {
const originalKeyDecoded = decodeURIComponent(key);
const match = Array.from(sourceMembers.keys()).find(
(memberKey) => decodeURIComponent(memberKey) === originalKeyDecoded
);
const match = index.get(decodeURIComponent(key));
if (match) {
logger.debug(`${match} matches already tracked member: ${key}`);
return match;
}
} catch (e: unknown) {
// Log the error and the key
const errMsg = e instanceof Error ? e.message : isString(e) ? e : 'unknown';
logger.debug(`Could not decode metadata key: ${key} due to: ${errMsg}`);
}
Expand Down
71 changes: 39 additions & 32 deletions yarn.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -742,6 +742,31 @@
ts-retry-promise "^0.8.1"
zod "^4.1.12"

"@salesforce/core@^8.28.1":
version "8.28.1"
resolved "https://registry.yarnpkg.com/@salesforce/core/-/core-8.28.1.tgz#d83f8d0a8a83057c5a363aa6b5211b583fb9206b"
integrity sha512-k9lPsULo+lOEZvpm1J1nJOFwKp5O5IfNqya7pw627QdKGcsWZm6v9caVHKUX9IjyB+S3dasNqaZT5O7l76C4oQ==
dependencies:
"@jsforce/jsforce-node" "^3.10.13"
"@salesforce/kit" "^3.2.4"
"@salesforce/ts-types" "^2.0.12"
ajv "^8.18.0"
change-case "^4.1.2"
fast-levenshtein "^3.0.0"
faye "^1.4.1"
form-data "^4.0.4"
js2xmlparser "^4.0.1"
jsonwebtoken "9.0.3"
jszip "3.10.1"
memfs "4.38.1"
pino "^9.7.0"
pino-abstract-transport "^1.2.0"
pino-pretty "^11.3.0"
proper-lockfile "^4.1.2"
semver "^7.7.3"
ts-retry-promise "^0.8.1"
zod "^4.1.12"

"@salesforce/dev-config@^4.3.1":
version "4.3.1"
resolved "https://registry.npmjs.org/@salesforce/dev-config/-/dev-config-4.3.1.tgz"
Expand Down Expand Up @@ -786,6 +811,13 @@
dependencies:
"@salesforce/ts-types" "^2.0.12"

"@salesforce/kit@^3.2.6":
version "3.2.6"
resolved "https://registry.yarnpkg.com/@salesforce/kit/-/kit-3.2.6.tgz#6a6c13b463b51694a43d61094af67171086ed4f5"
integrity sha512-O8S4LWerHa9Zosqh+IoQjgLtpxMOfObRxaRnUdRV4MLtFUi+bQxQiyFvve6eEaBaMP1b1xVDQpvSvQ+PXEDGFQ==
dependencies:
"@salesforce/ts-types" "^2.0.12"

"@salesforce/prettier-config@^0.0.3":
version "0.0.3"
resolved "https://registry.npmjs.org/@salesforce/prettier-config/-/prettier-config-0.0.3.tgz"
Expand All @@ -796,10 +828,10 @@
resolved "https://registry.yarnpkg.com/@salesforce/schemas/-/schemas-1.10.3.tgz#52c867fdd60679cf216110aa49542b7ad391f5d1"
integrity sha512-FKfvtrYTcvTXE9advzS25/DEY9yJhEyLvStm++eQFtnAaX1pe4G3oGHgiQ0q55BM5+0AlCh0+0CVtQv1t4oJRA==

"@salesforce/source-deploy-retrieve@^12.32.1":
version "12.32.1"
resolved "https://registry.yarnpkg.com/@salesforce/source-deploy-retrieve/-/source-deploy-retrieve-12.32.1.tgz#9d027b0fb9e1ef61ebb896af804109518f71bd19"
integrity sha512-q3u6bxgv3b4UscVvtWXCllgf5WouQd86IG7Kg76kyFryTTMBoSJIygRXNm5EMwlsHwBvfJhTn8zdpesJ1HkgPQ==
"@salesforce/source-deploy-retrieve@^12.32.3":
version "12.32.3"
resolved "https://registry.yarnpkg.com/@salesforce/source-deploy-retrieve/-/source-deploy-retrieve-12.32.3.tgz#274917ccc900f5ee624246ad5e5b0d024c075b5e"
integrity sha512-fHB4SAelLJPim4GA/xI0ob9aXVosIISGM3nkyxVif3zj9XAa6F5xzof1SbtM2QQ4PUXl+xkr6dwcRMcVuH26kQ==
dependencies:
"@salesforce/core" "^8.27.1"
"@salesforce/kit" "^3.2.4"
Expand Down Expand Up @@ -5834,16 +5866,7 @@ stop-iteration-iterator@^1.1.0:
es-errors "^1.3.0"
internal-slot "^1.1.0"

"string-width-cjs@npm:string-width@^4.2.0":
version "4.2.3"
resolved "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz"
integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
dependencies:
emoji-regex "^8.0.0"
is-fullwidth-code-point "^3.0.0"
strip-ansi "^6.0.1"

string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3:
"string-width-cjs@npm:string-width@^4.2.0", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3:
version "4.2.3"
resolved "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz"
integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
Expand Down Expand Up @@ -5915,14 +5938,7 @@ stringify-entities@^4.0.0:
character-entities-html4 "^2.0.0"
character-entities-legacy "^3.0.0"

"strip-ansi-cjs@npm:strip-ansi@^6.0.1":
version "6.0.1"
resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz"
integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
dependencies:
ansi-regex "^5.0.1"

strip-ansi@6.0.1, strip-ansi@^6.0.0, strip-ansi@^6.0.1:
"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@6.0.1, strip-ansi@^6.0.0, strip-ansi@^6.0.1:
version "6.0.1"
resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz"
integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
Expand Down Expand Up @@ -6524,7 +6540,7 @@ workerpool@^6.5.1:
resolved "https://registry.npmjs.org/workerpool/-/workerpool-6.5.1.tgz"
integrity sha512-Fs4dNYcsdpYSAfVxhnl1L5zTksjvOJxtC5hzMNl+1t9B8hTJTdKDyZ5ju7ztgPy+ft9tBFXoOlDNiOT9WUXZlA==

"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0":
"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0:
version "7.0.0"
resolved "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz"
integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
Expand All @@ -6542,15 +6558,6 @@ wrap-ansi@^6.2.0:
string-width "^4.1.0"
strip-ansi "^6.0.0"

wrap-ansi@^7.0.0:
version "7.0.0"
resolved "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz"
integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
dependencies:
ansi-styles "^4.0.0"
string-width "^4.1.0"
strip-ansi "^6.0.0"

wrap-ansi@^8.1.0:
version "8.1.0"
resolved "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz"
Expand Down
Loading