From f078b6e1c05e22c95475101a475a6b4055b89aee Mon Sep 17 00:00:00 2001 From: Joey Wunderlich Date: Thu, 28 May 2026 10:52:31 -0700 Subject: [PATCH 1/4] fix caching regression / clearing of all caches --- webapp/src/idbworkspace.ts | 49 +++++++++++++++++++++++++++++++------- 1 file changed, 41 insertions(+), 8 deletions(-) diff --git a/webapp/src/idbworkspace.ts b/webapp/src/idbworkspace.ts index a26868b1ef32..3073fa86a4de 100644 --- a/webapp/src/idbworkspace.ts +++ b/webapp/src/idbworkspace.ts @@ -464,20 +464,34 @@ export function initGitHubDb() { repopath = repopath.toLowerCase(); const cache = await getGitHubCacheAsync(); - const id = this.latestVersionCacheKey(repopath); + let cachedVersion: string = null; try { const entry = await cache.getAsync(id); + cachedVersion = entry?.version; - if (entry && Date.now() - entry.cacheTime < GITHUB_TUTORIAL_CACHE_EXPIRATION_MILLIS) { - return entry.version; + if (cachedVersion && entry && !isExpired(entry)) { + return cachedVersion; } } catch (e) { } - const version = await this.mem.latestVersionAsync(repopath, config); + let version: string; + try { + version = await this.mem.latestVersionAsync(repopath, config); + } + catch (e) { + if (cachedVersion && isOfflineError(e)) { + pxt.debug(`github offline cache hit ${id}`); + return cachedVersion; + } + throw e; + } + if (!version) { + return cachedVersion; + } await this.cacheLatestVersionAsync(cache, repopath, version); @@ -561,7 +575,17 @@ export function initGitHubDb() { return readFromCache(); } - const tutorialResponse = await pxt.github.downloadMarkdownTutorialInfoAsync(repopath, tag, undefined, existing?.etag); + let tutorialResponse: { resp?: pxt.github.GHTutorialResponse, etag?: string }; + try { + tutorialResponse = await pxt.github.downloadMarkdownTutorialInfoAsync(repopath, tag, undefined, existing?.etag); + } + catch (e) { + if (existing && isOfflineError(e)) { + pxt.debug(`github offline tutorial cache hit ${id}`); + return readFromCache(); + } + throw e; + } const body = tutorialResponse.resp; @@ -741,12 +765,13 @@ export function initGitHubDb() { return repopath.toLowerCase(); } - async function purgeExpiredEntriesAsync() { + async function purgeExpiredTutorialEntriesAsync() { const cache = await getGitHubCacheAsync(); const entries = await cache.getAllAsync(); - const expired = entries.filter(isExpired); + // package and config entries are used for offline hex imports + const expired = entries.filter(isExpiredTutorialEntry); if (expired.length) { for (const entry of expired) { @@ -755,13 +780,21 @@ export function initGitHubDb() { } } + function isExpiredTutorialEntry(entry: GitHubCacheEntry) { + return entry.id.indexOf("tutorial-") === 0 && isExpired(entry); + } + function isExpired(entry: GitHubCacheEntry) { return !!entry.cacheTime && Date.now() - entry.cacheTime >= GITHUB_TUTORIAL_CACHE_EXPIRATION_MILLIS; } + function isOfflineError(e: any) { + return e?.isOffline || e?.statusCode === 0 || !pxt.Cloud.isOnline(); + } + if (!/skipgithubcache=1/i.test(window.location.href)) { pxt.github.db = new GithubDb(); - /* await */ purgeExpiredEntriesAsync(); + /* await */ purgeExpiredTutorialEntriesAsync(); } } From b3b2e9d146e54f7cf335d465e2b6b822098b7f6c Mon Sep 17 00:00:00 2001 From: Joey Wunderlich Date: Fri, 29 May 2026 10:05:55 -0700 Subject: [PATCH 2/4] glue in extensions to hex as well in extra space --- cli/cli.ts | 4 +- pxtcompiler/simpledriver.ts | 4 +- pxtlib/github.ts | 104 ++++++++++++++++++++++-------------- pxtlib/main.ts | 1 + pxtlib/package.ts | 76 ++++++++++++++++++++------ webapp/src/idbworkspace.ts | 21 ++++++-- webapp/src/workspace.ts | 36 ++++++++++++- 7 files changed, 182 insertions(+), 64 deletions(-) diff --git a/cli/cli.ts b/cli/cli.ts index c16596e9fcb0..4c5f27fdf744 100644 --- a/cli/cli.ts +++ b/cli/cli.ts @@ -276,8 +276,8 @@ class FileGithubDb implements pxt.github.IGithubDb { return this.loadAsync(repopath, tag, "pxt", (r, t) => this.db.loadConfigAsync(r, t)); } - loadPackageAsync(repopath: string, tag: string): Promise { - return this.loadAsync(repopath, tag, "pkg", (r, t) => this.db.loadPackageAsync(r, t)); + loadPackageAsync(repopath: string, tag: string, backupScriptText?: pxt.Map): Promise { + return this.loadAsync(repopath, tag, "pkg", (r, t) => this.db.loadPackageAsync(r, t, backupScriptText)); } loadTutorialMarkdown(repopath: string, tag?: string): Promise { diff --git a/pxtcompiler/simpledriver.ts b/pxtcompiler/simpledriver.ts index 17cba6698094..170b5c01e047 100644 --- a/pxtcompiler/simpledriver.ts +++ b/pxtcompiler/simpledriver.ts @@ -52,8 +52,8 @@ namespace pxt { return this.loadAsync(repopath, tag, "pxt", (r, t) => this.db.loadConfigAsync(r, t)); } - loadPackageAsync(repopath: string, tag: string): Promise { - return this.loadAsync(repopath, tag, "pkg", (r, t) => this.db.loadPackageAsync(r, t)); + loadPackageAsync(repopath: string, tag: string, backupScriptText?: pxt.Map): Promise { + return this.loadAsync(repopath, tag, "pkg", (r, t) => this.db.loadPackageAsync(r, t, backupScriptText)); } loadTutorialMarkdown(repopath: string, tag?: string): Promise { diff --git a/pxtlib/github.ts b/pxtlib/github.ts index 82610438cfeb..d919b9ed8457 100644 --- a/pxtlib/github.ts +++ b/pxtlib/github.ts @@ -145,13 +145,22 @@ namespace pxt.github { export interface CachedPackage { files: Map; + backupCopy?: boolean; + } + + function cachedPackageFromBackup(backupScriptText?: pxt.Map): CachedPackage { + if (!backupScriptText) return undefined; + return { + files: U.clone(backupScriptText), + backupCopy: true + }; } // caching export interface IGithubDb { latestVersionAsync(repopath: string, config: PackagesConfig): Promise; loadConfigAsync(repopath: string, tag: string): Promise; - loadPackageAsync(repopath: string, tag: string): Promise; + loadPackageAsync(repopath: string, tag: string, backupScriptText?: pxt.Map): Promise; loadTutorialMarkdown(repopath: string, tag?: string): Promise; cacheReposAsync(response: GHTutorialResponse): Promise; } @@ -273,7 +282,7 @@ namespace pxt.github { return resolved } - async loadPackageAsync(repopath: string, tag: string): Promise { + async loadPackageAsync(repopath: string, tag: string, backupScriptText?: pxt.Map): Promise { if (!tag) { pxt.debug(`load pkg: default to master branch`) tag = "master"; @@ -289,41 +298,42 @@ namespace pxt.github { } // try using github apis - return await this.githubLoadPackageAsync(repopath, tag); + return await this.githubLoadPackageAsync(repopath, tag, backupScriptText); } - private githubLoadPackageAsync(repopath: string, tag: string): Promise { - return tagToShaAsync(repopath, tag) - .then(sha => { - // cache lookup - const key = `${repopath}/${sha}`; - let res = this.packages[key]; - if (res) { - pxt.debug(`github cache ${repopath}/${tag}/text`); - return Promise.resolve(U.clone(res)); - } + private async githubLoadPackageAsync(repopath: string, tag: string, backupScriptText?: pxt.Map): Promise { + try { + const sha = await tagToShaAsync(repopath, tag); + + // cache lookup + const key = `${repopath}/${sha}`; + let res = this.packages[key]; + if (res) { + pxt.debug(`github cache ${repopath}/${tag}/text`); + return U.clone(res); + } - // load and cache - pxt.log(`Downloading ${repopath}/${tag} -> ${sha}`) - return downloadTextAsync(repopath, sha, pxt.CONFIG_NAME) - .then(pkg => { - const current: CachedPackage = { - files: {} - } - current.files[pxt.CONFIG_NAME] = pkg - const cfg: pxt.PackageConfig = JSON.parse(pkg) - return U.promiseMapAll(pxt.allPkgFiles(cfg).slice(1), - fn => downloadTextAsync(repopath, sha, fn) - .then(text => { - current.files[fn] = text - })) - .then(() => { - // cache! - this.packages[key] = current; - return U.clone(current); - }) - }) - }) + // load and cache + pxt.log(`Downloading ${repopath}/${tag} -> ${sha}`) + const pkg = await downloadTextAsync(repopath, sha, pxt.CONFIG_NAME); + const current: CachedPackage = { + files: {} + } + current.files[pxt.CONFIG_NAME] = pkg + const cfg: pxt.PackageConfig = JSON.parse(pkg) + await U.promiseMapAll(pxt.allPkgFiles(cfg).slice(1), async fn => { + current.files[fn] = await downloadTextAsync(repopath, sha, fn); + }); + + // cache! + this.packages[key] = current; + return U.clone(current); + } + catch (e) { + const backup = cachedPackageFromBackup(backupScriptText); + if (backup) return backup; + throw e; + } } async loadTutorialMarkdown(repopath: string, tag?: string) { @@ -731,7 +741,7 @@ namespace pxt.github { return await db.loadConfigAsync(repopath, tag) } - export async function downloadPackageAsync(repoWithTag: string, config: pxt.PackagesConfig): Promise { + export async function downloadPackageAsync(repoWithTag: string, config: pxt.PackagesConfig, backupScriptText?: pxt.Map): Promise { const p = parseRepoId(repoWithTag) if (!p) { pxt.log('Unknown GitHub syntax'); @@ -746,9 +756,16 @@ namespace pxt.github { // always try to upgrade unbound versions if (!p.tag) { - p.tag = await db.latestVersionAsync(p.slug, config) + try { + p.tag = await db.latestVersionAsync(p.slug, config) + } + catch (e) { + const backup = cachedPackageFromBackup(backupScriptText); + if (backup) return backup; + throw e; + } } - const cached = await db.loadPackageAsync(p.fullName, p.tag) + const cached = await db.loadPackageAsync(p.fullName, p.tag, backupScriptText) const dv = upgradedDisablesVariants(config, repoWithTag) if (dv) { const cfg = Package.parseAndValidConfig(cached.files[pxt.CONFIG_NAME]) @@ -775,7 +792,11 @@ namespace pxt.github { return { version, config }; } - export async function cacheProjectDependenciesAsync(cfg: pxt.PackageConfig): Promise { + export async function cacheProjectDependenciesAsync(cfg: pxt.PackageConfig, backupExtensions?: pxt.Map>): Promise { + await cacheProjectDependenciesCoreAsync(cfg, {}, backupExtensions); + } + + async function cacheProjectDependenciesCoreAsync(cfg: pxt.PackageConfig, checked: pxt.Map, backupExtensions?: pxt.Map>): Promise { const ghExtensions = Object.keys(cfg.dependencies) ?.filter(dep => isGithubId(cfg.dependencies[dep])); @@ -786,10 +807,15 @@ namespace pxt.github { ghExtensions.map( async ext => { const extSrc = cfg.dependencies[ext]; - const ghPkg = await downloadPackageAsync(extSrc, pkgConfig); + if (checked[extSrc]) return; + checked[extSrc] = true; + + const ghPkg = await downloadPackageAsync(extSrc, pkgConfig, backupExtensions?.[extSrc]); if (!ghPkg) { throw new Error(lf("Cannot load extension {0} from {1}", ext, extSrc)); } + const ghPkgCfg = Package.parseAndValidConfig(ghPkg.files[pxt.CONFIG_NAME]); + if (ghPkgCfg) await cacheProjectDependenciesCoreAsync(ghPkgCfg, checked, backupExtensions); } ) ); diff --git a/pxtlib/main.ts b/pxtlib/main.ts index c43fa9e77506..dde9def5c8ed 100644 --- a/pxtlib/main.ts +++ b/pxtlib/main.ts @@ -523,6 +523,7 @@ namespace pxt { export const TUTORIAL_CODE_STOP = "_onCodeStop.ts"; export const TUTORIAL_INFO_FILE = "tutorial-info-cache.json"; export const TUTORIAL_CUSTOM_TS = "tutorial.custom.ts"; + export const PACKAGED_EXTENSIONS = "_packaged-extensions.json"; export const BREAKPOINT_TABLET = 991; // TODO (shakao) revisit when tutorial stuff is more settled export const PALETTES_FILE = "_palettes.json"; export const HISTORY_FILE = "_history"; diff --git a/pxtlib/package.ts b/pxtlib/package.ts index 47254756dc80..7e9ed669b553 100644 --- a/pxtlib/package.ts +++ b/pxtlib/package.ts @@ -1279,7 +1279,7 @@ namespace pxt { !opts.target.isNative if (!noFileEmbed) { - const files = await this.filesToBePublishedAsync(true) + const files = await this.filesToBePublishedAsync(true, !appTarget.compile.useUF2) const headerString = JSON.stringify({ name: this.config.name, comment: this.config.description, @@ -1355,24 +1355,66 @@ namespace pxt { return cfg; } - filesToBePublishedAsync(allowPrivate = false) { + async filesToBePublishedAsync(allowPrivate = false, packExternalExtensions = false): Promise> { const files: Map = {}; - return this.loadAsync() - .then(() => { - if (!allowPrivate && !this.config.public) - U.userError('Only packages with "public":true can be published') - const cfg = this.prepareConfigToBePublished(); - files[pxt.CONFIG_NAME] = pxt.Package.stringifyConfig(cfg); - for (let f of this.getFiles()) { - // already stored - if (f === pxt.CONFIG_NAME || f === HISTORY_FILE) continue; - let str = this.readFile(f) - if (str == null) - U.userError("referenced file missing: " + f) - files[f] = str + await this.loadAsync(); + + if (!allowPrivate && !this.config.public) + U.userError('Only packages with "public":true can be published') + + const cfg = this.prepareConfigToBePublished(); + files[pxt.CONFIG_NAME] = pxt.Package.stringifyConfig(cfg); + for (let f of this.getFiles()) { + // already stored + if (f === pxt.CONFIG_NAME || f === HISTORY_FILE) continue; + let str = this.readFile(f) + if (str == null) + U.userError("referenced file missing: " + f) + files[f] = str + } + + if (packExternalExtensions) { + const packagedExtensions = this.packagedExternalExtensions(); + if (Object.keys(packagedExtensions).length) + files[pxt.PACKAGED_EXTENSIONS] = JSON.stringify(packagedExtensions); + } + + return U.sortObjectFields(files) + } + + private packagedExternalExtensions(): pxt.Map> { + const packaged: pxt.Map> = {}; + const seen: pxt.Map = {}; + + const packDeps = (pkg: Package) => { + for (const dep of pkg.resolvedDependencies()) { + if (!dep) continue; + + const seenKey = `${dep.id}:${dep.version()}`; + if (seen[seenKey]) continue; + seen[seenKey] = true; + + if (dep.verProtocol() === "github" || dep.verProtocol() === "pub") { + const depFiles: pxt.Map = {}; + for (const fn of dep.getFiles()) { + if (fn === pxt.CONFIG_NAME || fn === HISTORY_FILE) continue; + const content = dep.readFile(fn); + if (content != null) depFiles[fn] = content; + } + depFiles[pxt.CONFIG_NAME] = pxt.Package.stringifyConfig(dep.config); + packaged[dep._verspec] = depFiles; + if (dep.version() !== dep._verspec) + packaged[dep.version()] = depFiles; + if (dep.verProtocol() === "pub") + packaged[dep.verArgument()] = depFiles; } - return U.sortObjectFields(files) - }) + + packDeps(dep); + } + } + + packDeps(this); + return packaged; } saveToJsonAsync(): Promise { diff --git a/webapp/src/idbworkspace.ts b/webapp/src/idbworkspace.ts index 3073fa86a4de..6b714edfc640 100644 --- a/webapp/src/idbworkspace.ts +++ b/webapp/src/idbworkspace.ts @@ -522,7 +522,7 @@ export function initGitHubDb() { } } - async loadPackageAsync(repopath: string, tag: string): Promise { + async loadPackageAsync(repopath: string, tag: string, backupScriptText?: pxt.Map): Promise { repopath = repopath.toLowerCase() if (!tag) { pxt.debug(`dep: default to master`) @@ -530,7 +530,7 @@ export function initGitHubDb() { } // don't cache master if (tag == "master") - return this.mem.loadPackageAsync(repopath, tag); + return this.loadPackageFromMemoryAsync(repopath, tag, backupScriptText); const id = this.packageCacheKey(repopath, tag); const cache = await getGitHubCacheAsync(); @@ -542,7 +542,7 @@ export function initGitHubDb() { } catch (e) { pxt.debug(`github offline cache miss ${id}`); - const p = await this.mem.loadPackageAsync(repopath, tag); + const p = await this.loadPackageFromMemoryAsync(repopath, tag, backupScriptText); await this.cachePackageAsync(cache, repopath, tag, p); @@ -725,6 +725,21 @@ export function initGitHubDb() { } } + private async loadPackageFromMemoryAsync(repopath: string, tag: string, backupScriptText?: pxt.Map): Promise { + try { + return await this.mem.loadPackageAsync(repopath, tag, backupScriptText); + } + catch (e) { + if (backupScriptText) { + return { + files: pxt.Util.clone(backupScriptText), + backupCopy: true + }; + } + throw e; + } + } + private async cacheTutorialResponseAsync(cache: pxt.BrowserUtils.IDBObjectStoreWrapper, repopath: string, tag?: string, etag?: string) { const id = this.tutorialCacheKey(repopath, tag); diff --git a/webapp/src/workspace.ts b/webapp/src/workspace.ts index 8a3bf4034740..b84692052bbc 100644 --- a/webapp/src/workspace.ts +++ b/webapp/src/workspace.ts @@ -770,11 +770,45 @@ export async function installAsync(h0: InstallHeader, text: ScriptText, dontOver pxt.shell.setEditorLanguagePref(cfg.preferredEditor); } - await pxt.github.cacheProjectDependenciesAsync(cfg) + let backupExtensionFiles: pxt.Map>; + if (text[pxt.PACKAGED_EXTENSIONS]) { + backupExtensionFiles = pxt.Util.jsonTryParse(text[pxt.PACKAGED_EXTENSIONS]); + delete text[pxt.PACKAGED_EXTENSIONS]; + } + + await cachePublishedScriptBackupsAsync(backupExtensionFiles); + await pxt.github.cacheProjectDependenciesAsync(cfg, backupExtensionFiles) await importAsync(h, text); return h; } +async function cachePublishedScriptBackupsAsync(backupExtensionFiles?: pxt.Map>) { + if (!backupExtensionFiles) return; + + const config = await pxt.packagesConfigAsync(); + const scriptCache = await getScriptCacheAsync(); + const seen: pxt.Map = {}; + + await Promise.all(Object.keys(backupExtensionFiles).map(async key => { + if (pxt.github.isGithubId(key)) return; + + const pubId = key.slice(0, 4) === "pub:" ? key.slice(4) : key; + if (!pubId || seen[pubId]) return; + seen[pubId] = true; + + const files = backupExtensionFiles[key]; + if (!files || !pxt.Package.parseAndValidConfig(files[pxt.CONFIG_NAME])) return; + + try { + const id = encodeURIComponent(pxt.github.upgradedPackageId(config, pubId)); + await scriptCache.setAsync({ id, files }); + } + catch (e) { + pxt.log("Unable to cache packaged extension in DB"); + } + })); +} + export async function renameAsync(h: Header, newName: string): Promise
{ const text = await getTextAsync(h.id); From c25dccd9444e16c09362dce990bbd412fa6cf5cd Mon Sep 17 00:00:00 2001 From: Joey Wunderlich Date: Fri, 29 May 2026 11:29:06 -0700 Subject: [PATCH 3/4] cache hex info as well, so ideally we can warm cache even when offline --- pxtlib/cpp.ts | 18 ++++++++++++++---- pxtlib/main.ts | 1 + pxtlib/package.ts | 18 ++++++++++++++++++ webapp/src/workspace.ts | 34 ++++++++++++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 4 deletions(-) diff --git a/pxtlib/cpp.ts b/pxtlib/cpp.ts index c01c39ee6bbb..4e72171153e5 100644 --- a/pxtlib/cpp.ts +++ b/pxtlib/cpp.ts @@ -1514,6 +1514,19 @@ namespace pxt.hexloader { }) } + export function stringifyHexInfoForCache(hexInfo: pxtc.HexInfo) { + if (!hexInfo?.hex) return undefined; + + const cachedMeta = U.clone(hexInfo); + cachedMeta.hex = compressHex(cachedMeta.hex); + return JSON.stringify(cachedMeta); + } + + export function storeHexInfoCacheEntryAsync(host: Host, sha: string, cachedHexInfo: string) { + if (!sha || !cachedHexInfo) return Promise.resolve(); + return storeWithLimitAsync(host, "hex-keys", "hex-" + sha, cachedHexInfo); + } + export function getHexInfoAsync(host: Host, extInfo: pxtc.ExtensionInfo, cloudModule?: any): Promise { if (!extInfo.sha) return Promise.resolve(null) @@ -1543,10 +1556,7 @@ namespace pxt.hexloader { else { return downloadHexInfoAsync(extInfo) .then(meta => { - let origHex = meta.hex - meta.hex = compressHex(meta.hex) - let store = JSON.stringify(meta) - meta.hex = origHex + let store = stringifyHexInfoForCache(meta) return storeWithLimitAsync(host, "hex-keys", key, store) .then(() => meta) }).catch(e => { diff --git a/pxtlib/main.ts b/pxtlib/main.ts index dde9def5c8ed..0cd5f1cc65fa 100644 --- a/pxtlib/main.ts +++ b/pxtlib/main.ts @@ -524,6 +524,7 @@ namespace pxt { export const TUTORIAL_INFO_FILE = "tutorial-info-cache.json"; export const TUTORIAL_CUSTOM_TS = "tutorial.custom.ts"; export const PACKAGED_EXTENSIONS = "_packaged-extensions.json"; + export const PACKAGED_EXT_INFO = "_packaged-ext-info.json"; export const BREAKPOINT_TABLET = 991; // TODO (shakao) revisit when tutorial stuff is more settled export const PALETTES_FILE = "_palettes.json"; export const HISTORY_FILE = "_history"; diff --git a/pxtlib/package.ts b/pxtlib/package.ts index 7e9ed669b553..e6f466e165b2 100644 --- a/pxtlib/package.ts +++ b/pxtlib/package.ts @@ -1280,6 +1280,9 @@ namespace pxt { if (!noFileEmbed) { const files = await this.filesToBePublishedAsync(true, !appTarget.compile.useUF2) + const packagedExtInfo = this.packagedExtensionHexInfo(opts); + if (Object.keys(packagedExtInfo).length) + files[pxt.PACKAGED_EXT_INFO] = JSON.stringify(packagedExtInfo); const headerString = JSON.stringify({ name: this.config.name, comment: this.config.description, @@ -1417,6 +1420,21 @@ namespace pxt { return packaged; } + private packagedExtensionHexInfo(opts: pxtc.CompileOptions): pxt.Map { + const packaged: pxt.Map = {}; + const targets = [opts, ...(opts.otherMultiVariants || [])]; + + for (const target of targets) { + const extInfo = target.extinfo; + if (!extInfo?.sha || !extInfo.hexinfo?.hex) continue; + + const serialized = pxt.hexloader.stringifyHexInfoForCache(extInfo.hexinfo); + if (serialized) packaged[extInfo.sha] = serialized; + } + + return packaged; + } + saveToJsonAsync(): Promise { return this.filesToBePublishedAsync(true) .then(files => { diff --git a/webapp/src/workspace.ts b/webapp/src/workspace.ts index b84692052bbc..cd1d6ba8d5b4 100644 --- a/webapp/src/workspace.ts +++ b/webapp/src/workspace.ts @@ -775,7 +775,13 @@ export async function installAsync(h0: InstallHeader, text: ScriptText, dontOver backupExtensionFiles = pxt.Util.jsonTryParse(text[pxt.PACKAGED_EXTENSIONS]); delete text[pxt.PACKAGED_EXTENSIONS]; } + let packagedExtInfo: pxt.Map; + if (text[pxt.PACKAGED_EXT_INFO]) { + packagedExtInfo = pxt.Util.jsonTryParse(text[pxt.PACKAGED_EXT_INFO]); + delete text[pxt.PACKAGED_EXT_INFO]; + } + await cachePackagedExtInfoAsync(packagedExtInfo); await cachePublishedScriptBackupsAsync(backupExtensionFiles); await pxt.github.cacheProjectDependenciesAsync(cfg, backupExtensionFiles) await importAsync(h, text); @@ -809,6 +815,34 @@ async function cachePublishedScriptBackupsAsync(backupExtensionFiles?: pxt.Map

) { + if (!packagedExtInfo) return; + + const hostCache = await indexedDBWorkspace.getObjectStoreAsync<{ id: string, val: string }>(indexedDBWorkspace.HOSTCACHE_TABLE); + const cacheHost = { + cacheStoreAsync: async (id: string, val: string) => { + await hostCache.setAsync({ id, val }); + }, + cacheGetAsync: async (id: string) => { + try { + return (await hostCache.getAsync(id)).val; + } + catch (e) { + return null; + } + } + } as pxt.Host; + + await Promise.all(Object.keys(packagedExtInfo).map(async sha => { + try { + await pxt.hexloader.storeHexInfoCacheEntryAsync(cacheHost, sha, packagedExtInfo[sha]); + } + catch (e) { + pxt.log("Unable to cache packaged extension hex info in DB"); + } + })); +} + export async function renameAsync(h: Header, newName: string): Promise

{ const text = await getTextAsync(h.id); From 7c725ac287531a58adf9ef4749b99642e6910ac3 Mon Sep 17 00:00:00 2001 From: Joey Wunderlich Date: Fri, 5 Jun 2026 10:00:40 -0700 Subject: [PATCH 4/4] backup -> fallback, clean up unused test marker, shallow clones instead of jsonifying --- cli/cli.ts | 4 +-- pxtcompiler/simpledriver.ts | 4 +-- pxtlib/cpp.ts | 6 +++-- pxtlib/github.ts | 50 ++++++++++++++++++++----------------- webapp/src/idbworkspace.ts | 15 ++++++----- webapp/src/workspace.ts | 16 ++++++------ 6 files changed, 50 insertions(+), 45 deletions(-) diff --git a/cli/cli.ts b/cli/cli.ts index 4c5f27fdf744..753ecedb0e45 100644 --- a/cli/cli.ts +++ b/cli/cli.ts @@ -276,8 +276,8 @@ class FileGithubDb implements pxt.github.IGithubDb { return this.loadAsync(repopath, tag, "pxt", (r, t) => this.db.loadConfigAsync(r, t)); } - loadPackageAsync(repopath: string, tag: string, backupScriptText?: pxt.Map): Promise { - return this.loadAsync(repopath, tag, "pkg", (r, t) => this.db.loadPackageAsync(r, t, backupScriptText)); + loadPackageAsync(repopath: string, tag: string, fallbackPackageFiles?: pxt.Map): Promise { + return this.loadAsync(repopath, tag, "pkg", (r, t) => this.db.loadPackageAsync(r, t, fallbackPackageFiles)); } loadTutorialMarkdown(repopath: string, tag?: string): Promise { diff --git a/pxtcompiler/simpledriver.ts b/pxtcompiler/simpledriver.ts index 170b5c01e047..b8e2537d44bc 100644 --- a/pxtcompiler/simpledriver.ts +++ b/pxtcompiler/simpledriver.ts @@ -52,8 +52,8 @@ namespace pxt { return this.loadAsync(repopath, tag, "pxt", (r, t) => this.db.loadConfigAsync(r, t)); } - loadPackageAsync(repopath: string, tag: string, backupScriptText?: pxt.Map): Promise { - return this.loadAsync(repopath, tag, "pkg", (r, t) => this.db.loadPackageAsync(r, t, backupScriptText)); + loadPackageAsync(repopath: string, tag: string, fallbackPackageFiles?: pxt.Map): Promise { + return this.loadAsync(repopath, tag, "pkg", (r, t) => this.db.loadPackageAsync(r, t, fallbackPackageFiles)); } loadTutorialMarkdown(repopath: string, tag?: string): Promise { diff --git a/pxtlib/cpp.ts b/pxtlib/cpp.ts index 4e72171153e5..cc17710f6c77 100644 --- a/pxtlib/cpp.ts +++ b/pxtlib/cpp.ts @@ -1517,8 +1517,10 @@ namespace pxt.hexloader { export function stringifyHexInfoForCache(hexInfo: pxtc.HexInfo) { if (!hexInfo?.hex) return undefined; - const cachedMeta = U.clone(hexInfo); - cachedMeta.hex = compressHex(cachedMeta.hex); + const cachedMeta = { + ...hexInfo, + hex: compressHex(hexInfo.hex) + }; return JSON.stringify(cachedMeta); } diff --git a/pxtlib/github.ts b/pxtlib/github.ts index d919b9ed8457..88d0887aa430 100644 --- a/pxtlib/github.ts +++ b/pxtlib/github.ts @@ -145,14 +145,18 @@ namespace pxt.github { export interface CachedPackage { files: Map; - backupCopy?: boolean; } - function cachedPackageFromBackup(backupScriptText?: pxt.Map): CachedPackage { - if (!backupScriptText) return undefined; + function copyCachedPackage(cachedPackage: CachedPackage): CachedPackage { return { - files: U.clone(backupScriptText), - backupCopy: true + files: { ...cachedPackage.files } + }; + } + + function cachedPackageFromFallback(fallbackPackageFiles?: pxt.Map): CachedPackage { + if (!fallbackPackageFiles) return undefined; + return { + files: { ...fallbackPackageFiles } }; } @@ -160,7 +164,7 @@ namespace pxt.github { export interface IGithubDb { latestVersionAsync(repopath: string, config: PackagesConfig): Promise; loadConfigAsync(repopath: string, tag: string): Promise; - loadPackageAsync(repopath: string, tag: string, backupScriptText?: pxt.Map): Promise; + loadPackageAsync(repopath: string, tag: string, fallbackPackageFiles?: pxt.Map): Promise; loadTutorialMarkdown(repopath: string, tag?: string): Promise; cacheReposAsync(response: GHTutorialResponse): Promise; } @@ -282,7 +286,7 @@ namespace pxt.github { return resolved } - async loadPackageAsync(repopath: string, tag: string, backupScriptText?: pxt.Map): Promise { + async loadPackageAsync(repopath: string, tag: string, fallbackPackageFiles?: pxt.Map): Promise { if (!tag) { pxt.debug(`load pkg: default to master branch`) tag = "master"; @@ -291,17 +295,17 @@ namespace pxt.github { // try using github proxy first if (hasProxy()) { try { - return await this.proxyWithCdnLoadPackageAsync(repopath, tag).then(v => U.clone(v)); + return await this.proxyWithCdnLoadPackageAsync(repopath, tag).then(copyCachedPackage); } catch (e) { ghProxyHandleException(e); } } // try using github apis - return await this.githubLoadPackageAsync(repopath, tag, backupScriptText); + return await this.githubLoadPackageAsync(repopath, tag, fallbackPackageFiles); } - private async githubLoadPackageAsync(repopath: string, tag: string, backupScriptText?: pxt.Map): Promise { + private async githubLoadPackageAsync(repopath: string, tag: string, fallbackPackageFiles?: pxt.Map): Promise { try { const sha = await tagToShaAsync(repopath, tag); @@ -310,7 +314,7 @@ namespace pxt.github { let res = this.packages[key]; if (res) { pxt.debug(`github cache ${repopath}/${tag}/text`); - return U.clone(res); + return copyCachedPackage(res); } // load and cache @@ -327,11 +331,11 @@ namespace pxt.github { // cache! this.packages[key] = current; - return U.clone(current); + return copyCachedPackage(current); } catch (e) { - const backup = cachedPackageFromBackup(backupScriptText); - if (backup) return backup; + const fallbackPackage = cachedPackageFromFallback(fallbackPackageFiles); + if (fallbackPackage) return fallbackPackage; throw e; } } @@ -741,7 +745,7 @@ namespace pxt.github { return await db.loadConfigAsync(repopath, tag) } - export async function downloadPackageAsync(repoWithTag: string, config: pxt.PackagesConfig, backupScriptText?: pxt.Map): Promise { + export async function downloadPackageAsync(repoWithTag: string, config: pxt.PackagesConfig, fallbackPackageFiles?: pxt.Map): Promise { const p = parseRepoId(repoWithTag) if (!p) { pxt.log('Unknown GitHub syntax'); @@ -760,12 +764,12 @@ namespace pxt.github { p.tag = await db.latestVersionAsync(p.slug, config) } catch (e) { - const backup = cachedPackageFromBackup(backupScriptText); - if (backup) return backup; + const fallbackPackage = cachedPackageFromFallback(fallbackPackageFiles); + if (fallbackPackage) return fallbackPackage; throw e; } } - const cached = await db.loadPackageAsync(p.fullName, p.tag, backupScriptText) + const cached = await db.loadPackageAsync(p.fullName, p.tag, fallbackPackageFiles) const dv = upgradedDisablesVariants(config, repoWithTag) if (dv) { const cfg = Package.parseAndValidConfig(cached.files[pxt.CONFIG_NAME]) @@ -792,11 +796,11 @@ namespace pxt.github { return { version, config }; } - export async function cacheProjectDependenciesAsync(cfg: pxt.PackageConfig, backupExtensions?: pxt.Map>): Promise { - await cacheProjectDependenciesCoreAsync(cfg, {}, backupExtensions); + export async function cacheProjectDependenciesAsync(cfg: pxt.PackageConfig, fallbackPackageFilesById?: pxt.Map>): Promise { + await cacheProjectDependenciesCoreAsync(cfg, {}, fallbackPackageFilesById); } - async function cacheProjectDependenciesCoreAsync(cfg: pxt.PackageConfig, checked: pxt.Map, backupExtensions?: pxt.Map>): Promise { + async function cacheProjectDependenciesCoreAsync(cfg: pxt.PackageConfig, checked: pxt.Map, fallbackPackageFilesById?: pxt.Map>): Promise { const ghExtensions = Object.keys(cfg.dependencies) ?.filter(dep => isGithubId(cfg.dependencies[dep])); @@ -810,12 +814,12 @@ namespace pxt.github { if (checked[extSrc]) return; checked[extSrc] = true; - const ghPkg = await downloadPackageAsync(extSrc, pkgConfig, backupExtensions?.[extSrc]); + const ghPkg = await downloadPackageAsync(extSrc, pkgConfig, fallbackPackageFilesById?.[extSrc]); if (!ghPkg) { throw new Error(lf("Cannot load extension {0} from {1}", ext, extSrc)); } const ghPkgCfg = Package.parseAndValidConfig(ghPkg.files[pxt.CONFIG_NAME]); - if (ghPkgCfg) await cacheProjectDependenciesCoreAsync(ghPkgCfg, checked, backupExtensions); + if (ghPkgCfg) await cacheProjectDependenciesCoreAsync(ghPkgCfg, checked, fallbackPackageFilesById); } ) ); diff --git a/webapp/src/idbworkspace.ts b/webapp/src/idbworkspace.ts index 6b714edfc640..f86e391e26a1 100644 --- a/webapp/src/idbworkspace.ts +++ b/webapp/src/idbworkspace.ts @@ -522,7 +522,7 @@ export function initGitHubDb() { } } - async loadPackageAsync(repopath: string, tag: string, backupScriptText?: pxt.Map): Promise { + async loadPackageAsync(repopath: string, tag: string, fallbackPackageFiles?: pxt.Map): Promise { repopath = repopath.toLowerCase() if (!tag) { pxt.debug(`dep: default to master`) @@ -530,7 +530,7 @@ export function initGitHubDb() { } // don't cache master if (tag == "master") - return this.loadPackageFromMemoryAsync(repopath, tag, backupScriptText); + return this.loadPackageFromMemoryAsync(repopath, tag, fallbackPackageFiles); const id = this.packageCacheKey(repopath, tag); const cache = await getGitHubCacheAsync(); @@ -542,7 +542,7 @@ export function initGitHubDb() { } catch (e) { pxt.debug(`github offline cache miss ${id}`); - const p = await this.loadPackageFromMemoryAsync(repopath, tag, backupScriptText); + const p = await this.loadPackageFromMemoryAsync(repopath, tag, fallbackPackageFiles); await this.cachePackageAsync(cache, repopath, tag, p); @@ -725,15 +725,14 @@ export function initGitHubDb() { } } - private async loadPackageFromMemoryAsync(repopath: string, tag: string, backupScriptText?: pxt.Map): Promise { + private async loadPackageFromMemoryAsync(repopath: string, tag: string, fallbackPackageFiles?: pxt.Map): Promise { try { - return await this.mem.loadPackageAsync(repopath, tag, backupScriptText); + return await this.mem.loadPackageAsync(repopath, tag, fallbackPackageFiles); } catch (e) { - if (backupScriptText) { + if (fallbackPackageFiles) { return { - files: pxt.Util.clone(backupScriptText), - backupCopy: true + files: { ...fallbackPackageFiles } }; } throw e; diff --git a/webapp/src/workspace.ts b/webapp/src/workspace.ts index cd1d6ba8d5b4..0722eb8e412c 100644 --- a/webapp/src/workspace.ts +++ b/webapp/src/workspace.ts @@ -770,9 +770,9 @@ export async function installAsync(h0: InstallHeader, text: ScriptText, dontOver pxt.shell.setEditorLanguagePref(cfg.preferredEditor); } - let backupExtensionFiles: pxt.Map>; + let packagedExtensionFiles: pxt.Map>; if (text[pxt.PACKAGED_EXTENSIONS]) { - backupExtensionFiles = pxt.Util.jsonTryParse(text[pxt.PACKAGED_EXTENSIONS]); + packagedExtensionFiles = pxt.Util.jsonTryParse(text[pxt.PACKAGED_EXTENSIONS]); delete text[pxt.PACKAGED_EXTENSIONS]; } let packagedExtInfo: pxt.Map; @@ -782,27 +782,27 @@ export async function installAsync(h0: InstallHeader, text: ScriptText, dontOver } await cachePackagedExtInfoAsync(packagedExtInfo); - await cachePublishedScriptBackupsAsync(backupExtensionFiles); - await pxt.github.cacheProjectDependenciesAsync(cfg, backupExtensionFiles) + await cachePackagedScriptExtensionsAsync(packagedExtensionFiles); + await pxt.github.cacheProjectDependenciesAsync(cfg, packagedExtensionFiles) await importAsync(h, text); return h; } -async function cachePublishedScriptBackupsAsync(backupExtensionFiles?: pxt.Map>) { - if (!backupExtensionFiles) return; +async function cachePackagedScriptExtensionsAsync(packagedExtensionFiles?: pxt.Map>) { + if (!packagedExtensionFiles) return; const config = await pxt.packagesConfigAsync(); const scriptCache = await getScriptCacheAsync(); const seen: pxt.Map = {}; - await Promise.all(Object.keys(backupExtensionFiles).map(async key => { + await Promise.all(Object.keys(packagedExtensionFiles).map(async key => { if (pxt.github.isGithubId(key)) return; const pubId = key.slice(0, 4) === "pub:" ? key.slice(4) : key; if (!pubId || seen[pubId]) return; seen[pubId] = true; - const files = backupExtensionFiles[key]; + const files = packagedExtensionFiles[key]; if (!files || !pxt.Package.parseAndValidConfig(files[pxt.CONFIG_NAME])) return; try {