diff --git a/cli/cli.ts b/cli/cli.ts index a385c2b3a89e..c3d71ea714fa 100644 --- a/cli/cli.ts +++ b/cli/cli.ts @@ -276,8 +276,8 @@ class FileGithubDb implements pxt.github.IGithubDb { return this.loadAsync(repopath, tag, "pxt", (r, t) => this.db.loadConfigAsync(r, t)); } - loadPackageAsync(repopath: string, tag: string): Promise { - return this.loadAsync(repopath, tag, "pkg", (r, t) => this.db.loadPackageAsync(r, t)); + loadPackageAsync(repopath: string, tag: string, fallbackPackageFiles?: pxt.Map): Promise { + return this.loadAsync(repopath, tag, "pkg", (r, t) => this.db.loadPackageAsync(r, t, fallbackPackageFiles)); } loadTutorialMarkdown(repopath: string, tag?: string): Promise { diff --git a/pxtcompiler/simpledriver.ts b/pxtcompiler/simpledriver.ts index 17cba6698094..b8e2537d44bc 100644 --- a/pxtcompiler/simpledriver.ts +++ b/pxtcompiler/simpledriver.ts @@ -52,8 +52,8 @@ namespace pxt { return this.loadAsync(repopath, tag, "pxt", (r, t) => this.db.loadConfigAsync(r, t)); } - loadPackageAsync(repopath: string, tag: string): Promise { - return this.loadAsync(repopath, tag, "pkg", (r, t) => this.db.loadPackageAsync(r, t)); + loadPackageAsync(repopath: string, tag: string, fallbackPackageFiles?: pxt.Map): Promise { + return this.loadAsync(repopath, tag, "pkg", (r, t) => this.db.loadPackageAsync(r, t, fallbackPackageFiles)); } loadTutorialMarkdown(repopath: string, tag?: string): Promise { diff --git a/pxtlib/cpp.ts b/pxtlib/cpp.ts index 0ce71f9cde70..00d2c8dfbde0 100644 --- a/pxtlib/cpp.ts +++ b/pxtlib/cpp.ts @@ -1516,6 +1516,21 @@ namespace pxt.hexloader { }) } + export function stringifyHexInfoForCache(hexInfo: pxtc.HexInfo) { + if (!hexInfo?.hex) return undefined; + + const cachedMeta = { + ...hexInfo, + hex: compressHex(hexInfo.hex) + }; + return JSON.stringify(cachedMeta); + } + + export function storeHexInfoCacheEntryAsync(host: Host, sha: string, cachedHexInfo: string) { + if (!sha || !cachedHexInfo) return Promise.resolve(); + return storeWithLimitAsync(host, "hex-keys", "hex-" + sha, cachedHexInfo); + } + export function getHexInfoAsync(host: Host, extInfo: pxtc.ExtensionInfo, cloudModule?: any): Promise { if (!extInfo.sha) return Promise.resolve(null) @@ -1545,10 +1560,7 @@ namespace pxt.hexloader { else { return downloadHexInfoAsync(extInfo) .then(meta => { - let origHex = meta.hex - meta.hex = compressHex(meta.hex) - let store = JSON.stringify(meta) - meta.hex = origHex + let store = stringifyHexInfoForCache(meta) return storeWithLimitAsync(host, "hex-keys", key, store) .then(() => meta) }).catch(e => { diff --git a/pxtlib/github.ts b/pxtlib/github.ts index 82610438cfeb..88d0887aa430 100644 --- a/pxtlib/github.ts +++ b/pxtlib/github.ts @@ -147,11 +147,24 @@ namespace pxt.github { files: Map; } + function copyCachedPackage(cachedPackage: CachedPackage): CachedPackage { + return { + files: { ...cachedPackage.files } + }; + } + + function cachedPackageFromFallback(fallbackPackageFiles?: pxt.Map): CachedPackage { + if (!fallbackPackageFiles) return undefined; + return { + files: { ...fallbackPackageFiles } + }; + } + // caching export interface IGithubDb { latestVersionAsync(repopath: string, config: PackagesConfig): Promise; loadConfigAsync(repopath: string, tag: string): Promise; - loadPackageAsync(repopath: string, tag: string): Promise; + loadPackageAsync(repopath: string, tag: string, fallbackPackageFiles?: pxt.Map): Promise; loadTutorialMarkdown(repopath: string, tag?: string): Promise; cacheReposAsync(response: GHTutorialResponse): Promise; } @@ -273,7 +286,7 @@ namespace pxt.github { return resolved } - async loadPackageAsync(repopath: string, tag: string): Promise { + async loadPackageAsync(repopath: string, tag: string, fallbackPackageFiles?: pxt.Map): Promise { if (!tag) { pxt.debug(`load pkg: default to master branch`) tag = "master"; @@ -282,48 +295,49 @@ namespace pxt.github { // try using github proxy first if (hasProxy()) { try { - return await this.proxyWithCdnLoadPackageAsync(repopath, tag).then(v => U.clone(v)); + return await this.proxyWithCdnLoadPackageAsync(repopath, tag).then(copyCachedPackage); } catch (e) { ghProxyHandleException(e); } } // try using github apis - return await this.githubLoadPackageAsync(repopath, tag); + return await this.githubLoadPackageAsync(repopath, tag, fallbackPackageFiles); } - private githubLoadPackageAsync(repopath: string, tag: string): Promise { - return tagToShaAsync(repopath, tag) - .then(sha => { - // cache lookup - const key = `${repopath}/${sha}`; - let res = this.packages[key]; - if (res) { - pxt.debug(`github cache ${repopath}/${tag}/text`); - return Promise.resolve(U.clone(res)); - } + private async githubLoadPackageAsync(repopath: string, tag: string, fallbackPackageFiles?: pxt.Map): Promise { + try { + const sha = await tagToShaAsync(repopath, tag); + + // cache lookup + const key = `${repopath}/${sha}`; + let res = this.packages[key]; + if (res) { + pxt.debug(`github cache ${repopath}/${tag}/text`); + return copyCachedPackage(res); + } - // load and cache - pxt.log(`Downloading ${repopath}/${tag} -> ${sha}`) - return downloadTextAsync(repopath, sha, pxt.CONFIG_NAME) - .then(pkg => { - const current: CachedPackage = { - files: {} - } - current.files[pxt.CONFIG_NAME] = pkg - const cfg: pxt.PackageConfig = JSON.parse(pkg) - return U.promiseMapAll(pxt.allPkgFiles(cfg).slice(1), - fn => downloadTextAsync(repopath, sha, fn) - .then(text => { - current.files[fn] = text - })) - .then(() => { - // cache! - this.packages[key] = current; - return U.clone(current); - }) - }) - }) + // load and cache + pxt.log(`Downloading ${repopath}/${tag} -> ${sha}`) + const pkg = await downloadTextAsync(repopath, sha, pxt.CONFIG_NAME); + const current: CachedPackage = { + files: {} + } + current.files[pxt.CONFIG_NAME] = pkg + const cfg: pxt.PackageConfig = JSON.parse(pkg) + await U.promiseMapAll(pxt.allPkgFiles(cfg).slice(1), async fn => { + current.files[fn] = await downloadTextAsync(repopath, sha, fn); + }); + + // cache! + this.packages[key] = current; + return copyCachedPackage(current); + } + catch (e) { + const fallbackPackage = cachedPackageFromFallback(fallbackPackageFiles); + if (fallbackPackage) return fallbackPackage; + throw e; + } } async loadTutorialMarkdown(repopath: string, tag?: string) { @@ -731,7 +745,7 @@ namespace pxt.github { return await db.loadConfigAsync(repopath, tag) } - export async function downloadPackageAsync(repoWithTag: string, config: pxt.PackagesConfig): Promise { + export async function downloadPackageAsync(repoWithTag: string, config: pxt.PackagesConfig, fallbackPackageFiles?: pxt.Map): Promise { const p = parseRepoId(repoWithTag) if (!p) { pxt.log('Unknown GitHub syntax'); @@ -746,9 +760,16 @@ namespace pxt.github { // always try to upgrade unbound versions if (!p.tag) { - p.tag = await db.latestVersionAsync(p.slug, config) + try { + p.tag = await db.latestVersionAsync(p.slug, config) + } + catch (e) { + const fallbackPackage = cachedPackageFromFallback(fallbackPackageFiles); + if (fallbackPackage) return fallbackPackage; + throw e; + } } - const cached = await db.loadPackageAsync(p.fullName, p.tag) + const cached = await db.loadPackageAsync(p.fullName, p.tag, fallbackPackageFiles) const dv = upgradedDisablesVariants(config, repoWithTag) if (dv) { const cfg = Package.parseAndValidConfig(cached.files[pxt.CONFIG_NAME]) @@ -775,7 +796,11 @@ namespace pxt.github { return { version, config }; } - export async function cacheProjectDependenciesAsync(cfg: pxt.PackageConfig): Promise { + export async function cacheProjectDependenciesAsync(cfg: pxt.PackageConfig, fallbackPackageFilesById?: pxt.Map>): Promise { + await cacheProjectDependenciesCoreAsync(cfg, {}, fallbackPackageFilesById); + } + + async function cacheProjectDependenciesCoreAsync(cfg: pxt.PackageConfig, checked: pxt.Map, fallbackPackageFilesById?: pxt.Map>): Promise { const ghExtensions = Object.keys(cfg.dependencies) ?.filter(dep => isGithubId(cfg.dependencies[dep])); @@ -786,10 +811,15 @@ namespace pxt.github { ghExtensions.map( async ext => { const extSrc = cfg.dependencies[ext]; - const ghPkg = await downloadPackageAsync(extSrc, pkgConfig); + if (checked[extSrc]) return; + checked[extSrc] = true; + + const ghPkg = await downloadPackageAsync(extSrc, pkgConfig, fallbackPackageFilesById?.[extSrc]); if (!ghPkg) { throw new Error(lf("Cannot load extension {0} from {1}", ext, extSrc)); } + const ghPkgCfg = Package.parseAndValidConfig(ghPkg.files[pxt.CONFIG_NAME]); + if (ghPkgCfg) await cacheProjectDependenciesCoreAsync(ghPkgCfg, checked, fallbackPackageFilesById); } ) ); diff --git a/pxtlib/main.ts b/pxtlib/main.ts index f9a52338d729..9496a1c27bd7 100644 --- a/pxtlib/main.ts +++ b/pxtlib/main.ts @@ -541,6 +541,8 @@ namespace pxt { export const TUTORIAL_CODE_STOP = "_onCodeStop.ts"; export const TUTORIAL_INFO_FILE = "tutorial-info-cache.json"; export const TUTORIAL_CUSTOM_TS = "tutorial.custom.ts"; + export const PACKAGED_EXTENSIONS = "_packaged-extensions.json"; + export const PACKAGED_EXT_INFO = "_packaged-ext-info.json"; export const BREAKPOINT_TABLET = 991; // TODO (shakao) revisit when tutorial stuff is more settled export const PALETTES_FILE = "_palettes.json"; export const HISTORY_FILE = "_history"; diff --git a/pxtlib/package.ts b/pxtlib/package.ts index 0b445fd9e484..15bcbd6584c2 100644 --- a/pxtlib/package.ts +++ b/pxtlib/package.ts @@ -1281,7 +1281,10 @@ namespace pxt { !opts.target.isNative if (!noFileEmbed) { - const files = await this.filesToBePublishedAsync(true) + const files = await this.filesToBePublishedAsync(true, !appTarget.compile.useUF2) + const packagedExtInfo = this.packagedExtensionHexInfo(opts); + if (Object.keys(packagedExtInfo).length) + files[pxt.PACKAGED_EXT_INFO] = JSON.stringify(packagedExtInfo); const headerString = JSON.stringify({ name: this.config.name, comment: this.config.description, @@ -1357,24 +1360,81 @@ namespace pxt { return cfg; } - filesToBePublishedAsync(allowPrivate = false) { + async filesToBePublishedAsync(allowPrivate = false, packExternalExtensions = false): Promise> { const files: Map = {}; - return this.loadAsync() - .then(() => { - if (!allowPrivate && !this.config.public) - U.userError('Only packages with "public":true can be published') - const cfg = this.prepareConfigToBePublished(); - files[pxt.CONFIG_NAME] = pxt.Package.stringifyConfig(cfg); - for (let f of this.getFiles()) { - // already stored - if (f === pxt.CONFIG_NAME || f === HISTORY_FILE) continue; - let str = this.readFile(f) - if (str == null) - U.userError("referenced file missing: " + f) - files[f] = str + await this.loadAsync(); + + if (!allowPrivate && !this.config.public) + U.userError('Only packages with "public":true can be published') + + const cfg = this.prepareConfigToBePublished(); + files[pxt.CONFIG_NAME] = pxt.Package.stringifyConfig(cfg); + for (let f of this.getFiles()) { + // already stored + if (f === pxt.CONFIG_NAME || f === HISTORY_FILE) continue; + let str = this.readFile(f) + if (str == null) + U.userError("referenced file missing: " + f) + files[f] = str + } + + if (packExternalExtensions) { + const packagedExtensions = this.packagedExternalExtensions(); + if (Object.keys(packagedExtensions).length) + files[pxt.PACKAGED_EXTENSIONS] = JSON.stringify(packagedExtensions); + } + + return U.sortObjectFields(files) + } + + private packagedExternalExtensions(): pxt.Map> { + const packaged: pxt.Map> = {}; + const seen: pxt.Map = {}; + + const packDeps = (pkg: Package) => { + for (const dep of pkg.resolvedDependencies()) { + if (!dep) continue; + + const seenKey = `${dep.id}:${dep.version()}`; + if (seen[seenKey]) continue; + seen[seenKey] = true; + + if (dep.verProtocol() === "github" || dep.verProtocol() === "pub") { + const depFiles: pxt.Map = {}; + for (const fn of dep.getFiles()) { + if (fn === pxt.CONFIG_NAME || fn === HISTORY_FILE) continue; + const content = dep.readFile(fn); + if (content != null) depFiles[fn] = content; + } + depFiles[pxt.CONFIG_NAME] = pxt.Package.stringifyConfig(dep.config); + packaged[dep._verspec] = depFiles; + if (dep.version() !== dep._verspec) + packaged[dep.version()] = depFiles; + if (dep.verProtocol() === "pub") + packaged[dep.verArgument()] = depFiles; } - return U.sortObjectFields(files) - }) + + packDeps(dep); + } + } + + packDeps(this); + return packaged; + } + + private packagedExtensionHexInfo(opts: pxtc.CompileOptions): pxt.Map { + const packaged: pxt.Map = {}; + const targets = [opts, ...(opts.otherMultiVariants || [])]; + + for (const target of targets) { + const extInfo = target.extinfo; + if (!extInfo?.sha || !extInfo.hexinfo?.hex) continue; + + const serialized = pxt.hexloader.stringifyHexInfoForCache(extInfo.hexinfo); + if (serialized) packaged[extInfo.sha] = serialized; + } + + return packaged; } saveToJsonAsync(): Promise { diff --git a/webapp/src/idbworkspace.ts b/webapp/src/idbworkspace.ts index a26868b1ef32..f86e391e26a1 100644 --- a/webapp/src/idbworkspace.ts +++ b/webapp/src/idbworkspace.ts @@ -464,20 +464,34 @@ export function initGitHubDb() { repopath = repopath.toLowerCase(); const cache = await getGitHubCacheAsync(); - const id = this.latestVersionCacheKey(repopath); + let cachedVersion: string = null; try { const entry = await cache.getAsync(id); + cachedVersion = entry?.version; - if (entry && Date.now() - entry.cacheTime < GITHUB_TUTORIAL_CACHE_EXPIRATION_MILLIS) { - return entry.version; + if (cachedVersion && entry && !isExpired(entry)) { + return cachedVersion; } } catch (e) { } - const version = await this.mem.latestVersionAsync(repopath, config); + let version: string; + try { + version = await this.mem.latestVersionAsync(repopath, config); + } + catch (e) { + if (cachedVersion && isOfflineError(e)) { + pxt.debug(`github offline cache hit ${id}`); + return cachedVersion; + } + throw e; + } + if (!version) { + return cachedVersion; + } await this.cacheLatestVersionAsync(cache, repopath, version); @@ -508,7 +522,7 @@ export function initGitHubDb() { } } - async loadPackageAsync(repopath: string, tag: string): Promise { + async loadPackageAsync(repopath: string, tag: string, fallbackPackageFiles?: pxt.Map): Promise { repopath = repopath.toLowerCase() if (!tag) { pxt.debug(`dep: default to master`) @@ -516,7 +530,7 @@ export function initGitHubDb() { } // don't cache master if (tag == "master") - return this.mem.loadPackageAsync(repopath, tag); + return this.loadPackageFromMemoryAsync(repopath, tag, fallbackPackageFiles); const id = this.packageCacheKey(repopath, tag); const cache = await getGitHubCacheAsync(); @@ -528,7 +542,7 @@ export function initGitHubDb() { } catch (e) { pxt.debug(`github offline cache miss ${id}`); - const p = await this.mem.loadPackageAsync(repopath, tag); + const p = await this.loadPackageFromMemoryAsync(repopath, tag, fallbackPackageFiles); await this.cachePackageAsync(cache, repopath, tag, p); @@ -561,7 +575,17 @@ export function initGitHubDb() { return readFromCache(); } - const tutorialResponse = await pxt.github.downloadMarkdownTutorialInfoAsync(repopath, tag, undefined, existing?.etag); + let tutorialResponse: { resp?: pxt.github.GHTutorialResponse, etag?: string }; + try { + tutorialResponse = await pxt.github.downloadMarkdownTutorialInfoAsync(repopath, tag, undefined, existing?.etag); + } + catch (e) { + if (existing && isOfflineError(e)) { + pxt.debug(`github offline tutorial cache hit ${id}`); + return readFromCache(); + } + throw e; + } const body = tutorialResponse.resp; @@ -701,6 +725,20 @@ export function initGitHubDb() { } } + private async loadPackageFromMemoryAsync(repopath: string, tag: string, fallbackPackageFiles?: pxt.Map): Promise { + try { + return await this.mem.loadPackageAsync(repopath, tag, fallbackPackageFiles); + } + catch (e) { + if (fallbackPackageFiles) { + return { + files: { ...fallbackPackageFiles } + }; + } + throw e; + } + } + private async cacheTutorialResponseAsync(cache: pxt.BrowserUtils.IDBObjectStoreWrapper, repopath: string, tag?: string, etag?: string) { const id = this.tutorialCacheKey(repopath, tag); @@ -741,12 +779,13 @@ export function initGitHubDb() { return repopath.toLowerCase(); } - async function purgeExpiredEntriesAsync() { + async function purgeExpiredTutorialEntriesAsync() { const cache = await getGitHubCacheAsync(); const entries = await cache.getAllAsync(); - const expired = entries.filter(isExpired); + // package and config entries are used for offline hex imports + const expired = entries.filter(isExpiredTutorialEntry); if (expired.length) { for (const entry of expired) { @@ -755,13 +794,21 @@ export function initGitHubDb() { } } + function isExpiredTutorialEntry(entry: GitHubCacheEntry) { + return entry.id.indexOf("tutorial-") === 0 && isExpired(entry); + } + function isExpired(entry: GitHubCacheEntry) { return !!entry.cacheTime && Date.now() - entry.cacheTime >= GITHUB_TUTORIAL_CACHE_EXPIRATION_MILLIS; } + function isOfflineError(e: any) { + return e?.isOffline || e?.statusCode === 0 || !pxt.Cloud.isOnline(); + } + if (!/skipgithubcache=1/i.test(window.location.href)) { pxt.github.db = new GithubDb(); - /* await */ purgeExpiredEntriesAsync(); + /* await */ purgeExpiredTutorialEntriesAsync(); } } diff --git a/webapp/src/workspace.ts b/webapp/src/workspace.ts index 8a3bf4034740..0722eb8e412c 100644 --- a/webapp/src/workspace.ts +++ b/webapp/src/workspace.ts @@ -770,11 +770,79 @@ export async function installAsync(h0: InstallHeader, text: ScriptText, dontOver pxt.shell.setEditorLanguagePref(cfg.preferredEditor); } - await pxt.github.cacheProjectDependenciesAsync(cfg) + let packagedExtensionFiles: pxt.Map>; + if (text[pxt.PACKAGED_EXTENSIONS]) { + packagedExtensionFiles = pxt.Util.jsonTryParse(text[pxt.PACKAGED_EXTENSIONS]); + delete text[pxt.PACKAGED_EXTENSIONS]; + } + let packagedExtInfo: pxt.Map; + if (text[pxt.PACKAGED_EXT_INFO]) { + packagedExtInfo = pxt.Util.jsonTryParse(text[pxt.PACKAGED_EXT_INFO]); + delete text[pxt.PACKAGED_EXT_INFO]; + } + + await cachePackagedExtInfoAsync(packagedExtInfo); + await cachePackagedScriptExtensionsAsync(packagedExtensionFiles); + await pxt.github.cacheProjectDependenciesAsync(cfg, packagedExtensionFiles) await importAsync(h, text); return h; } +async function cachePackagedScriptExtensionsAsync(packagedExtensionFiles?: pxt.Map>) { + if (!packagedExtensionFiles) return; + + const config = await pxt.packagesConfigAsync(); + const scriptCache = await getScriptCacheAsync(); + const seen: pxt.Map = {}; + + await Promise.all(Object.keys(packagedExtensionFiles).map(async key => { + if (pxt.github.isGithubId(key)) return; + + const pubId = key.slice(0, 4) === "pub:" ? key.slice(4) : key; + if (!pubId || seen[pubId]) return; + seen[pubId] = true; + + const files = packagedExtensionFiles[key]; + if (!files || !pxt.Package.parseAndValidConfig(files[pxt.CONFIG_NAME])) return; + + try { + const id = encodeURIComponent(pxt.github.upgradedPackageId(config, pubId)); + await scriptCache.setAsync({ id, files }); + } + catch (e) { + pxt.log("Unable to cache packaged extension in DB"); + } + })); +} + +async function cachePackagedExtInfoAsync(packagedExtInfo?: pxt.Map) { + if (!packagedExtInfo) return; + + const hostCache = await indexedDBWorkspace.getObjectStoreAsync<{ id: string, val: string }>(indexedDBWorkspace.HOSTCACHE_TABLE); + const cacheHost = { + cacheStoreAsync: async (id: string, val: string) => { + await hostCache.setAsync({ id, val }); + }, + cacheGetAsync: async (id: string) => { + try { + return (await hostCache.getAsync(id)).val; + } + catch (e) { + return null; + } + } + } as pxt.Host; + + await Promise.all(Object.keys(packagedExtInfo).map(async sha => { + try { + await pxt.hexloader.storeHexInfoCacheEntryAsync(cacheHost, sha, packagedExtInfo[sha]); + } + catch (e) { + pxt.log("Unable to cache packaged extension hex info in DB"); + } + })); +} + export async function renameAsync(h: Header, newName: string): Promise
{ const text = await getTextAsync(h.id);