Fixes for local github cache expiration, projects loading while offline

cli/cli.ts

@@ -276,8 +276,8 @@ class FileGithubDb implements pxt.github.IGithubDb {
         return this.loadAsync(repopath, tag, "pxt", (r, t) => this.db.loadConfigAsync(r, t));
     }
 
-    loadPackageAsync(repopath: string, tag: string): Promise<pxt.github.CachedPackage> {
-        return this.loadAsync(repopath, tag, "pkg", (r, t) => this.db.loadPackageAsync(r, t));
+    loadPackageAsync(repopath: string, tag: string, fallbackPackageFiles?: pxt.Map<string>): Promise<pxt.github.CachedPackage> {
+        return this.loadAsync(repopath, tag, "pkg", (r, t) => this.db.loadPackageAsync(r, t, fallbackPackageFiles));
     }
 
     loadTutorialMarkdown(repopath: string, tag?: string): Promise<pxt.github.CachedPackage> {

pxtcompiler/simpledriver.ts

@@ -52,8 +52,8 @@ namespace pxt {
             return this.loadAsync(repopath, tag, "pxt", (r, t) => this.db.loadConfigAsync(r, t));
         }
 
-        loadPackageAsync(repopath: string, tag: string): Promise<pxt.github.CachedPackage> {
-            return this.loadAsync(repopath, tag, "pkg", (r, t) => this.db.loadPackageAsync(r, t));
+        loadPackageAsync(repopath: string, tag: string, fallbackPackageFiles?: pxt.Map<string>): Promise<pxt.github.CachedPackage> {
+            return this.loadAsync(repopath, tag, "pkg", (r, t) => this.db.loadPackageAsync(r, t, fallbackPackageFiles));
         }
 
         loadTutorialMarkdown(repopath: string, tag?: string): Promise<pxt.github.CachedPackage> {

pxtlib/cpp.ts

@@ -1516,6 +1516,21 @@ namespace pxt.hexloader {
             })
     }
 
+    export function stringifyHexInfoForCache(hexInfo: pxtc.HexInfo) {
+        if (!hexInfo?.hex) return undefined;
+
+        const cachedMeta = {
+            ...hexInfo,
+            hex: compressHex(hexInfo.hex)
+        };
+        return JSON.stringify(cachedMeta);
+    }
+
+    export function storeHexInfoCacheEntryAsync(host: Host, sha: string, cachedHexInfo: string) {
+        if (!sha || !cachedHexInfo) return Promise.resolve();
+        return storeWithLimitAsync(host, "hex-keys", "hex-" + sha, cachedHexInfo);
+    }
+
     export function getHexInfoAsync(host: Host, extInfo: pxtc.ExtensionInfo, cloudModule?: any): Promise<pxtc.HexInfo> {
         if (!extInfo.sha)
             return Promise.resolve<any>(null)
@@ -1545,10 +1560,7 @@ namespace pxt.hexloader {
                 else {
                     return downloadHexInfoAsync(extInfo)
                         .then(meta => {
-                            let origHex = meta.hex
-                            meta.hex = compressHex(meta.hex)
-                            let store = JSON.stringify(meta)
-                            meta.hex = origHex
+                            let store = stringifyHexInfoForCache(meta)
                             return storeWithLimitAsync(host, "hex-keys", key, store)
                                 .then(() => meta)
                         }).catch(e => {

pxtlib/github.ts

@@ -147,11 +147,24 @@ namespace pxt.github {
         files: Map<string>;
     }
 
+    function copyCachedPackage(cachedPackage: CachedPackage): CachedPackage {
+        return {
+            files: { ...cachedPackage.files }
+        };
+    }
+
+    function cachedPackageFromFallback(fallbackPackageFiles?: pxt.Map<string>): CachedPackage {
+        if (!fallbackPackageFiles) return undefined;
+        return {
+            files: { ...fallbackPackageFiles }
+        };
+    }
+
     // caching
     export interface IGithubDb {
         latestVersionAsync(repopath: string, config: PackagesConfig): Promise<string>;
         loadConfigAsync(repopath: string, tag: string): Promise<pxt.PackageConfig>;
-        loadPackageAsync(repopath: string, tag: string): Promise<CachedPackage>;
+        loadPackageAsync(repopath: string, tag: string, fallbackPackageFiles?: pxt.Map<string>): Promise<CachedPackage>;
         loadTutorialMarkdown(repopath: string, tag?: string): Promise<CachedPackage>;
         cacheReposAsync(response: GHTutorialResponse): Promise<void>;
     }
@@ -273,7 +286,7 @@ namespace pxt.github {
             return resolved
         }
 
-        async loadPackageAsync(repopath: string, tag: string): Promise<CachedPackage> {
+        async loadPackageAsync(repopath: string, tag: string, fallbackPackageFiles?: pxt.Map<string>): Promise<CachedPackage> {
             if (!tag) {
                 pxt.debug(`load pkg: default to master branch`)
                 tag = "master";
@@ -282,48 +295,49 @@ namespace pxt.github {
             // try using github proxy first
             if (hasProxy()) {
                 try {
-                    return await this.proxyWithCdnLoadPackageAsync(repopath, tag).then(v => U.clone(v));
+                    return await this.proxyWithCdnLoadPackageAsync(repopath, tag).then(copyCachedPackage);
                 } catch (e) {
                     ghProxyHandleException(e);
                 }
             }
 
             // try using github apis
-            return await this.githubLoadPackageAsync(repopath, tag);
+            return await this.githubLoadPackageAsync(repopath, tag, fallbackPackageFiles);
         }
 
-        private githubLoadPackageAsync(repopath: string, tag: string): Promise<CachedPackage> {
-            return tagToShaAsync(repopath, tag)
-                .then(sha => {
-                    // cache lookup
-                    const key = `${repopath}/${sha}`;
-                    let res = this.packages[key];
-                    if (res) {
-                        pxt.debug(`github cache ${repopath}/${tag}/text`);
-                        return Promise.resolve(U.clone(res));
-                    }
+        private async githubLoadPackageAsync(repopath: string, tag: string, fallbackPackageFiles?: pxt.Map<string>): Promise<CachedPackage> {
+            try {
+                const sha = await tagToShaAsync(repopath, tag);
+
+                // cache lookup
+                const key = `${repopath}/${sha}`;
+                let res = this.packages[key];
+                if (res) {
+                    pxt.debug(`github cache ${repopath}/${tag}/text`);
+                    return copyCachedPackage(res);
+                }
 
-                    // load and cache
-                    pxt.log(`Downloading ${repopath}/${tag} -> ${sha}`)
-                    return downloadTextAsync(repopath, sha, pxt.CONFIG_NAME)
-                        .then(pkg => {
-                            const current: CachedPackage = {
-                                files: {}
-                            }
-                            current.files[pxt.CONFIG_NAME] = pkg
-                            const cfg: pxt.PackageConfig = JSON.parse(pkg)
-                            return U.promiseMapAll(pxt.allPkgFiles(cfg).slice(1),
-                                fn => downloadTextAsync(repopath, sha, fn)
-                                    .then(text => {
-                                        current.files[fn] = text
-                                    }))
-                                .then(() => {
-                                    // cache!
-                                    this.packages[key] = current;
-                                    return U.clone(current);
-                                })
-                        })
-                })
+                // load and cache
+                pxt.log(`Downloading ${repopath}/${tag} -> ${sha}`)
+                const pkg = await downloadTextAsync(repopath, sha, pxt.CONFIG_NAME);
+                const current: CachedPackage = {
+                    files: {}
+                }
+                current.files[pxt.CONFIG_NAME] = pkg
+                const cfg: pxt.PackageConfig = JSON.parse(pkg)
+                await U.promiseMapAll(pxt.allPkgFiles(cfg).slice(1), async fn => {
+                    current.files[fn] = await downloadTextAsync(repopath, sha, fn);
+                });
+
+                // cache!
+                this.packages[key] = current;
+                return copyCachedPackage(current);
+            }
+            catch (e) {
+                const fallbackPackage = cachedPackageFromFallback(fallbackPackageFiles);
+                if (fallbackPackage) return fallbackPackage;
+                throw e;
+            }
         }
 
         async loadTutorialMarkdown(repopath: string, tag?: string) {
@@ -731,7 +745,7 @@ namespace pxt.github {
         return await db.loadConfigAsync(repopath, tag)
     }
 
-    export async function downloadPackageAsync(repoWithTag: string, config: pxt.PackagesConfig): Promise<CachedPackage> {
+    export async function downloadPackageAsync(repoWithTag: string, config: pxt.PackagesConfig, fallbackPackageFiles?: pxt.Map<string>): Promise<CachedPackage> {
         const p = parseRepoId(repoWithTag)
         if (!p) {
             pxt.log('Unknown GitHub syntax');
@@ -746,9 +760,16 @@ namespace pxt.github {
 
         // always try to upgrade unbound versions
         if (!p.tag) {
-            p.tag = await db.latestVersionAsync(p.slug, config)
+            try {
+                p.tag = await db.latestVersionAsync(p.slug, config)
+            }
+            catch (e) {
+                const fallbackPackage = cachedPackageFromFallback(fallbackPackageFiles);
+                if (fallbackPackage) return fallbackPackage;
+                throw e;
+            }
         }
-        const cached = await db.loadPackageAsync(p.fullName, p.tag)
+        const cached = await db.loadPackageAsync(p.fullName, p.tag, fallbackPackageFiles)
         const dv = upgradedDisablesVariants(config, repoWithTag)
         if (dv) {
             const cfg = Package.parseAndValidConfig(cached.files[pxt.CONFIG_NAME])
@@ -775,7 +796,11 @@ namespace pxt.github {
         return { version, config };
     }
 
-    export async function cacheProjectDependenciesAsync(cfg: pxt.PackageConfig): Promise<void> {
+    export async function cacheProjectDependenciesAsync(cfg: pxt.PackageConfig, fallbackPackageFilesById?: pxt.Map<pxt.Map<string>>): Promise<void> {
+        await cacheProjectDependenciesCoreAsync(cfg, {}, fallbackPackageFilesById);
+    }
+
+    async function cacheProjectDependenciesCoreAsync(cfg: pxt.PackageConfig, checked: pxt.Map<boolean>, fallbackPackageFilesById?: pxt.Map<pxt.Map<string>>): Promise<void> {
         const ghExtensions = Object.keys(cfg.dependencies)
             ?.filter(dep => isGithubId(cfg.dependencies[dep]));
 
@@ -786,10 +811,15 @@ namespace pxt.github {
                 ghExtensions.map(
                     async ext => {
                         const extSrc = cfg.dependencies[ext];
-                        const ghPkg = await downloadPackageAsync(extSrc, pkgConfig);
+                        if (checked[extSrc]) return;
+                        checked[extSrc] = true;
+
+                        const ghPkg = await downloadPackageAsync(extSrc, pkgConfig, fallbackPackageFilesById?.[extSrc]);
                         if (!ghPkg) {
                             throw new Error(lf("Cannot load extension {0} from {1}", ext, extSrc));
                         }
+                        const ghPkgCfg = Package.parseAndValidConfig(ghPkg.files[pxt.CONFIG_NAME]);
+                        if (ghPkgCfg) await cacheProjectDependenciesCoreAsync(ghPkgCfg, checked, fallbackPackageFilesById);
                     }
                 )
             );

pxtlib/main.ts

@@ -541,6 +541,8 @@ namespace pxt {
     export const TUTORIAL_CODE_STOP = "_onCodeStop.ts";
     export const TUTORIAL_INFO_FILE = "tutorial-info-cache.json";
     export const TUTORIAL_CUSTOM_TS = "tutorial.custom.ts";
+    export const PACKAGED_EXTENSIONS = "_packaged-extensions.json";
+    export const PACKAGED_EXT_INFO = "_packaged-ext-info.json";
     export const BREAKPOINT_TABLET = 991; // TODO (shakao) revisit when tutorial stuff is more settled
     export const PALETTES_FILE = "_palettes.json";
     export const HISTORY_FILE = "_history";

pxtlib/package.ts

@@ -1281,7 +1281,10 @@ namespace pxt {
                 !opts.target.isNative
 
             if (!noFileEmbed) {
-                const files = await this.filesToBePublishedAsync(true)
+                const files = await this.filesToBePublishedAsync(true, !appTarget.compile.useUF2)
+                const packagedExtInfo = this.packagedExtensionHexInfo(opts);
+                if (Object.keys(packagedExtInfo).length)
+                    files[pxt.PACKAGED_EXT_INFO] = JSON.stringify(packagedExtInfo);
                 const headerString = JSON.stringify({
                     name: this.config.name,
                     comment: this.config.description,
@@ -1357,24 +1360,81 @@ namespace pxt {
             return cfg;
         }
 
-        filesToBePublishedAsync(allowPrivate = false) {
+        async filesToBePublishedAsync(allowPrivate = false, packExternalExtensions = false): Promise<Map<string>> {
             const files: Map<string> = {};
-            return this.loadAsync()
-                .then(() => {
-                    if (!allowPrivate && !this.config.public)
-                        U.userError('Only packages with "public":true can be published')
-                    const cfg = this.prepareConfigToBePublished();
-                    files[pxt.CONFIG_NAME] = pxt.Package.stringifyConfig(cfg);
-                    for (let f of this.getFiles()) {
-                        // already stored
-                        if (f === pxt.CONFIG_NAME || f === HISTORY_FILE) continue;
-                        let str = this.readFile(f)
-                        if (str == null)
-                            U.userError("referenced file missing: " + f)
-                        files[f] = str
+            await this.loadAsync();
+
+            if (!allowPrivate && !this.config.public)
+                U.userError('Only packages with "public":true can be published')
+
+            const cfg = this.prepareConfigToBePublished();
+            files[pxt.CONFIG_NAME] = pxt.Package.stringifyConfig(cfg);
+            for (let f of this.getFiles()) {
+                // already stored
+                if (f === pxt.CONFIG_NAME || f === HISTORY_FILE) continue;
+                let str = this.readFile(f)
+                if (str == null)
+                    U.userError("referenced file missing: " + f)
+                files[f] = str
+            }
+
+            if (packExternalExtensions) {
+                const packagedExtensions = this.packagedExternalExtensions();
+                if (Object.keys(packagedExtensions).length)
+                    files[pxt.PACKAGED_EXTENSIONS] = JSON.stringify(packagedExtensions);
+            }
+
+            return U.sortObjectFields(files)
+        }
+
+        private packagedExternalExtensions(): pxt.Map<pxt.Map<string>> {
+            const packaged: pxt.Map<pxt.Map<string>> = {};
+            const seen: pxt.Map<boolean> = {};
+
+            const packDeps = (pkg: Package) => {
+                for (const dep of pkg.resolvedDependencies()) {
+                    if (!dep) continue;
+
+                    const seenKey = `${dep.id}:${dep.version()}`;
+                    if (seen[seenKey]) continue;
+                    seen[seenKey] = true;
+
+                    if (dep.verProtocol() === "github" || dep.verProtocol() === "pub") {
+                        const depFiles: pxt.Map<string> = {};
+                        for (const fn of dep.getFiles()) {
+                            if (fn === pxt.CONFIG_NAME || fn === HISTORY_FILE) continue;
+                            const content = dep.readFile(fn);
+                            if (content != null) depFiles[fn] = content;
+                        }
+                        depFiles[pxt.CONFIG_NAME] = pxt.Package.stringifyConfig(dep.config);
+                        packaged[dep._verspec] = depFiles;
+                        if (dep.version() !== dep._verspec)
+                            packaged[dep.version()] = depFiles;
+                        if (dep.verProtocol() === "pub")
+                            packaged[dep.verArgument()] = depFiles;
                     }
-                    return U.sortObjectFields(files)
-                })
+
+                    packDeps(dep);
+                }
+            }
+
+            packDeps(this);
+            return packaged;
+        }
+
+        private packagedExtensionHexInfo(opts: pxtc.CompileOptions): pxt.Map<string> {
+            const packaged: pxt.Map<string> = {};
+            const targets = [opts, ...(opts.otherMultiVariants || [])];
+
+            for (const target of targets) {
+                const extInfo = target.extinfo;
+                if (!extInfo?.sha || !extInfo.hexinfo?.hex) continue;
+
+                const serialized = pxt.hexloader.stringifyHexInfoForCache(extInfo.hexinfo);
+                if (serialized) packaged[extInfo.sha] = serialized;
+            }
+
+            return packaged;
         }
 
         saveToJsonAsync(): Promise<pxt.cpp.HexFile> {