diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 13560007..77c6bb17 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -177,6 +177,7 @@ {"id":"bd-an6z4","title":"Error-docs content authoring (umbrella): populate pages for the 133 current catalog entries","description":"Author /docs/errors/Q-X-Y.qmd pages for each entry in error_catalog.json. Open one sub-issue per subsystem when ready to work on it; closing a sub-issue requires all pages in that subsystem at status=stub or better.\n\nSubsystem sizes (133 total):\n- markdown: 36\n- yaml: 21\n- writer: 21\n- xml: 16\n- listing: 16\n- template: 7\n- navigation: 7\n- project: 3\n- theme: 2\n- internal: 2\n- lua: 1\n- cli: 1\n\nEach page is hand-authored; tooling (bd-tooling) generates stubs but does NOT generate prose. Per-page quality bar is in the foundation plan.\n\nPlan: claude-notes/plans/2026-05-22-error-docs-content.md","status":"closed","priority":3,"issue_type":"epic","created_at":"2026-05-22T17:09:37.728930Z","created_by":"cscheid","updated_at":"2026-05-24T21:16:46.562313Z","closed_at":"2026-05-24T21:16:46.561908Z","source_repo":".","compaction_level":0,"original_size":0,"labels":["content","documentation","error-reporting"],"dependencies":[{"issue_id":"bd-an6z4","depends_on_id":"bd-4ltdl","type":"blocks","created_at":"2026-05-24T21:07:57.094765Z","created_by":"cscheid","metadata":"{}","thread_id":""},{"issue_id":"bd-an6z4","depends_on_id":"bd-732bn","type":"blocks","created_at":"2026-05-24T21:07:57.904153Z","created_by":"cscheid","metadata":"{}","thread_id":""},{"issue_id":"bd-an6z4","depends_on_id":"bd-8k4ny","type":"blocks","created_at":"2026-05-24T21:07:57.632775Z","created_by":"cscheid","metadata":"{}","thread_id":""},{"issue_id":"bd-an6z4","depends_on_id":"bd-8zim0","type":"blocks","created_at":"2026-05-24T20:54:12.747207Z","created_by":"cscheid","metadata":"{}","thread_id":""},{"issue_id":"bd-an6z4","depends_on_id":"bd-94x8a","type":"parent-child","created_at":"2026-05-22T17:09:37.728930Z","created_by":"cscheid","metadata":"{}","thread_id":""},{"issue_id":"bd-an6z4","depends_on_id":"bd-bj5yp","type":"blocks","created_at":"2026-05-22T20:01:42.876087Z","created_by":"cscheid","metadata":"{}","thread_id":""},{"issue_id":"bd-an6z4","depends_on_id":"bd-c263g","type":"blocks","created_at":"2026-05-24T21:07:58.176873Z","created_by":"cscheid","metadata":"{}","thread_id":""},{"issue_id":"bd-an6z4","depends_on_id":"bd-gkqxl","type":"blocks","created_at":"2026-05-24T21:07:57.361666Z","created_by":"cscheid","metadata":"{}","thread_id":""},{"issue_id":"bd-an6z4","depends_on_id":"bd-lgxdr","type":"blocks","created_at":"2026-05-22T20:27:57.887826Z","created_by":"cscheid","metadata":"{}","thread_id":""},{"issue_id":"bd-an6z4","depends_on_id":"bd-nvlxn","type":"blocks","created_at":"2026-05-22T17:09:41.059965Z","created_by":"cscheid","metadata":"{}","thread_id":""},{"issue_id":"bd-an6z4","depends_on_id":"bd-oqbpi","type":"blocks","created_at":"2026-05-24T20:35:56.160185Z","created_by":"cscheid","metadata":"{}","thread_id":""},{"issue_id":"bd-an6z4","depends_on_id":"bd-s9emf","type":"blocks","created_at":"2026-05-24T21:07:56.828284Z","created_by":"cscheid","metadata":"{}","thread_id":""},{"issue_id":"bd-an6z4","depends_on_id":"bd-tep4x","type":"blocks","created_at":"2026-05-24T20:24:43.820419Z","created_by":"cscheid","metadata":"{}","thread_id":""},{"issue_id":"bd-an6z4","depends_on_id":"bd-uhmzq","type":"blocks","created_at":"2026-05-24T21:07:56.562544Z","created_by":"cscheid","metadata":"{}","thread_id":""}]} {"id":"bd-anhg","title":"Cargo: upgrade comrak v0.50.0 → v0.52.0","description":"Major upgrade surfaced by cargo-upgrade survey 2026-05-04. Current 0.50.0 is range-pinned in workspace; latest is 0.52.0. Type: pre-1.0 minor (semver-breaking); two minor steps. Review changelog and bump deliberately. See claude-notes/plans/2026-05-04-cargo-upgrade-survey.md and bd-hb8h.","status":"closed","priority":3,"issue_type":"chore","created_at":"2026-05-04T18:15:54.717242Z","created_by":"cscheid","updated_at":"2026-05-04T20:30:45.398021Z","closed_at":"2026-05-04T20:30:45.397884Z","close_reason":"merged: 0a75e8e4","source_repo":".","compaction_level":0,"original_size":0,"labels":["cargo","deps"],"dependencies":[{"issue_id":"bd-anhg","depends_on_id":"bd-hb8h","type":"discovered-from","created_at":"2026-05-04T18:16:04.700823Z","created_by":"cscheid","metadata":"{}","thread_id":""}]} {"id":"bd-anxz","title":"Upload dialog: editable filenames with whitespace sanitization","description":"Add ability to edit filenames in the upload dialog before uploading. Default names should replace all whitespace characters (tabs, newlines, non-breaking spaces, etc.) with hyphens so files are easily referenceable in markdown. Plan: claude-notes/plans/2026-02-13-upload-filename-editing.md","status":"in_progress","priority":2,"issue_type":"feature","created_at":"2026-02-13T21:52:18.638260Z","created_by":"cscheid","updated_at":"2026-02-13T21:57:42.099346Z","source_repo":".","compaction_level":0,"original_size":0} +{"id":"bd-apv23","title":"Import a project from a ZIP archive (hub-client)","description":"Add an 'Import from ZIP' action to the hub-client project selector that creates a NEW project from an uploaded .zip — the inverse of the existing 'Export to ZIP' (ts-packages/quarto-sync-client/src/export-zip.ts).\n\nDesign: parse ZIP (fflate unzipSync) into a file list in the shape createNewProject already accepts (base64 binary, plain-string text), then reuse the existing onProjectCreated path (App.handleProjectCreated -> createNewProject -> IDB + project set + navigate). No Automerge-layer changes needed.\n\nNew code:\n- ts-packages/quarto-sync-client/src/import-zip.ts (parseProjectZip, pure, mirrors export-zip.ts)\n- preview-runtime wrapper importProjectFromZip -> ProjectFile[]\n- ProjectSelector 'Import from ZIP' button + form (title default from filename, sync server, file picker)\n\nEdge cases: strip common top-level dir (GitHub zips), skip __MACOSX/.DS_Store/dirs, zip-slip guard, binary-vs-text via isBinaryExtension (+ optional UTF-8 sniff), empty/invalid zip errors, export->import round-trip fidelity.\n\nTDD: unit tests for parse (round-trip headline test), preview-runtime shape conversion test, ProjectSelector component test, Playwright e2e + manual browser verification. hub-client change -> npm run build:all + changelog two-commit workflow.\n\nPlan: claude-notes/plans/2026-06-01-import-from-zip.md","status":"closed","priority":2,"issue_type":"feature","created_at":"2026-06-01T21:54:21.009894Z","created_by":"cscheid","updated_at":"2026-06-02T11:31:37.056638Z","closed_at":"2026-06-02T11:31:37.056456Z","close_reason":"Implemented import-from-zip: parser + wrapper + UI + tests + e2e; verified end-to-end in real browser.","source_repo":".","compaction_level":0,"original_size":0,"labels":["hub-client"]} {"id":"bd-apvo","title":"project.lib-dir: user-config override","description":"Phase 5 hardcodes WebsiteProjectType::lib_dir() = 'site_libs' as a String return. The trait signature is owned String specifically so the override can plumb through without API churn — the implementation just needs to read project.config.metadata['project.lib-dir'] when present. Tracked from Phase 5 D4. See claude-notes/plans/2026-04-24-websites-phase-5.md.","status":"open","priority":3,"issue_type":"feature","created_at":"2026-04-25T01:31:29.127942Z","created_by":"cscheid","updated_at":"2026-04-25T01:31:29.127942Z","source_repo":".","compaction_level":0,"original_size":0,"dependencies":[{"issue_id":"bd-apvo","depends_on_id":"bd-u5pr","type":"discovered-from","created_at":"2026-04-25T01:31:29.127942Z","created_by":"cscheid","metadata":"{}","thread_id":""}]} {"id":"bd-ascs","title":"LSP outline: include cross-referenceable elements (fig, thm, tbl)","description":"Cross-referenceable divs (#fig-, #thm-, #tbl-, etc.) do not appear in the LSP document outline. Headers that semantically belong to a cross-ref target (e.g. '## Line' inside ::: {#thm-line}) currently appear as standalone outline entries instead of being folded into their owning theorem/figure. See claude-notes/plans/2026-04-17-crossref-outline.md for the full plan.","status":"in_progress","priority":2,"issue_type":"feature","created_at":"2026-04-17T18:42:58.811042Z","created_by":"cscheid","updated_at":"2026-04-17T22:17:28.658125Z","source_repo":".","compaction_level":0,"original_size":0} {"id":"bd-ayj6","title":"[websites phase 9] Hub-client project rendering","description":"Plan: claude-notes/plans/2026-04-23-website-project-epic.md § Phase 9.\n\nDeliverables:\n- New WASM API entry points:\n - build_project_nav(project_dir) -> ProjectNavState\n - render_page_in_project(file_path, project_nav_state) -> HTML\n- Hub-client state: project-scoped nav cache, invalidation on profile-affecting edits (title, frontmatter, draft flag, _quarto.yml sidebar changes).\n- Live preview: editing a page title updates sibling sidebars within one render cycle.\n- API shape must leave room for future Q2 'quarto preview' (local hub-client instance).\n- End-to-end smoke test in a real browser session per CLAUDE.md § End-to-end verification.\n\nBlocked by Phases 2, 3, 5.","status":"closed","priority":1,"issue_type":"feature","created_at":"2026-04-23T18:43:28.398865Z","created_by":"cscheid","updated_at":"2026-04-29T00:31:38.160801Z","closed_at":"2026-04-29T00:31:38.160419Z","close_reason":"Phase 9 implemented: Pass2Renderer trait extraction (9.0), ProjectPipeline un-gate for WASM (9.1), WASM Pass-2 renderer + cross-platform flush_site_libs (9.2), render_page_in_project WASM entry point with RenderMode::ActivePage (9.3), hub-client renderToHtml switch + Preview useEffect deps (9.4), hub-smoke fixture + native integration tests (9.5), close-out (9.6). Plan: claude-notes/plans/2026-04-27-websites-phase-9.md. 8072 workspace tests pass; cargo xtask verify (Rust + hub-client WASM build + tests) passes. Browser smoke GIF deferred to a follow-up session.","source_repo":".","compaction_level":0,"original_size":0,"dependencies":[{"issue_id":"bd-ayj6","depends_on_id":"bd-0tr6","type":"parent-child","created_at":"2026-04-23T18:43:28.398865Z","created_by":"cscheid","metadata":"{}","thread_id":""},{"issue_id":"bd-ayj6","depends_on_id":"bd-h4l6","type":"blocks","created_at":"2026-04-23T18:43:50.534096Z","created_by":"cscheid","metadata":"{}","thread_id":""},{"issue_id":"bd-ayj6","depends_on_id":"bd-mre3","type":"blocks","created_at":"2026-04-23T18:43:48.471854Z","created_by":"cscheid","metadata":"{}","thread_id":""},{"issue_id":"bd-ayj6","depends_on_id":"bd-mw7x","type":"blocks","created_at":"2026-04-23T18:43:49.497210Z","created_by":"cscheid","metadata":"{}","thread_id":""},{"issue_id":"bd-ayj6","depends_on_id":"bd-xee1","type":"blocks","created_at":"2026-04-23T18:44:39.567807Z","created_by":"cscheid","metadata":"{}","thread_id":""}]} diff --git a/claude-notes/plans/2026-06-01-import-from-zip.md b/claude-notes/plans/2026-06-01-import-from-zip.md new file mode 100644 index 00000000..1b7895a7 --- /dev/null +++ b/claude-notes/plans/2026-06-01-import-from-zip.md @@ -0,0 +1,255 @@ +# Import a project from a ZIP archive (hub-client) + +**Beads:** bd-apv23 +**Status:** Design — awaiting user go-ahead before implementation. +**Date:** 2026-06-01 + +## Overview + +hub-client can already **export** a project's contents to a ZIP +(`ts-packages/quarto-sync-client/src/export-zip.ts`, surfaced as the +"Export to ZIP" button on `ProjectTab`). This plan adds the inverse: +**import a project from a ZIP**. A user uploads a `.zip` from the +project-selector landing page, and we create a brand-new hub-client +project whose files are the contents of that archive — reusing the +existing "create new project" Automerge/IndexedDB/project-set +machinery. + +Goal: a user can take a ZIP exported by hub-client (or a plausible +GitHub-style download of a Quarto project) and turn it into a new, +fully synced hub-client project in one action. + +## Existing code this builds on + +| Concern | Location | Notes | +| --- | --- | --- | +| ZIP export (the inverse) | `ts-packages/quarto-sync-client/src/export-zip.ts` | Uses `fflate` `zipSync`; `Record`. | +| `fflate` dependency | `ts-packages/quarto-sync-client/package.json` | `unzipSync`, `strToU8`, `strFromU8` available. Already a dep. | +| Create-project UI | `hub-client/src/components/ProjectSelector.tsx` | `handleCreateProject` → `wasmCreateProject` → `onProjectCreated(files, title, type, syncServer)`. | +| Create-project handler | `hub-client/src/App.tsx` `handleProjectCreated` (≈L475) | Maps scaffold `ProjectFile[]` → `CreateProjectOptions.files`, calls `createNewProject`, writes IDB + project set, navigates. | +| Core project creation | `ts-packages/quarto-sync-client/src/client.ts` `createNewProject` (≈L808) | Iterates `options.files`; **binary content is base64**, decoded via `atob` (L877); text content is a plain string. | +| File-shape: scaffold | `ts-packages/preview-runtime/src/wasmRenderer.ts` `ProjectFile` (L915) | snake_case: `{ path, content_type: 'text'\|'binary', content, mime_type? }`. | +| File-shape: create input | `ts-packages/quarto-sync-client/src/types.ts` `CreateProjectOptions` (L213) | camelCase: `{ path, content, contentType, mimeType? }`. | +| Binary classification | `ts-packages/quarto-automerge-schema/src/index.ts` `isBinaryExtension` (L393), `isTextExtension`, `inferMimeType` (L409) | Extension-based heuristics already used by the asset-upload path. | + +Key fact that makes this tractable: **`createNewProject` already +accepts a heterogeneous file list with base64 binary content.** Import +is therefore "parse ZIP → build a file list in that shape → run the +existing create path." No changes to the Automerge layer are required. + +## Architecture / data flow + +``` +ProjectSelector ── user clicks "Import from ZIP", picks file + title + sync server + │ + ├─ read File → ArrayBuffer → Uint8Array + ├─ parseProjectZip(zipBytes) [NEW: quarto-sync-client/src/import-zip.ts] + │ unzipSync → Record + │ strip common top-level dir, drop junk/dirs, zip-slip guard + │ classify each entry (isBinaryExtension): text→strFromU8, binary→base64 + │ → ProjectFile[] (snake_case, same shape wasmCreateProject returns) + │ + └─ onProjectCreated(files, title, 'imported', syncServer) [REUSED, unchanged] + │ + └─ App.handleProjectCreated → createNewProject(...) → IDB + project set + navigate +``` + +### Design decision: reuse `onProjectCreated` (recommended) + +Have `parseProjectZip` return `ProjectFile[]` (the **snake_case** +scaffold shape) so the result flows straight into the existing +`onProjectCreated` callback with zero changes to `App.handleProjectCreated`. +This maximizes reuse — IDB write, project-set add, and navigation all +come for free. + +- The pure parse logic lives in `quarto-sync-client` (mirrors + `export-zip.ts`, same package, same `fflate` dep). It returns the + neutral camelCase `CreateProjectOptions['files']` shape that already + lives in that package's `types.ts`. +- A thin wrapper in `preview-runtime` + (`importProjectFromZip(zipBytes): ProjectFile[]`) converts to the + snake_case `ProjectFile` shape and is what `ProjectSelector` imports + — symmetric with how `exportProjectAsZip` is surfaced through + `preview-runtime/src/automergeSync.ts`. + +*Alternative considered:* a separate `handleProjectImported` in `App.tsx` +that calls `createNewProject` directly with the camelCase shape. Rejected +— it duplicates the IDB/project-set/navigation block. If we ever need +import-specific post-processing, refactor the shared tail of +`handleProjectCreated` into a helper rather than forking it. + +### UI placement + +Add an **"Import from ZIP"** button in the project-actions area next to +"Create New Project" / "Connect to Project" (NOT the bottom +"Import/Export from JSON" row — that row backs up the *project list +metadata*, a different concern from project *contents*). + +Clicking opens a small form (reusing the create-form styling): +- **Project title** — text input, pre-filled from the ZIP filename + (minus `.zip`), editable. +- **Sync server URL** — pre-filled from the current `syncServer` state + (same default as the create form). +- **ZIP file** — ``. +- Submit is disabled until a file is chosen and title/sync-server are + non-empty. + +On submit: read the file, `parseProjectZip`, then `onProjectCreated`. +Surface parse errors inline via the existing `formError` channel. + +## Edge cases & decisions (for review) + +1. **Top-level directory stripping.** GitHub "Download ZIP" wraps + everything in `repo-main/`. If *every* entry shares one leading path + segment, strip it so paths become project-relative (`index.qmd`, not + `repo-main/index.qmd`). A hub-client-exported ZIP has no such wrapper, + so this is a no-op for round-trips. **Recommend: strip single common + prefix.** +2. **Junk entries.** Skip directory entries (path ends in `/`), + `__MACOSX/…`, `.DS_Store`, and `.git/…`. **Recommend: skip.** +3. **Binary vs text classification.** Use `isBinaryExtension`. Risk: + an unknown extension defaults to text and a true-binary file would be + mangled by UTF-8 decoding. **Recommend:** extension first, plus a + UTF-8-decodability fallback — if a non-binary-extension entry fails to + decode cleanly as UTF-8 (or contains NUL bytes), treat it as binary + with `application/octet-stream`. Flag for discussion: is the extra + sniffing worth it, or is extension-only acceptable for v1? +4. **Zip-slip / path safety.** Reject or normalize entries with absolute + paths or `..` traversal. **Recommend: reject the whole import with a + clear error** rather than silently dropping. +5. **Empty / no-usable-files ZIP.** Error out with a friendly message + ("No files found in the archive"). +6. **Invalid / corrupt ZIP.** `unzipSync` throws; catch and surface as + `formError`. +7. **Synchronous unzip on the main thread.** `unzipSync` blocks; large + archives could jank the UI. **v1: accept it**, but log a note and + open a follow-up (`discovered-from`) to move to async `unzip`/worker + if we hit large real-world archives. +8. **Title default.** ZIP filename sans extension; user-editable. +9. **No project-type / validation gate.** Unlike create, import does not + pick a Quarto project type or require `_quarto.yml`. We import + whatever is there. (Optional future: warn if no `.qmd`/`_quarto.yml` + present.) +10. **Round-trip fidelity.** Importing a ZIP produced by "Export to ZIP" + must reproduce the same file set + contents. This is the headline + regression test. + +## Work items (TDD — tests first) + +### Phase 1 — Parse layer (pure, unit-tested) ✅ DONE +- [x] Write `ts-packages/quarto-sync-client/src/import-zip.test.ts` + (mirror `export-zip.test.ts`) covering: basic text+binary, + Unicode round-trip, nested paths, top-level-dir stripping, + `__MACOSX`/`.DS_Store`/`.git` skipping, unknown-extension + binary-vs-text sniffing, zip-slip rejection, empty ZIP error, + invalid-ZIP error, and **export→import round-trip** (zip a fixture + with `zipSync`, parse it back, assert equality). 20 tests; verified + they failed before impl. +- [x] Implement `parseProjectZip(zipBytes: Uint8Array): CreateProjectOptions['files']` + in `ts-packages/quarto-sync-client/src/import-zip.ts`. +- [x] Export it from the package index. +- [x] Typecheck clean; full package suite green (68 tests). + +### Phase 2 — preview-runtime wrapper ✅ DONE +- [x] Add `importProjectFromZip(zipBytes: Uint8Array): ProjectFile[]` to + `ts-packages/preview-runtime/src/automergeSync.ts` (converts + camelCase → `ProjectFile` snake_case), symmetric with + `exportProjectAsZip`. Auto-exported via `export * from './automergeSync'`. +- [x] Added shape-conversion + error-propagation tests to + `automergeSync.test.ts` (29 tests pass; typecheck clean). + +### Phase 3 — UI ✅ DONE +- [x] Add "Import from ZIP" button + form to + `hub-client/src/components/ProjectSelector.tsx` (state, handler, + file reader, error surfacing) and matching CSS (`.import-btn`). +- [x] Wire submit → `importProjectFromZip` → existing `onProjectCreated` + (project type passed as the `'imported'` sentinel, which + `App.handleProjectCreated` ignores). Title prefilled from the ZIP + filename; sync-server shares the create-form default. +- [x] Component test (`ProjectSelector.import.test.tsx`, jsdom): button + reveals form, filename prefills title, submit reads file bytes → + `importProjectFromZip` → `onProjectCreated`, parse errors surfaced, + submit disabled with no file. 5 tests. +- [x] hub-client typecheck clean; full unit suite green (561 tests). +- NOTE (root-caused): real fflate `zipSync` produces a **corrupt** + archive under vitest's **jsdom** environment (a one-file zip balloons + 120 B → 518 B and reads back as `index.qmd/`, `index.qmd/0/` …). This + is a jsdom multi-realm `instanceof` artifact, *not* an fflate bug and + *not* a different fflate build (the `esm/` node and browser builds + differ only in the async Worker shim; `zipSync`/`unzipSync` are + byte-identical): + 1. jsdom installs its own `TextEncoder`; `.encode()` returns a + `Uint8Array` from jsdom's realm, so `x instanceof Uint8Array` + (the test realm's global) is **false** (jsdom/jsdom#2524). + 2. fflate's `strToU8` uses `TextEncoder`, so its output is one of + these foreign-realm arrays. + 3. fflate's `zipSync` flatten (`fltn`, `esm/index.mjs:1653`) decides + file-vs-directory with `val instanceof u8` (`u8 = Uint8Array` + captured at module load). The foreign array fails the check, so + the content bytes are recursed into as a "directory" → the + `name/0/`, `name/1/` … entries. + Plain `new Uint8Array([...])` (same realm as the global) is unaffected, + which is why the binary fixture in the probe round-tripped while the + `strToU8` text entry did not. Real browsers are single-realm, so + production is unaffected (proven by the Phase 4 e2e). Consequence: real + ZIP parsing is tested only in **node-env** suites; the jsdom component + test mocks `importProjectFromZip`. + +### Phase 4 — End-to-end verification (REQUIRED before "done") ✅ DONE +- [x] Add a Playwright e2e in `hub-client/e2e/import-zip.spec.ts` that + uploads a fixture ZIP and asserts the new project opens in the + editor with the expected files (mirrors share-link-project-set + + project-loading specs). +- [x] Real-browser verification via the e2e run (Chromium against a live + hub) — this *is* the "drive a real browser session" requirement. + +**End-to-end evidence (CLAUDE.md policy):** + +Invocation: +``` +cd hub-client +npx playwright test e2e/import-zip.spec.ts --project=chromium +``` +(Playwright's `webServer` serves the production bundle via `vite preview`; +`globalSetup` boots the Rust hub on :3030.) + +Fixture ZIP built in-test with fflate: `_quarto.yml`, `index.qmd` +(title "Imported From Zip", body "Hello from an imported zip"), and a +real 1×1 `logo.png` (binary round-trip). + +Observed (all assertions passed, `1 passed (12.9s)`): +- After "Import from ZIP" → upload → "Import Project", the URL navigated + to `/#/p/` (a new project was created). +- The file sidebar showed **both** `index.qmd` (text) and `logo.png` + (binary) — confirming binary base64 round-trip survived + `createNewProject`. +- Opening `index.qmd` rendered, in the preview iframe, the text + "Hello from an imported zip" and "came from the uploaded archive". + +This exercises the real path (real fflate unzip → `importProjectFromZip` +→ `onProjectCreated` → `createNewProject` → Automerge → editor render), +and confirms the fflate/jsdom unit-test quirk does not affect production. + +### Phase 5 — Build, changelog, ship +- [x] Production build green: `VITE_E2E=1 npm run build` (tsc -b + vite + build, the strict gate). Required first rebuilding the dependency + package dists (`quarto-automerge-schema`, `quarto-sync-client`, + `preview-runtime`) since vite resolves them via `exports.import` + → `dist/`, not source. +- [x] TS test gate green: hub-client `test:ci` legs — unit 561, + integration 66, wasm 79; deps — quarto-sync-client 136, + preview-runtime 62. (No Rust changes, so the WASM artifact and + `cargo xtask verify` are unaffected; `build:wasm` would rebuild + the same bytes from unchanged Rust.) +- [ ] **Pending user go-ahead to commit** — two-commit changelog + workflow: commit code, then add a `hub-client/changelog.md` entry + under today's date header with the short hash. Close bd-apv23 + + `br sync --flush-only` + commit `.beads/`. + +## Out of scope (note for follow-ups) +- Importing *into* an existing project (merge/overlay). This plan only + creates a *new* project. +- Async/worker-based unzip for very large archives (item 7). +- Importing engine capture sidecars / `captures` metadata — only file + contents are imported; captures regenerate on render. +- Drag-and-drop of a ZIP onto the landing page (could layer on later). diff --git a/hub-client/changelog.md b/hub-client/changelog.md index e833af93..aeb657e3 100644 --- a/hub-client/changelog.md +++ b/hub-client/changelog.md @@ -15,6 +15,10 @@ be in reverse chronological order (latest first). --> +### 2026-06-02 + +- [`301ca456`](https://github.com/quarto-dev/q2/commits/301ca456): Add "Import from ZIP" to the project selector — create a new project from an uploaded .zip archive (the inverse of "Export to ZIP"). + ### 2026-05-27 - [`9aa29ee1`](https://github.com/quarto-dev/q2/commits/9aa29ee1): View toggle buttons now order markup-left / preview-right (matching the editor-left / preview-right layout) instead of preview-left / markup-right. diff --git a/hub-client/e2e/import-zip.spec.ts b/hub-client/e2e/import-zip.spec.ts new file mode 100644 index 00000000..6226fc77 --- /dev/null +++ b/hub-client/e2e/import-zip.spec.ts @@ -0,0 +1,119 @@ +/** + * E2E: "Import from ZIP" creates a new project from an uploaded archive. + * + * Drives the real landing-page UI through the real browser pipeline + * (real fflate unzip → createNewProject → Automerge → editor render), + * which also confirms the feature is unaffected by the fflate/jsdom + * unit-test quirk noted in ProjectSelector.import.test.tsx. + * + * See claude-notes/plans/2026-06-01-import-from-zip.md (bd-apv23). + */ + +import { test, expect, type Page } from '@playwright/test'; +import { zipSync, strToU8 } from 'fflate'; +import { getServerUrl } from './helpers/projectFactory'; + +/** + * Bring a fresh browser context to the project-selector landing page with + * a connected project set, so the "Import from ZIP" action is available. + * Mirrors the bootstrap in share-link-project-set.spec.ts. + */ +async function bootstrapProjectSet(page: Page, syncServer: string): Promise { + await page.goto('/'); + await expect(page.getByRole('heading', { name: 'Quarto Hub' })).toBeVisible(); + await expect( + page.getByText(/Get started by creating a new project set/i), + ).toBeVisible(); + + await page.locator('#setup-sync-server').fill(syncServer); + await page.getByRole('button', { name: /Create New Project Set/i }).click(); + + await expect(page.getByRole('heading', { name: 'Your Projects' })).toBeVisible({ + timeout: 20000, + }); +} + +/** A minimal Quarto project zipped the way exportProjectAsZip would. */ +function buildFixtureZip(): Uint8Array { + // A tiny valid PNG (1x1) to exercise the binary round-trip. + const pngBytes = new Uint8Array([ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, + 0x49, 0x48, 0x44, 0x52, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, + 0x08, 0x06, 0x00, 0x00, 0x00, 0x1f, 0x15, 0xc4, 0x89, 0x00, 0x00, 0x00, + 0x0a, 0x49, 0x44, 0x41, 0x54, 0x78, 0x9c, 0x63, 0x00, 0x01, 0x00, 0x00, + 0x05, 0x00, 0x01, 0x0d, 0x0a, 0x2d, 0xb4, 0x00, 0x00, 0x00, 0x00, 0x49, + 0x45, 0x4e, 0x44, 0xae, 0x42, 0x60, 0x82, + ]); + + return zipSync( + { + '_quarto.yml': strToU8('project:\n type: default\n'), + 'index.qmd': strToU8( + [ + '---', + 'title: Imported From Zip', + '---', + '', + '## Hello from an imported zip', + '', + 'This paragraph came from the uploaded archive.', + ].join('\n'), + ), + 'logo.png': pngBytes, + }, + { level: 6 }, + ); +} + +test.describe('Import from ZIP', () => { + // Bootstrap a project set + import + create-project + first render is more + // than the default 30s budget allows. + test.setTimeout(60_000); + + test('creates a new project from an uploaded ZIP and renders it', async ({ page }) => { + const syncServer = getServerUrl(); + + await bootstrapProjectSet(page, syncServer); + + // Open the import form. + await page.getByRole('button', { name: /Import from ZIP/i }).click(); + + // Upload the fixture archive from memory. + await page.getByLabel('ZIP File').setInputFiles({ + name: 'My Imported Project.zip', + mimeType: 'application/zip', + buffer: Buffer.from(buildFixtureZip()), + }); + + // The title prefills from the filename. + await expect(page.locator('#importTitle')).toHaveValue('My Imported Project'); + + // Point the new project at the local hub server, then import. + await page.locator('#importSyncServer').fill(syncServer); + await page.getByRole('button', { name: /Import Project/i }).click(); + + // Creation navigates into the project (/#/p/). + await expect.poll(() => page.url(), { timeout: 30000 }).toContain('/p/'); + + // The imported files appear in the sidebar (text + binary both landed). + await expect(page.locator('.file-name', { hasText: 'index.qmd' })).toBeVisible({ + timeout: 15000, + }); + await expect(page.locator('.file-name', { hasText: 'logo.png' })).toBeVisible(); + + // Open index.qmd and confirm the imported content renders in the preview. + const match = page.url().match(/\/p\/([^/]+)/); + expect(match).not.toBeNull(); + const localId = match![1]; + await page.goto(`/#/p/${localId}/file/index.qmd`); + + const previewFrame = page.frameLocator('iframe.preview-active'); + await expect(previewFrame.locator('body')).toContainText( + 'Hello from an imported zip', + { timeout: 30000 }, + ); + await expect(previewFrame.locator('body')).toContainText( + 'came from the uploaded archive', + ); + }); +}); diff --git a/hub-client/src/components/ProjectSelector.css b/hub-client/src/components/ProjectSelector.css index 139fc538..25a38b4f 100644 --- a/hub-client/src/components/ProjectSelector.css +++ b/hub-client/src/components/ProjectSelector.css @@ -366,6 +366,14 @@ border-color: var(--posit-blue-dark-1); } +.action-btn.import-btn { + border-color: var(--posit-orange); +} + +.action-btn.import-btn:hover { + border-color: var(--posit-orange); +} + .add-btn { width: 100%; padding: 16px; diff --git a/hub-client/src/components/ProjectSelector.import.test.tsx b/hub-client/src/components/ProjectSelector.import.test.tsx new file mode 100644 index 00000000..653b138e --- /dev/null +++ b/hub-client/src/components/ProjectSelector.import.test.tsx @@ -0,0 +1,167 @@ +/** + * Tests for the "Import from ZIP" flow in ProjectSelector. + * + * Scope: the component's UI wiring — the button reveals the form, + * choosing a ZIP prefills the title, submitting reads the file bytes and + * routes them through importProjectFromZip, and the parsed files reach + * onProjectCreated (with parse errors surfaced instead). + * + * importProjectFromZip is mocked here for two reasons: (1) it lives in + * the WASM-bearing @quarto/preview-runtime, which we don't want to load + * in a jsdom unit test, and (2) the actual ZIP parsing is exhaustively + * covered by node-env unit tests (quarto-sync-client/import-zip.test.ts + * and preview-runtime/automergeSync.test.ts). + * + * Do NOT call the real fflate zipSync/unzipSync from a jsdom test: under + * vitest's jsdom environment, jsdom's TextEncoder.encode() returns a + * Uint8Array from a different JS realm, so `result instanceof Uint8Array` + * is false (jsdom/jsdom#2524). fflate's strToU8 uses TextEncoder, and its + * zipSync flatten step decides "file vs directory" with `val instanceof + * u8` — so a strToU8-produced array is misclassified as a directory and + * the archive comes out corrupt. Real browsers are single-realm, so this + * is a test-environment artifact only (the e2e in e2e/import-zip.spec.ts + * exercises the real path). + * + * @vitest-environment jsdom + */ + +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { render, screen, fireEvent, cleanup, waitFor } from '@testing-library/react'; +import { DEFAULT_SYNC_SERVER } from '../utils/routing'; + +const { importMock } = vi.hoisted(() => ({ importMock: vi.fn() })); + +vi.mock('@quarto/preview-runtime', () => ({ + getProjectChoices: vi.fn().mockResolvedValue([]), + createProject: vi.fn(), + importProjectFromZip: importMock, +})); + +vi.mock('../services/projectStorage', () => ({ + listProjects: vi.fn().mockResolvedValue([]), +})); + +vi.mock('../services/userSettings', () => ({ + getUserIdentity: vi.fn().mockResolvedValue(null), + updateUserName: vi.fn(), + updateUserColor: vi.fn(), + resetUserIdentity: vi.fn(), +})); + +vi.mock('./ThemeContext', () => ({ + useTheme: () => ({ colorScheme: 'auto', cycleColorScheme: vi.fn() }), +})); + +import ProjectSelector from './ProjectSelector'; + +const ZIP_BYTES = new Uint8Array([0x50, 0x4b, 0x03, 0x04, 1, 2, 3, 4]); + +/** Build a File whose arrayBuffer() resolves to the given bytes. */ +function zipFile(name: string, bytes: Uint8Array = ZIP_BYTES): File { + const file = new File([bytes], name, { type: 'application/zip' }); + // jsdom's Blob.arrayBuffer can be flaky across versions; pin it. + Object.defineProperty(file, 'arrayBuffer', { + value: async () => bytes.slice().buffer, + configurable: true, + }); + return file; +} + +/** Attach a FileList to a file input (the `files` prop is read-only). */ +function setInputFiles(input: HTMLElement, files: File[]) { + Object.defineProperty(input, 'files', { value: files, configurable: true }); + fireEvent.change(input); +} + +function renderSelector(onProjectCreated = vi.fn()) { + render( + , + ); + return { onProjectCreated }; +} + +afterEach(cleanup); + +describe('ProjectSelector — Import from ZIP', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('reveals the import form when the button is clicked', async () => { + renderSelector(); + + fireEvent.click(await screen.findByRole('button', { name: /Import from ZIP/i })); + + expect( + screen.getByText('Create a new project from the contents of a .zip archive'), + ).toBeTruthy(); + expect(screen.getByLabelText('ZIP File')).toBeTruthy(); + }); + + it('prefills the title from the ZIP filename', async () => { + renderSelector(); + fireEvent.click(await screen.findByRole('button', { name: /Import from ZIP/i })); + + setInputFiles(screen.getByLabelText('ZIP File'), [zipFile('My Project.zip')]); + + const titleInput = screen.getByLabelText('Project Title') as HTMLInputElement; + expect(titleInput.value).toBe('My Project'); + }); + + it('reads the file bytes, parses, and calls onProjectCreated', async () => { + const parsed = [ + { path: 'index.qmd', content_type: 'text', content: '# Hi' }, + { path: 'img/logo.png', content_type: 'binary', content: 'iVBORw==', mime_type: 'image/png' }, + ]; + importMock.mockReturnValue(parsed); + + const { onProjectCreated } = renderSelector(); + fireEvent.click(await screen.findByRole('button', { name: /Import from ZIP/i })); + + setInputFiles(screen.getByLabelText('ZIP File'), [zipFile('My Project.zip')]); + fireEvent.click(screen.getByRole('button', { name: /Import Project/i })); + + await waitFor(() => expect(onProjectCreated).toHaveBeenCalledTimes(1)); + + // The file's bytes were read and handed to the parser. + expect(importMock).toHaveBeenCalledTimes(1); + const passedBytes = importMock.mock.calls[0][0]; + expect(passedBytes).toBeInstanceOf(Uint8Array); + expect(Array.from(passedBytes)).toEqual(Array.from(ZIP_BYTES)); + + // The parsed files + form values flow to the create callback. + const [files, title, projectType, syncServer] = onProjectCreated.mock.calls[0]; + expect(files).toEqual(parsed); + expect(title).toBe('My Project'); + expect(projectType).toBe('imported'); + expect(syncServer).toBe(DEFAULT_SYNC_SERVER); + }); + + it('surfaces a parse error and does not create a project', async () => { + importMock.mockImplementation(() => { + throw new Error('No files found in the archive.'); + }); + + const { onProjectCreated } = renderSelector(); + fireEvent.click(await screen.findByRole('button', { name: /Import from ZIP/i })); + + setInputFiles(screen.getByLabelText('ZIP File'), [zipFile('Empty.zip')]); + fireEvent.click(screen.getByRole('button', { name: /Import Project/i })); + + await waitFor(() => expect(screen.getByText(/no files found/i)).toBeTruthy()); + expect(onProjectCreated).not.toHaveBeenCalled(); + }); + + it('requires a file before the import can be submitted', async () => { + renderSelector(); + fireEvent.click(await screen.findByRole('button', { name: /Import from ZIP/i })); + + // With no file chosen, the submit button is disabled. + const submit = screen.getByRole('button', { name: /Import Project/i }) as HTMLButtonElement; + expect(submit.disabled).toBe(true); + }); +}); diff --git a/hub-client/src/components/ProjectSelector.tsx b/hub-client/src/components/ProjectSelector.tsx index 128db02e..1aa4e6dd 100644 --- a/hub-client/src/components/ProjectSelector.tsx +++ b/hub-client/src/components/ProjectSelector.tsx @@ -9,6 +9,7 @@ import * as userSettingsService from '../services/userSettings'; import { getProjectChoices, createProject as wasmCreateProject, + importProjectFromZip, type ProjectChoice, type ProjectFile, } from '@quarto/preview-runtime'; @@ -81,6 +82,7 @@ export default function ProjectSelector({ const [loading, setLoading] = useState(true); const [showConnectForm, setShowConnectForm] = useState(false); const [showCreateForm, setShowCreateForm] = useState(false); + const [showImportForm, setShowImportForm] = useState(false); // Connect form state const [indexDocId, setIndexDocId] = useState(''); @@ -95,6 +97,11 @@ export default function ProjectSelector({ const [projectChoices, setProjectChoices] = useState([]); const [loadingChoices, setLoadingChoices] = useState(false); + // Import-from-ZIP form state + const [importTitle, setImportTitle] = useState(''); + const [importFile, setImportFile] = useState(null); + const [isImporting, setIsImporting] = useState(false); + // User identity state const [userSettings, setUserSettings] = useState(null); const [editingName, setEditingName] = useState(false); @@ -398,6 +405,73 @@ export default function ProjectSelector({ } }; + const handleImportFileChange = (e: React.ChangeEvent) => { + setFormError(null); + const file = e.target.files?.[0] ?? null; + setImportFile(file); + // Prefill the title from the ZIP filename (minus extension) if the + // user hasn't already typed one. Stays editable afterward. + if (file && !importTitle.trim()) { + setImportTitle(file.name.replace(/\.zip$/i, '')); + } + }; + + const handleCancelImport = () => { + setShowImportForm(false); + setImportFile(null); + setImportTitle(''); + setFormError(null); + }; + + const handleImportZip = async (e: React.FormEvent) => { + e.preventDefault(); + setFormError(null); + + if (!importFile) { + setFormError('Please choose a ZIP file to import'); + return; + } + + if (!importTitle.trim()) { + setFormError('Project title is required'); + return; + } + + if (!syncServer.trim()) { + setFormError('Sync Server URL is required'); + return; + } + + setIsImporting(true); + + try { + const bytes = new Uint8Array(await importFile.arrayBuffer()); + const files = importProjectFromZip(bytes); + + if (files.length === 0) { + setFormError('The archive contains no usable files'); + return; + } + + if (onProjectCreated) { + onProjectCreated(files, importTitle.trim(), 'imported', syncServer.trim()); + // Reset the form; the parent handles navigation into the project. + setShowImportForm(false); + setImportFile(null); + setImportTitle(''); + } else { + setFormError(`Imported ${files.length} file(s), but no handler is wired to create the project.`); + } + } catch (err) { + console.error('Failed to import project from ZIP:', err); + setFormError( + err instanceof Error ? `Failed to import ZIP: ${err.message}` : 'Failed to import ZIP.', + ); + } finally { + setIsImporting(false); + } + }; + const handleDeleteProject = async (e: React.MouseEvent, project: ProjectEntry) => { e.stopPropagation(); if (confirm(`Delete "${project.description}"?`)) { @@ -573,11 +647,11 @@ export default function ProjectSelector({ {/* Show buttons when no form is visible */} - {!showConnectForm && !showCreateForm && ( + {!showConnectForm && !showCreateForm && !showImportForm && (
+
)} @@ -661,6 +747,53 @@ export default function ProjectSelector({ )} + {/* Import from ZIP form */} + {showImportForm && ( +
+

Import from ZIP

+

Create a new project from the contents of a .zip archive

+
+ + +
+
+ + setImportTitle(e.target.value)} + placeholder="My Imported Project" + /> +
+
+ + setSyncServer(e.target.value)} + placeholder="wss://sync.automerge.org" + /> +
+
+ + +
+
+ )} + {/* Connect to Project form */} {showConnectForm && (
diff --git a/ts-packages/preview-runtime/src/automergeSync.test.ts b/ts-packages/preview-runtime/src/automergeSync.test.ts index 2fd6c701..bfd912eb 100644 --- a/ts-packages/preview-runtime/src/automergeSync.test.ts +++ b/ts-packages/preview-runtime/src/automergeSync.test.ts @@ -7,6 +7,7 @@ */ import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { zipSync, strToU8 } from 'fflate'; import type { FileEntry, Patch } from '@quarto/quarto-sync-client'; import { setSyncHandlers, @@ -15,6 +16,7 @@ import { applyEditorOperations, isFileBinary, setImmediateFileChangeCallback, + importProjectFromZip, _resetForTesting, _setClientForTesting, _getCallbacksForTesting, @@ -443,6 +445,41 @@ describe('automergeSync', () => { }); }); + describe('importProjectFromZip', () => { + it('converts parsed files into the snake_case ProjectFile shape', () => { + const pngBytes = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a]); + const zip = zipSync( + { + 'index.qmd': strToU8('# Hello'), + 'images/logo.png': pngBytes, + }, + { level: 0 }, + ); + + const files = importProjectFromZip(zip); + const byPath = Object.fromEntries(files.map(f => [f.path, f])); + + // Text entry: snake_case keys, plain-string content. + expect(byPath['index.qmd']).toEqual({ + path: 'index.qmd', + content_type: 'text', + content: '# Hello', + mime_type: undefined, + }); + + // Binary entry: base64 content + inferred mime_type. + const png = byPath['images/logo.png']; + expect(png.content_type).toBe('binary'); + expect(png.mime_type).toBe('image/png'); + expect(Uint8Array.from(atob(png.content), c => c.charCodeAt(0))).toEqual(pngBytes); + }); + + it('propagates parse errors (e.g. empty archive)', () => { + const empty = zipSync({}, { level: 0 }); + expect(() => importProjectFromZip(empty)).toThrow(/no files/i); + }); + }); + describe('test isolation', () => { it('should have clean state after reset', () => { _resetForTesting(); diff --git a/ts-packages/preview-runtime/src/automergeSync.ts b/ts-packages/preview-runtime/src/automergeSync.ts index a1f89f9b..894aa51a 100644 --- a/ts-packages/preview-runtime/src/automergeSync.ts +++ b/ts-packages/preview-runtime/src/automergeSync.ts @@ -8,6 +8,7 @@ import { createSyncClient, exportProjectAsZip as exportZip, + parseProjectZip, type SyncClient, type SyncClientCallbacks, type Patch, @@ -21,7 +22,7 @@ import { type FilePayload, } from '@quarto/quarto-sync-client'; -import { vfsAddFile, vfsAddBinaryFile, vfsRemoveFile, vfsClear, initWasm } from './wasmRenderer'; +import { vfsAddFile, vfsAddBinaryFile, vfsRemoveFile, vfsClear, initWasm, type ProjectFile } from './wasmRenderer'; // Re-export types for use in other components export type { Patch, EditorContentChange, FileEntry, ActorIdentity, CaptureRef, CreateBinaryFileResult, CreateProjectOptions, CreateProjectResult }; @@ -305,6 +306,29 @@ export function exportProjectAsZip(): Uint8Array { return exportZip(ensureClient()); } +/** + * Parse a ZIP archive into the scaffold-file shape used by the + * "create new project" flow. + * + * The inverse of {@link exportProjectAsZip}: it produces `ProjectFile[]` + * (the same snake_case shape `create_project` returns from WASM), so the + * result flows straight into the existing project-creation callback + * without any further mapping. Pure — does not need a connected client. + * + * @param zipBytes - Raw bytes of an uploaded `.zip` + * @returns Files ready to hand to the project-creation path + * @throws If the archive is corrupt, contains an unsafe path, or yields + * no usable files. + */ +export function importProjectFromZip(zipBytes: Uint8Array): ProjectFile[] { + return parseProjectZip(zipBytes).map(f => ({ + path: f.path, + content_type: f.contentType, + content: f.content, + mime_type: f.mimeType, + })); +} + // ============================================================================ // Testing Utilities // ============================================================================ diff --git a/ts-packages/quarto-sync-client/src/import-zip.test.ts b/ts-packages/quarto-sync-client/src/import-zip.test.ts new file mode 100644 index 00000000..852396d9 --- /dev/null +++ b/ts-packages/quarto-sync-client/src/import-zip.test.ts @@ -0,0 +1,249 @@ +import { describe, it, expect } from 'vitest'; +import { zipSync, strToU8, strFromU8 } from 'fflate'; +import { parseProjectZip } from './import-zip.js'; + +/** Decode a base64 string (as produced for binary entries) back to bytes. */ +function fromBase64(b64: string): Uint8Array { + const bin = atob(b64); + const out = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i); + return out; +} + +/** Build a ZIP from a map of path -> bytes. */ +function makeZip(files: Record): Uint8Array { + return zipSync(files, { level: 0 }); +} + +describe('parseProjectZip', () => { + it('parses text files as UTF-8 strings', () => { + const zip = makeZip({ + 'index.qmd': strToU8('# Hello\n\nThis is a test.'), + 'styles.css': strToU8('body { color: red; }'), + }); + + const files = parseProjectZip(zip); + const byPath = Object.fromEntries(files.map(f => [f.path, f])); + + expect(files).toHaveLength(2); + expect(byPath['index.qmd'].contentType).toBe('text'); + expect(byPath['index.qmd'].content).toBe('# Hello\n\nThis is a test.'); + expect(byPath['styles.css'].contentType).toBe('text'); + expect(byPath['styles.css'].content).toBe('body { color: red; }'); + }); + + it('parses binary files as base64 with an inferred MIME type', () => { + const pngBytes = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + const zip = makeZip({ 'image.png': pngBytes }); + + const files = parseProjectZip(zip); + expect(files).toHaveLength(1); + expect(files[0].path).toBe('image.png'); + expect(files[0].contentType).toBe('binary'); + expect(files[0].mimeType).toBe('image/png'); + expect(fromBase64(files[0].content)).toEqual(pngBytes); + }); + + it('preserves Unicode text content', () => { + const zip = makeZip({ + 'unicode.qmd': strToU8('Héllo wörld! 日本語テスト 🎉'), + }); + + const files = parseProjectZip(zip); + expect(files[0].content).toBe('Héllo wörld! 日本語テスト 🎉'); + expect(files[0].contentType).toBe('text'); + }); + + it('preserves nested directory paths', () => { + const zip = makeZip({ + 'a/b/c/deep.txt': strToU8('deep content'), + }); + + const files = parseProjectZip(zip); + expect(files).toHaveLength(1); + expect(files[0].path).toBe('a/b/c/deep.txt'); + expect(files[0].content).toBe('deep content'); + }); + + it('handles a mix of text and binary files', () => { + const gifBytes = new Uint8Array([0x47, 0x49, 0x46, 0x38, 0x39, 0x61]); + const zip = makeZip({ + 'index.qmd': strToU8('---\ntitle: Test\n---'), + 'src/utils.ts': strToU8('export const x = 1;'), + 'images/logo.gif': gifBytes, + }); + + const files = parseProjectZip(zip); + const byPath = Object.fromEntries(files.map(f => [f.path, f])); + + expect(files).toHaveLength(3); + expect(byPath['index.qmd'].contentType).toBe('text'); + expect(byPath['src/utils.ts'].contentType).toBe('text'); + expect(byPath['images/logo.gif'].contentType).toBe('binary'); + expect(fromBase64(byPath['images/logo.gif'].content)).toEqual(gifBytes); + }); + + describe('top-level directory stripping', () => { + it('strips a single common leading directory (GitHub-style download)', () => { + const zip = makeZip({ + 'my-repo-main/index.qmd': strToU8('# Home'), + 'my-repo-main/about.qmd': strToU8('# About'), + 'my-repo-main/images/logo.png': new Uint8Array([0x89, 0x50, 0x4e, 0x47]), + }); + + const files = parseProjectZip(zip); + const paths = files.map(f => f.path).sort(); + expect(paths).toEqual(['about.qmd', 'images/logo.png', 'index.qmd']); + }); + + it('does not strip when entries do not all share one leading segment', () => { + const zip = makeZip({ + 'index.qmd': strToU8('# Home'), + 'images/logo.png': new Uint8Array([0x89, 0x50, 0x4e, 0x47]), + }); + + const files = parseProjectZip(zip); + const paths = files.map(f => f.path).sort(); + expect(paths).toEqual(['images/logo.png', 'index.qmd']); + }); + + it('does not strip a common prefix when only one file is present', () => { + // A single nested file is meaningful structure, not a wrapper dir. + const zip = makeZip({ + 'docs/index.qmd': strToU8('# Home'), + }); + + const files = parseProjectZip(zip); + expect(files.map(f => f.path)).toEqual(['docs/index.qmd']); + }); + }); + + describe('junk filtering', () => { + it('skips directory entries', () => { + // fflate represents directory entries as zero-length entries with a + // trailing slash; construct one explicitly. + const zip = zipSync( + { + 'dir/': new Uint8Array(0), + 'dir/file.qmd': strToU8('content'), + }, + { level: 0 }, + ); + + const files = parseProjectZip(zip); + expect(files.map(f => f.path)).toEqual(['dir/file.qmd']); + }); + + it('skips __MACOSX and .DS_Store junk', () => { + const zip = makeZip({ + 'index.qmd': strToU8('# Home'), + '__MACOSX/._index.qmd': new Uint8Array([0, 1, 2]), + '.DS_Store': new Uint8Array([0, 1, 2]), + 'sub/.DS_Store': new Uint8Array([0, 1, 2]), + }); + + const files = parseProjectZip(zip); + expect(files.map(f => f.path)).toEqual(['index.qmd']); + }); + + it('skips .git internal files', () => { + const zip = makeZip({ + 'index.qmd': strToU8('# Home'), + '.git/config': strToU8('[core]'), + '.git/objects/ab/cdef': new Uint8Array([1, 2, 3]), + }); + + const files = parseProjectZip(zip); + expect(files.map(f => f.path)).toEqual(['index.qmd']); + }); + }); + + describe('binary-vs-text classification of unknown extensions', () => { + it('treats an unknown extension with valid UTF-8 as text', () => { + const zip = makeZip({ + 'data.unknownext': strToU8('plain text payload'), + }); + + const files = parseProjectZip(zip); + expect(files[0].contentType).toBe('text'); + expect(files[0].content).toBe('plain text payload'); + }); + + it('treats an unknown extension containing NUL bytes as binary', () => { + const bytes = new Uint8Array([0x00, 0x01, 0x02, 0x00, 0xff]); + const zip = makeZip({ 'data.unknownext': bytes }); + + const files = parseProjectZip(zip); + expect(files[0].contentType).toBe('binary'); + expect(fromBase64(files[0].content)).toEqual(bytes); + }); + + it('treats an unknown extension with invalid UTF-8 as binary', () => { + // 0xC3 starts a 2-byte sequence but 0x28 is not a valid continuation. + const bytes = new Uint8Array([0x41, 0xc3, 0x28, 0x42]); + const zip = makeZip({ 'data.unknownext': bytes }); + + const files = parseProjectZip(zip); + expect(files[0].contentType).toBe('binary'); + expect(fromBase64(files[0].content)).toEqual(bytes); + }); + }); + + describe('path-safety (zip-slip)', () => { + it('rejects entries that escape via ..', () => { + const zip = makeZip({ + '../evil.qmd': strToU8('pwned'), + }); + expect(() => parseProjectZip(zip)).toThrow(/unsafe path/i); + }); + + it('rejects absolute-path entries', () => { + const zip = makeZip({ + '/etc/passwd': strToU8('root:x:0:0'), + }); + expect(() => parseProjectZip(zip)).toThrow(/unsafe path/i); + }); + }); + + describe('error handling', () => { + it('throws on an empty archive', () => { + const zip = makeZip({}); + expect(() => parseProjectZip(zip)).toThrow(/no files/i); + }); + + it('throws on an archive with only junk', () => { + const zip = makeZip({ + '.DS_Store': new Uint8Array([1, 2, 3]), + '__MACOSX/._x': new Uint8Array([1, 2, 3]), + }); + expect(() => parseProjectZip(zip)).toThrow(/no files/i); + }); + + it('throws on corrupt ZIP bytes', () => { + const garbage = new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]); + expect(() => parseProjectZip(garbage)).toThrow(); + }); + }); + + it('round-trips a hub-client-style export (zipSync -> parseProjectZip)', () => { + // Mirror what exportProjectAsZip produces: text via strToU8, binary raw. + const original: Record = { + 'index.qmd': strToU8('# Hello\n\nThis is a test.'), + 'src/utils.ts': strToU8('export const x = 1;'), + 'images/logo.gif': new Uint8Array([0x47, 0x49, 0x46, 0x38]), + 'unicode.qmd': strToU8('日本語テスト 🎉'), + }; + const zip = zipSync(original, { level: 6 }); + + const files = parseProjectZip(zip); + expect(files).toHaveLength(4); + + for (const f of files) { + const expected = original[f.path]; + expect(expected, `unexpected path ${f.path}`).toBeDefined(); + const actual = + f.contentType === 'binary' ? fromBase64(f.content) : strToU8(f.content); + expect(actual).toEqual(expected); + } + }); +}); diff --git a/ts-packages/quarto-sync-client/src/import-zip.ts b/ts-packages/quarto-sync-client/src/import-zip.ts new file mode 100644 index 00000000..fad46ae0 --- /dev/null +++ b/ts-packages/quarto-sync-client/src/import-zip.ts @@ -0,0 +1,185 @@ +/** + * Project ZIP import utility. + * + * Parses a ZIP archive into a list of files in the shape accepted by + * {@link CreateProjectOptions.files}, so the result can be handed + * straight to the existing "create new project" path. This is the + * inverse of {@link exportProjectAsZip} (see `export-zip.ts`). + * + * The function is pure: it does not touch a SyncClient, Automerge, or + * the network. Callers read the uploaded file's bytes and pass them in. + */ + +import { unzipSync, strFromU8 } from 'fflate'; +import { isBinaryExtension, isTextExtension, inferMimeType } from '@quarto/quarto-automerge-schema'; +import type { CreateProjectOptions } from './types.js'; + +/** A single parsed file, in the shape `createNewProject` consumes. */ +type ParsedFile = CreateProjectOptions['files'][number]; + +/** Validates UTF-8 strictly so unknown extensions can be sniffed. */ +const STRICT_UTF8 = new TextDecoder('utf-8', { fatal: true }); + +/** + * Parse a ZIP archive into a list of project files. + * + * Text files are returned as UTF-8 strings; binary files are returned + * base64-encoded with an inferred MIME type — matching what + * `createNewProject` expects (it `atob`-decodes binary content). + * + * Behavior: + * - A single common leading directory (e.g. the `repo-main/` wrapper a + * GitHub "Download ZIP" adds) is stripped so paths are project-relative. + * - Directory entries and editor/OS junk (`__MACOSX/`, `.DS_Store`, + * `.git/`) are dropped. + * - Entries whose path escapes the project root (absolute paths or `..` + * traversal) cause the whole import to be rejected (zip-slip guard). + * + * @param zipBytes - Raw bytes of the uploaded `.zip` + * @returns Files ready to pass as `CreateProjectOptions.files` + * @throws If the archive is corrupt, contains an unsafe path, or yields + * no usable files. + */ +export function parseProjectZip(zipBytes: Uint8Array): ParsedFile[] { + // unzipSync throws on corrupt input; let that propagate. + const entries = unzipSync(zipBytes); + + // Keep only real file entries (skip directories and junk). We filter + // before stripping the common prefix so junk like `__MACOSX/` can't + // defeat the prefix detection. + const usablePaths = Object.keys(entries).filter( + path => !isDirectoryEntry(path) && !isJunkPath(path), + ); + + if (usablePaths.length === 0) { + throw new Error('No files found in the archive.'); + } + + const prefix = commonLeadingDirectory(usablePaths); + + const files: ParsedFile[] = []; + for (const rawPath of usablePaths) { + const path = prefix ? rawPath.slice(prefix.length) : rawPath; + + // Stripping a prefix should never produce an empty path for a file + // entry, but guard anyway. + if (path === '') continue; + + if (!isSafePath(path)) { + throw new Error(`Archive contains an unsafe path: ${rawPath}`); + } + + const bytes = entries[rawPath]; + files.push(classifyEntry(path, bytes)); + } + + if (files.length === 0) { + throw new Error('No files found in the archive.'); + } + + return files; +} + +/** Directory entries are zero-length and end in a slash. */ +function isDirectoryEntry(path: string): boolean { + return path.endsWith('/'); +} + +/** OS/editor metadata and VCS internals that should never be imported. */ +function isJunkPath(path: string): boolean { + const segments = path.split('/'); + return segments.some( + seg => seg === '__MACOSX' || seg === '.DS_Store' || seg === '.git', + ); +} + +/** + * Reject paths that would escape the project root once written. We work + * on forward-slash ZIP paths; a backslash is treated as a literal + * filename character by ZIP, but we reject it too since downstream + * consumers may interpret it as a separator on Windows. + */ +function isSafePath(path: string): boolean { + if (path.startsWith('/')) return false; + if (path.includes('\\')) return false; + const segments = path.split('/'); + return !segments.some(seg => seg === '..'); +} + +/** + * If every entry shares the same leading directory segment, return that + * segment including its trailing slash (e.g. `"repo-main/"`); otherwise + * return `null`. + * + * A single-file archive is left untouched: one nested file is meaningful + * structure, not a wrapper directory. + */ +function commonLeadingDirectory(paths: string[]): string | null { + if (paths.length < 2) return null; + + const firstSlash = paths[0].indexOf('/'); + if (firstSlash === -1) return null; + + const prefix = paths[0].slice(0, firstSlash + 1); // includes trailing '/' + return paths.every(p => p.startsWith(prefix)) ? prefix : null; +} + +/** + * Classify an entry as text or binary and produce its content payload. + * + * Extension is authoritative when known. For unknown extensions we sniff: + * content that contains a NUL byte or is not valid UTF-8 is treated as + * binary, everything else as text. + */ +function classifyEntry(path: string, bytes: Uint8Array): ParsedFile { + let binary: boolean; + if (isBinaryExtension(path)) { + binary = true; + } else if (isTextExtension(path)) { + binary = false; + } else { + binary = looksBinary(bytes); + } + + if (binary) { + return { + path, + content: toBase64(bytes), + contentType: 'binary', + mimeType: inferMimeType(path), + }; + } + + return { + path, + content: strFromU8(bytes), + contentType: 'text', + }; +} + +/** Heuristic: NUL byte or invalid UTF-8 => binary. */ +function looksBinary(bytes: Uint8Array): boolean { + if (bytes.includes(0)) return true; + try { + STRICT_UTF8.decode(bytes); + return false; + } catch { + return true; + } +} + +/** + * Base64-encode bytes for transport as a string. Matches the encoding + * `createNewProject` decodes with `atob`. Chunked to avoid building a + * single multi-megabyte intermediate string and to stay clear of + * argument-count limits on `String.fromCharCode`. + */ +function toBase64(bytes: Uint8Array): string { + const CHUNK = 0x8000; + let binary = ''; + for (let i = 0; i < bytes.length; i += CHUNK) { + const slice = bytes.subarray(i, i + CHUNK); + binary += String.fromCharCode(...slice); + } + return btoa(binary); +} diff --git a/ts-packages/quarto-sync-client/src/index.ts b/ts-packages/quarto-sync-client/src/index.ts index e996fafe..11075e20 100644 --- a/ts-packages/quarto-sync-client/src/index.ts +++ b/ts-packages/quarto-sync-client/src/index.ts @@ -62,6 +62,7 @@ export type { // Export utilities export { computeSHA256 } from './hash.js'; export { exportProjectAsZip } from './export-zip.js'; +export { parseProjectZip } from './import-zip.js'; // Export replay API export { createReplaySession } from './replay.js';