diff --git a/Cargo.lock b/Cargo.lock index ef774825..7fd35a44 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4506,7 +4506,6 @@ dependencies = [ "tar", "tempfile", "thiserror 2.0.18", - "unicode-normalization", "webbrowser", "windows", "winreg", @@ -5376,15 +5375,6 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" -[[package]] -name = "unicode-normalization" -version = "0.1.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" -dependencies = [ - "tinyvec", -] - [[package]] name = "unicode-segmentation" version = "1.13.3" diff --git a/Cargo.toml b/Cargo.toml index 02a78dc4..e62d51ea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -152,7 +152,6 @@ chrono = "0.4" tempfile = "3" image = "0.25" dirs = "6" -unicode-normalization = "0.1" # Baseline is the pure-Rust backend (miniz_oxide) so the slim/cross builds need # no C zlib. The desktop `native-zlib` feature adds `flate2/zlib-ng`, which # flate2 prioritizes over the Rust backend when both are enabled. diff --git a/README.md b/README.md index 6c8ee852..7c14bc7f 100644 --- a/README.md +++ b/README.md @@ -145,6 +145,12 @@ What's in the MiSTer build: `optical rip --device /dev/sr0 --format iso|bincue`), plus `optical convert` (ISO ↔ BIN/CUE ↔ CD-CHD) and `optical browse` / `extract`. For devices with a CD/DVD drive such as the SuperStation One. +- **Remote ripping off-device:** run `rb-daemon` here and drive this drive + from the desktop app / CLI — the device only issues SCSI reads while the + desktop does the heavy CHD encoding, so the armv7 CPU isn't taxed. + `optical rip --device rb://this-device:7341/dev/sr0`, or the desktop + Optical tab's "Add remote daemon..." button. See + [`docs/remote_ripping.md`](docs/remote_ripping.md). What's excluded: - GUI windows and the update-checker self-replace UI (only meaningful diff --git a/docs/full_MiSTer_support_status.md b/docs/full_MiSTer_support_status.md index 396c806d..c62ec369 100644 --- a/docs/full_MiSTer_support_status.md +++ b/docs/full_MiSTer_support_status.md @@ -46,6 +46,12 @@ support the disk types (floppy / hard disk / CD-ROM) of the outstanding cores. images. Built into the desktop release and — as of opticaldiscs 0.4.5 — the MiSTer `rb-cli-mini` armv7 build, for devices with an attached drive (e.g. the SuperStation One). +- **Remote optical ripping:** the desktop app / CLI can drive a *remote* + daemon's optical drive over the rb-daemon — the device (e.g. a MiSTer) only + issues SCSI reads while the desktop does all the encoding (CHD compression + never taxes the armv7 CPU). `optical drives --remote host:port` to discover; + `optical rip --device rb://host:port/dev/sr0`, or the GUI Optical tab's + "Add remote daemon..." button. See `docs/remote_ripping.md`. Legend for the **Support** column: diff --git a/docs/hfsplus_btree_growth_plan.md b/docs/hfsplus_btree_growth_plan.md new file mode 100644 index 00000000..b4250c94 --- /dev/null +++ b/docs/hfsplus_btree_growth_plan.md @@ -0,0 +1,326 @@ +# HFS+ catalog B-tree growth & variable-length keys — implementation plan + +**Status:** complete (P1–P5 landed; §4b grow-on-full intentionally deferred). +- **P1 (key-format descriptor / variable-length index keys, §4a) — landed.** The + shared B-tree helpers in `hfs_common.rs` are now key-format-aware + (`BTreeKeyFormat`), the HFS+ catalog/attributes inserts and the defrag builders + thread the matching descriptor, and classic HFS is byte-identical + (`CLASSIC_CATALOG` delegates to the legacy `normalize_catalog_index_key`). +- **P2 blank sizing (§4c) — landed.** `build_blank_hfsplus_front` sizes the + catalog from the volume (~0.5%, mirroring classic `default_btree_sizes`), and + `create_blank_hfsplus_sized` lets clone/test callers pin a larger catalog. The + volume-level gate is met: 20k shuffled multi-dir inserts into a sized blank, + catalog depth ≥ 3, `hfsplus_fsck` clean. +- **P2 grow-on-full (§4b) — deferred** (see §4b note). Classic HFS itself has no + grow-on-full path and relies solely on pre-sizing; we mirror that. The blank + auto-sizes and the clone path already over-sizes its target catalog, so the + remaining gap (a *foreign*, under-sized catalog filled past capacity via live + `put`s) matches a pre-existing classic-HFS limitation. Revisit if a real + workload needs it. +- **P3 (extents-overflow + attributes through the descriptor, depth-1 removed) — + landed.** `insert_extents_overflow_record` now splits + grows the root like the + catalog/attributes inserts (`HFSPLUS_EXTENTS`, fixed 10-byte index keys); the + attributes path was already wired in P1 (`HFSPLUS_ATTRIBUTES`). Covered by two + buffer-level multi-level tests (fixed- and variable-index-key) plus a real-path + integration test: a 520-block fragmented file's 64 overflow records split the + extents tree and read back byte-for-byte. +- **P4 (defrag/clone re-verification) — landed.** A 64 MiB source with a + multi-level catalog (300 files / 10 dirs) clones via `stream_defragmented_hfsplus` + to a target whose defrag-built catalog is itself multi-level, fsck-clean, and + round-trips byte-for-byte — confirming the variable-length index keys behave + through the clone path (`btree_insert_full` + `HFSPLUS_CATALOG`), not just the + live-edit path. +- **P5 (density rotation for HFS+, §4d) — landed.** The three HFS+ live insert + paths (`insert_catalog_record`, `insert_xattr_record`, + `insert_extents_overflow_record`) now delegate to `hfs_common::btree_insert_full` + — the same shared insert classic HFS uses — which tries a B*-style sibling + rotation before splitting. Shuffled multi-dir catalog inserts pack to ~0.84 leaf + occupancy (was ~0.69 with plain split). This also deleted ~250 lines of + hand-copied split/grow dance from `hfsplus.rs`. + +**All five phases (P1–P4 + P5) are complete; only the optional §4b grow-on-full +remains deferred.** + +**Relationship to shipped work:** the classic-HFS catalog scaling work is done — +bulk import (`PROMPT-hfs-catalog-btree-scaling.md`) and the incremental per-`put` +packing (`PROMPT-hfs-catalog-incremental-put-packing.md`, commit on +`remote-optical-ripping`). This plan covers the **HFS+** equivalent, which is a +*different and more fundamental* problem and was explicitly left as future work. + +--- + +## 1. Problem + +HFS+ catalog (and extents-overflow / attributes) B-trees **cannot grow past a +single index level without corrupting**, and a freshly built HFS+ volume gets a +catalog so small it exhausts almost immediately. So adding more than a couple +dozen files to an HFS+ volume via the live edit path breaks the volume. + +This is **pre-existing** and independent of the classic-HFS packing fix +(verified: behaviour is byte-identical before and after that commit). + +### Evidence (probe: 3000 shuffled `create_file`s into a blank 256 MiB HFS+ vol) + +``` +create_file #24 failed: disk full: no free B-tree nodes +hfsplus: ok=24/3000 node_size=4096 total=4 free=0 used=4 depth=2 leaf_records=50 +fsck errors=2: + - "leaf record (node=2, idx=0) ... sorts Greater relative to the preceding + record (node=1, idx=24); B-tree leaves must be strictly ascending" + - "bitmap reports 11 blocks allocated but total_blocks - free_blocks = 35" +``` + +Two distinct defects in one shot: the catalog is tiny (4 nodes) **and** it +corrupts the moment it splits to a second leaf and needs an index separator. + +## 2. Root cause + +The shared B-tree helpers in `src/fs/hfs_common.rs` are hardwired for the +**classic HFS** key format and are reused verbatim by HFS+: + +| Concern | Classic HFS | HFS+ | Shared code today | +|---|---|---|---| +| Key-length prefix | **1 byte** | **2 bytes** (big keys) | reads `record[0]` as a 1-byte len | +| Index separator key | fixed 37-byte normalized (`0x25`) | **variable length**, up to `maxKeyLength` (516) | `normalize_catalog_index_key` forces classic 37 | +| Key payload | `parentID(4) + nameLen(1) + MacRoman` | `parentID(4) + nameLen(2) + UTF-16BE` | classic offsets | + +So when an HFS+ leaf splits: + +- `btree_split_leaf_with_insert` computes the separator with + `key_len = first[0] as usize` (`hfs_common.rs:1227`). For an HFS+ key the first + byte is the **high byte of the 2-byte length** — usually `0` for short names — + so the "separator" becomes a 1-byte slice. +- `btree_insert_into_index` then runs it through `normalize_catalog_index_key` + (`hfs_common.rs:1600`), producing a 38-byte classic key full of zeros. +- `btree_find_insert_leaf` later compares `node[start..end-4]` with + `HfsPlusFilesystem::catalog_compare`, which reads `parentID` at `key[2..6]` + (`hfsplus.rs:1695`) — i.e. it expects the 2-byte length prefix. The normalized + classic separator has the wrong shape → misrouted descent → records land in + the wrong leaf → "leaves must be strictly ascending". + +The code already knows this: `insert_extents_overflow_record` is deliberately +**depth-1-only** "because the existing HFS+ B-tree growth helpers normalize keys +assuming the 1-byte HFS-classic format" (`hfsplus.rs:2009-2012`). + +Separately, `build_blank_hfsplus_front` reserves only ~4 catalog blocks and there +is **no grow-on-full path**, so even a correct tree would run out almost +immediately. + +## 3. Scope + +Three B-trees, two write paths: + +- **Trees:** catalog, extents-overflow, attributes (all HFS+ big-key trees). +- **Paths:** + - Live edit — `HfsPlusFilesystem::insert_catalog_record` (`hfsplus.rs:1807`), + `insert_extents_overflow_record` (`:2013`), attributes insert (`:~2197`). + These have their **own** copy of the find→split→grow dance (they call the + shared `btree_split_leaf_with_insert` / `btree_insert_into_index` / + `btree_grow_root`). + - Clone/defrag — `src/fs/hfsplus_defrag.rs` builds catalogs via the shared + `btree_insert_full` (`NODE_SIZE = 4096`, `:562`). It inherits the same broken + normalize, so defrag-built HFS+ catalogs that exceed one leaf level are also + suspect and must be re-verified once the fix lands. + +## 4. Design + +Make the shared B-tree machinery **key-format-aware** instead of classic-only. +Preferred approach: a small descriptor passed into the shared helpers, rather +than forking a second copy of the algorithm. + +### 4a. `BTreeKeyFormat` descriptor + +Introduce (in `hfs_common.rs`) a cheap, `Copy` descriptor derived from the +BTHeaderRec `attributes` + `maxKeyLength` (record bytes: `maxKeyLength` at +catalog offset 34..36 — already written at `hfs_common.rs:2435`; `attributes` at +offset 54..58): + +```rust +struct BTreeKeyFormat { + big_keys: bool, // kBTBigKeysMask (0x2): 2-byte key length + variable_index_keys: bool, // kBTVariableIndexKeysMask (0x4) + max_key_len: u16, // for fixed-index-key trees +} +``` + +- `key_len(record) -> usize` — read 1 or 2 byte length per `big_keys`. +- `key_portion(record) -> &[u8]` — length prefix + key bytes (the bytes a + separator must carry). +- `make_index_key(first_key) -> Vec` — classic: `normalize_catalog_index_key` + (fixed 37); HFS+ catalog: pass through the **variable** key verbatim (with its + 2-byte length); fixed-index-key trees: pad to `max_key_len`. + +Thread `&BTreeKeyFormat` through the functions that currently bake in the classic +assumption: + +- `btree_split_leaf_with_insert` (separator extraction, `:1227`) +- `btree_split_index_with_insert` (separator extraction, `:1570`) +- `btree_insert_into_index` (`normalize_catalog_index_key`, `:1600`) +- `btree_grow_root` (first-key extraction + normalize, `:1730/1737/1743`) +- `btree_find_insert_leaf` (already format-agnostic — it slices `end-4` and + defers to `cmp`; double-check the index-key/child-pointer split for big keys) +- `btree_try_rotate_leaf` / `btree_update_index_separator` (`:1363`, `:1314`) — + the rotation's in-place separator patch currently bails when lengths differ + (the `#[must_use]` guard added in the packing fix); with variable index keys it + must instead rebuild the separator record (length can change), which means the + rotation may need to fall back to split when the new separator doesn't fit, or + the parent index node must be re-packed. Simplest first cut: keep rotation + **classic-only** and let HFS+ use plain split+grow until 4d. + +Back-compat: classic HFS passes a `BTreeKeyFormat { big_keys:false, +variable_index_keys:false, max_key_len:37 }` and the behaviour is byte-identical +to today (lock this with a golden test). + +### 4b. B-tree file growth on full — DEFERRED + +> **Deferred (2026-06-28).** Implemented sizing (4c) instead and mirrored classic +> HFS, which has **no** grow-on-full path at all — it relies entirely on +> pre-sizing the catalog (`default_btree_sizes` / `create_blank_hfs_sized`). The +> HFS+ blank now auto-sizes the catalog from the volume and the clone/defrag path +> already over-sizes its target (3× source pad + volume-scaled floor), so the only +> remaining gap is a *foreign* volume whose catalog was built small elsewhere and +> then filled past capacity through live `put`s — the same limitation classic HFS +> has shipped with. The design below is retained for if a real workload needs it; +> the riskiest parts are the map-node bitmap extension and the extents-overflow +> spill, neither validated against real macOS here. +> +> **Full follow-on plan:** [`docs/todo_hfsplus_fork_growth.md`](todo_hfsplus_fork_growth.md) +> — phased implementation (contiguous tail growth → overflow spill → map nodes), +> risks, in-repo tests, and a **macOS `fsck_hfs` validation recipe** (the real +> acceptance bar, since HFS+ is still fully supported on macOS). + +Today `btree_alloc_node` returns `DiskFull` when the fork's nodes are exhausted. +Add a grow step (catalog first, then extents/attributes) invoked from the live +insert paths and the defrag builder when `free_nodes == 0`: + +1. Allocate N more allocation blocks for the B-tree fork (respect `clumpSize`). +2. Extend the fork extents in the Volume Header (`vh.catalog_file` etc.); spill + to the extents-overflow tree if the 8 inline extents are full. +3. Grow `catalog_data`, bump `totalNodes`/`freeNodes`, and extend the node + allocation bitmap — appending **map nodes** when the header-node bitmap fills + (mirror `hfs_common`'s classic map-node logic; HFS+ at `node_size=4096` the + header bitmap covers ~30,720 nodes, see `hfsplus_defrag.rs:1349/1378`). +4. Update the volume bitmap + VH free-block count. + +### 4c. Blank catalog sizing + +In `build_blank_hfsplus_front` (`hfsplus.rs:5167`) size the initial catalog from +volume size (Apple uses ~0.25% with a sane floor), like classic HFS's +`default_btree_sizes`, so a fresh volume holds thousands of records before 4b +ever fires. + +### 4d. Density rotation for HFS+ (after 4a–4c work) + +Once growth + variable keys are correct, give HFS+ the same packing win: + +- Call `btree_try_rotate_leaf` from `HfsPlusFilesystem::insert_catalog_record`'s + full-leaf branch (`hfsplus.rs:1829`), as `btree_insert_full` now does for + classic. +- Make the rotation's separator update handle variable-length index keys + (per 4a), or accept the classic-only fast-path + split fallback. + +This is largely free once the format abstraction exists; do it last so the +correctness work isn't gated on it. + +## 5. Phasing (each step independently testable / shippable) + +1. **P1 — key-format descriptor (4a), catalog only. [DONE]** Variable-length + index keys for the live `insert_catalog_record` path. `BTreeKeyFormat` threads + through `btree_split_leaf_with_insert` / `btree_split_index_with_insert` / + `btree_insert_into_index` / `btree_grow_root` / `btree_try_rotate_leaf` / + `btree_update_index_separator` / `btree_insert_full`. Gate met at the + catalog-buffer level: `btree_insert_full` with `HFSPLUS_CATALOG` grows a + shuffled multi-parent catalog to depth ≥ 3 with strictly-ascending leaves and + every record findable by descent + (`test_hfsplus_catalog_variable_index_keys_grow_multilevel`). The + *volume-level* fsck gate moves to P2 — a blank catalog is only 4 nodes today, + so it can't reach depth ≥ 3 until 4c sizing / 4b growth land. Classic HFS + output unchanged (the existing classic scaling/fsck tests still pass). +2. **P2 — blank sizing (4c) [DONE] + catalog growth (4b) [DEFERRED].** Gate met + via sizing: `create_blank_hfsplus_sized` + the volume-scaled blank default let + a 64 MiB volume with a 16 MiB catalog take 20k shuffled multi-dir inserts to + depth ≥ 3, fsck clean, no premature `DiskFull` + (`test_hfsplus_blank_sized_catalog_20k_inserts_fsck_clean`). 4b (grow the fork + on `free_nodes == 0`) is deferred — classic HFS relies on pre-sizing alone and + has no grow path either; the blank auto-sizes and the clone path over-sizes its + target, so live growth is only needed for a foreign under-sized catalog (a + pre-existing classic-HFS limitation). +3. **P3 — extents-overflow + attributes through the same descriptor. [DONE]** + Removed the depth-1 restriction on `insert_extents_overflow_record` (it now + splits + grows via `HFSPLUS_EXTENTS`); attributes already routed through + `HFSPLUS_ATTRIBUTES` in P1. Gate met: buffer-level extents and attributes trees + grow to depth ≥ 2 with strictly-ascending leaves and every record findable, and + a real fragmented-file create splits the extents tree (64 overflow records) and + reads back byte-for-byte (`test_hfsplus_extents_overflow_grows_multilevel`, + `test_hfsplus_attributes_grows_multilevel`, + `test_hfsplus_fragmented_file_splits_extents_overflow_btree_real_path`). +4. **P4 — defrag/clone re-verification. [DONE]** `hfsplus_defrag` builds its + target catalog through `btree_insert_full` + `HFSPLUS_CATALOG` (P1), so a + large-volume clone now rebuilds a multi-level catalog that is fsck-clean and + round-trips (`stream_clone_of_multilevel_catalog_is_fsck_clean`: 300 files / 10 + dirs, source and target catalog depth ≥ 2). +5. **P5 — density rotation for HFS+ (4d). [DONE]** The HFS+ live inserts now go + through `btree_insert_full` (which rotates into a sibling before splitting), + exactly as classic HFS does. Gate met: shuffled multi-dir inserts pack ~0.84 + leaf occupancy (`test_hfsplus_catalog_shuffled_inserts_pack_densely`), up from + ~0.69. `btree_try_rotate_leaf`'s separator update already handled variable + keys (in-place when the new separator matches the old length — always true for + the fixed-length and same-length-name cases — else it abandons the rotation and + splits), so no further index-key work was needed. + +## 6. Risks / gotchas + +- **Case folding & HFSX binary compare** — separators must compare identically to + leaf keys; `catalog_compare` already branches on `case_sensitive()` + (`keyCompareType`). Variable separators must preserve the exact key bytes so + the compare is consistent at every level. +- **Endianness** — every HFS+ field is big-endian; key length is a 2-byte BE + prefix (`build_catalog_key`, `hfsplus.rs:1666`). +- **Journal** — writes to a journaled HFS+ volume must go through / respect the + journal (`prepare_for_edit`, `hfsplus_journal`). Growth touches the VH, bitmap, + and fork extents — all journaled metadata. +- **fsck coverage** — `hfsplus_fsck` must validate index-key↔child-first-key + equality, sibling links, and node-bitmap/free-node accounting after growth and + rotation (the classic fixes leaned on these checks; extend them for HFS+). +- **Map nodes** — the node-allocation bitmap spilling into map nodes is the exact + area that broke classic HFS (`IndexSiblingLinkBroken`); replicate the classic + map-node handling carefully for the larger 4 KiB nodes. +- **Don't regress classic HFS** — the descriptor must make the classic path + byte-identical; pin it with a golden-image test. + +## 7. Acceptance criteria + +```text +1. Blank HFS+ volume + 20,000 shuffled multi-dir create_file()s -> all succeed, + hfsplus_fsck clean, every record readable, no IndexSiblingLinkBroken / order + errors, catalog depth >= 3. +2. Fragmented-file + xattr stress (extents-overflow + attributes past depth-1) + -> fsck clean. +3. Large HFS+ clone/resize via hfsplus_defrag -> multi-level catalog, fsck clean, + mounts on macOS / an emulator. +4. Classic HFS output byte-identical to pre-change (golden test). +5. (P5) shuffled-insert leaf occupancy ~80%+, matching classic HFS. +``` + +## 8. Key code pointers + +- `src/fs/hfs_common.rs` — `normalize_catalog_index_key` (:505), + `btree_split_leaf_with_insert` (:~1050, separator at :1227), + `btree_split_index_with_insert` (separator at :1570), + `btree_insert_into_index` (normalize at :1600), `btree_grow_root` (:~1700, + :1730/1737/1743), `btree_find_insert_leaf`, `btree_try_rotate_leaf` / + `btree_update_index_separator` (:1314/1363), `btree_alloc_node` (DiskFull), + `BTreeHeader` (:520), `max_key_len` write (:2435). +- `src/fs/hfsplus.rs` — `insert_catalog_record` (:1807, split :1835, index + :1864/1896), `insert_extents_overflow_record` (depth-1 note :2009), + attributes insert (:~2197), `build_catalog_key` (:1666), `catalog_compare` + (:1695) / `catalog_compare_keys` (:437), `create_blank_hfsplus` (:5148) / + `build_blank_hfsplus_front` (:5167), fsck via `crate::fs::hfsplus_fsck::check`. +- `src/fs/hfsplus_defrag.rs` — `btree_insert_full` call sites, `NODE_SIZE=4096` + (:562), map-node sizing (:1349/1378). + +## 9. Out of scope + +- HFS+ compression (decmpfs) and hard-link inode plumbing. +- Any new on-disk format; this is HFS+-spec-compliant B-tree behaviour the + current code approximates with the classic format. diff --git a/docs/remote_ripping.md b/docs/remote_ripping.md new file mode 100644 index 00000000..dd600111 --- /dev/null +++ b/docs/remote_ripping.md @@ -0,0 +1,396 @@ +# Remote optical ripping — desktop-driven, device-streamed + +**Status:** design / not yet implemented — tick the [Progress tracker](#progress-tracker) +as work lands. **Author context:** follow-on to the MiSTer optical work +(`optical` + `remote` are both compiled into `rb-cli-mini` and the desktop app +today). + +Legend: `[ ]` todo · `[~]` in progress · `[x]` done & verified · `[!]` blocked / +needs decision · `[-]` dropped. + +## Motivation + +The SuperStation One (and other MiSTer-family armv7 devices) can have a CD/DVD +drive attached, and `rb-cli optical rip` already runs on-device. But the +Cyclone V's ~800 MHz dual Cortex-A9 is a poor fit for the CPU-heavy part of +ripping: **CHD compression** (LZMA / zlib / FLAC over ~700 MB of sector data) +takes many minutes and pins the device. + +The fix: let the **MiSTer do only what must be local to the drive** — issue +SCSI `READ TOC` / `READ CD` commands, run the read-retry loop, eject — and have +the **desktop do all the encoding**. Only raw sector bytes cross the LAN; no +compression happens on the device. + +``` +MiSTer (rb-cli serve) Desktop (rb app / rb-cli) +────────────────────── ───────────────────────── +cd-da-reader: READ TOC / READ CD RemoteCdReader (read_toc, read_data_sectors) +RetryConfig backoff loop (near drive) write local .bin + .cue / .iso +ship raw 2352-byte sectors ───────► CHD compress (libchdman) ← CPU-heavy, stays here +eject ◄─────── +``` + +## Goals / non-goals + +**Goals** +- Drive a *remote* optical drive from the desktop GUI and the CLI. +- All encoding (ISO assembly, BIN/CUE assembly, CHD compression) on the desktop. +- A **single unified device picker**: local drives + every connected daemon's + drives in one list/pulldown, each tagged with where it lives. +- CLI/GUI parity (`rb-cli optical rip --device rb://host:port/dev/sr0`). + +**Non-goals (for the first cut)** +- Multi-session / multiple simultaneous rips against one daemon (the drive and + `cd-da-reader`'s handle are singular — see "One-session constraint"). +- Subchannel/C2-pointer faithful "secure" ripping, GD-ROM, multisession. +- Audio-CD metadata lookup (MusicBrainz, etc.). + +## The clean seam (why this is small) + +`src/optical/rip.rs` touches the physical drive through **exactly three** +`cd-da-reader` calls — `CdReader::open`, `read_toc`, `read_data_sectors` — plus +`eject_disc`. Everything else (`rip_iso`/`rip_bin_cue` write loops, +`generate_cue_sheet`, `convert::to_chd`) is already plain local code running in +the desktop worker thread. So we abstract those calls behind a trait and swap +the implementation; the encode half never moves. + +```rust +// new: src/optical/source.rs +pub trait OpticalSource: Send { + fn read_toc(&self) -> Result; + fn read_data_sectors(&self, lba: u32, count: u32, + mode: cd_da_reader::SectorReadMode) -> Result>; + fn eject(&self) -> Result<()>; +} + +pub struct LocalCdReader { inner: cd_da_reader::CdReader, device_path: String, retry: RetryConfig } +pub struct RemoteCdReader { conn: Arc>, handle: u64 } +``` + +- `LocalCdReader` wraps today's `cd_da_reader::CdReader` (retry applied locally, + `eject` = the existing `eject_disc` shell-out). +- `RemoteCdReader` proxies each method to the daemon (retry applied + **daemon-side**, `eject` proxied). + +`rip_iso` / `rip_bin_cue` change only in that they take `&dyn OpticalSource` +instead of constructing `CdReader` directly. `run_rip` builds the right impl +from the (new) target. **Note:** `read_data_sectors` drops the `&RetryConfig` +arg from the trait — retry belongs next to the drive, so it is supplied at +`open` time (passed verbatim to a `LocalCdReader`, serialized to the daemon for +a `RemoteCdReader`). + +### RipConfig target + +```rust +pub enum OpticalTarget { + Local(String), // "/dev/sr0" + Remote { conn: Arc>, device_path: String }, +} +// RipConfig.device_path: PathBuf -> RipConfig.device: OpticalTarget +``` + +`run_rip`: +```rust +let src: Box = match &config.device { + OpticalTarget::Local(p) => Box::new(LocalCdReader::open(p, &config.retry)?), + OpticalTarget::Remote{conn, device_path} => Box::new(RemoteCdReader::open(conn.clone(), device_path, &config.retry)?), +}; +``` + +## Daemon protocol additions (the "optical tier") + +The wire protocol (`src/remote/protocol.rs`) is JSON control frames +(`[u32 LE len][JSON]` via `write_control`/`read_control`) plus a bulk path +(`Response::FileBegin{size}` followed by a `ChunkWriter` stream of +`[u32 n][n bytes]…[u32 0]`). The block tier's `ReadBlock` already returns bytes +this way — the optical sector read reuses it verbatim. + +**New capability bit** (advertised in `Hello.capabilities` only when the daemon +was built with the `optical` feature — which `rb-cli-mini` is): +```rust +pub const CAP_FAMILY_O: u16 = 1 << 2; // optical drive proxy +``` + +**New `Request` variants** (`protocol.rs`), dispatched in `server.rs`'s +`handle_conn` match: + +| Request | Response | Server action | +|---|---|---| +| `ListOpticalDrives` | `OpticalDrives { drives: Vec }` | `cd_da_reader::CdReader::list_drives()` | +| `OpenOptical { path, retry: WireRetryConfig }` | `OpticalOpened { handle }` | `CdReader::open(path)`; store in `OpticalHandle` | +| `ReadToc { handle }` | `Toc { toc: WireToc }` | `reader.read_toc()` | +| `ReadOpticalSectors { handle, lba, count, mode }` | `FileBegin { size }` + chunk stream | `reader.read_data_sectors(lba,count,mode,&retry)` | +| `EjectOptical { handle }` | `Ok` | proxy of `eject_disc` against the device | +| `CloseOptical { handle }` | `Ok` | drop the `CdReader`, free the global handle | + +`ReadOpticalSectors` size = `count * mode.sector_size()` (2048 cooked / +2352 raw|audio), capped to the existing `MAX_RANGE_READ` (4 MiB) — the desktop +already requests `SECTORS_PER_CHUNK = 128` sectors (~301 KB raw) per call. + +**Server handle store:** add `struct OpticalHandle { reader: cd_da_reader::CdReader, retry: RetryConfig, device_path: String }` alongside the existing `BlockHandle`. Errors map to `Response::Error{message}` (the desktop re-wraps as an `anyhow`/rip error). + +### One-session constraint (important) + +`cd_da_reader::CdReader` stores the drive fd in a **process-global +`static mut DRIVE_HANDLE`**, so a process can hold **one** open optical drive at +a time. The daemon therefore serializes optical sessions: a second `OpenOptical` +while one is live returns `Error{"optical drive busy"}`. For a one-drive MiSTer +this is the natural limit; document it rather than engineer around it. + +### Elevation + +`cd_da_reader::open` uses `O_RDWR` (SCSI pass-through needs write access to the +device node). The daemon must run with permission to open `/dev/sr0` — i.e. as +root on the MiSTer, the same elevated `rb-daemon` posture already used for +remote physical-disk backup. + +## Wire DTOs + +`cd-da-reader`'s public types have all-`pub` fields but **no `serde`/`Clone`**, +so mirror them in `src/remote/protocol.rs` with `From` conversions (no change to +`cd-da-reader` needed — we both encode from and reconstruct the real types): + +```rust +#[derive(Serialize, Deserialize)] pub struct WireTrack { pub number:u8, pub start_lba:u32, pub start_msf:(u8,u8,u8), pub is_audio:bool } +#[derive(Serialize, Deserialize)] pub struct WireToc { pub first_track:u8, pub last_track:u8, pub tracks:Vec, pub leadout_lba:u32 } +#[derive(Serialize, Deserialize)] pub enum WireSectorMode { Audio, DataCooked, DataRaw } +#[derive(Serialize, Deserialize)] pub struct WireRetryConfig { /* mirrors RetryConfig's 5 fields */ } +#[derive(Serialize, Deserialize)] pub struct WireOpticalDrive { pub device_path:String, pub display_name:String, pub has_audio_cd:bool } +// From for WireToc, From<&WireToc> for cd_da_reader::Toc, etc. +``` + +## Unified device picker (local + remote in one list) + +A small testable core in `src/model/optical_devices.rs`: + +```rust +pub enum DeviceLocation { Local, Remote { conn: Arc>, label: String } } // label = "host:port" +pub struct RipDevice { pub display_name: String, pub device_path: String, pub location: DeviceLocation } + +pub fn list_rip_devices(remotes: &[Arc>]) -> Vec; +// local: cd_da_reader::CdReader::list_drives() -> RipDevice{ Local } +// remote: for each conn whose Hello advertised CAP_FAMILY_O, send ListOpticalDrives +// -> RipDevice{ Remote{ conn, label } } (skip daemons without the optical capability) +``` + +`RipDevice::into_target()` yields the right `OpticalTarget` (`Local(path)` or +`Remote{conn, device_path}`) for `RipConfig`. + +## GUI changes (`src/gui/optical_tab.rs`) + +- Replace the local-only drive combo (today `refresh_drives` → + `opticaldiscs::drives::list_drives`) with the **unified** list from + `list_rip_devices`. Each entry labeled by location: + - local: `TSSTcorp CD/DVDW SH-224 (/dev/sr0)` + - remote: `[mister.local:7341] HL-DT-ST GP65NB60 (/dev/sr0)` +- Drop the `SourceMode::PhysicalDrive` vs separate-remote split — a device is a + device; the picker carries its location. (`SourceMode::ImageFile` stays for + the convert/browse-an-image path.) +- Add a small **"Add remote daemon..."** affordance that pops the existing + host:port connect dialog (reuse `RemoteConnection::connect_shared` and the + dialog already used by `RemoteBrowsePanel` / the Backup tab's + `RemoteSourceState`). Connected daemons are remembered for the session (and + optionally persisted in `config.json`); `Refresh` re-queries local + all + remotes. +- `start_rip` / `start_rip_to_chd` build `RipConfig.device` = + `selected_device.into_target()`. **Nothing in the encode path changes** — + `rip_to_chd_worker` still rips to a local temp `.bin`/`.cue` and runs + `convert::to_chd` on the desktop. + +The connect → list → pick → run worker-thread/`Arc>` pattern is copied +straight from `BackupTab::RemoteSourceState`. + +## CLI changes (`src/cli/verbs/optical.rs`) + +- `rb-cli optical drives [--remote host:port]...` — lists local drives, plus + each `--remote` daemon's drives (one `ListOpticalDrives` per connection). + Output stays ` `, with remote rows prefixed + `[host:port] `. +- `rb-cli optical rip --device ...` — `--device` accepts a local + path (`/dev/sr0`) **or** an `rb://host:port/dev/sr0` URL. The single + `CdReader::open` seam parses the scheme: `rb://` → `RemoteRef::parse` + + `connect_shared` → `OpticalTarget::Remote`; else `OpticalTarget::Local`. +- All output formats (`--format iso|bincue`, and CHD via `optical convert` or a + future `--format chd`) work identically — encoding is always local. + +## The CHD constraint (already handled) + +CD-CHD creation is **strictly path-based**: `libchdman_rs::cd::create_from_cue` / +`create_from_iso` need a real on-disk `.cue`/`.bin`/`.iso` (MAME's `parse_toc`). +So the desktop materializes a local temp BIN+CUE, then compresses — which is +exactly what `rip_to_chd_worker` already does (rip → temp `.cue`/`.bin` → +`to_chd` → delete temps). The remote reader just feeds that same two-step flow; +no new wrinkle. + +## Data volumes / network + +- Raw sector = **2352 B** (BIN/CUE, all tracks); cooked ISO = 2048 B. +- Full ~74-min disc ≈ 333,000 sectors ≈ **~780 MB** raw streamed MiSTer→desktop. +- Desktop requests `SECTORS_PER_CHUNK = 128` sectors (~301 KB) per + `ReadOpticalSectors`; the daemon re-splits into ≤27-sector `READ CD` commands + internally (stays device-side). +- Tradeoff: you ship ~780 MB raw over the LAN instead of a smaller compressed + CHD — the right call when the alternative is pinning the armv7 CPU for minutes. + On wired gigabit (~110 MB/s) the transfer is a few seconds of wall-clock per + minute of audio; on 100 Mbit it is bounded by the link, not the device CPU. + +## Progress tracker + +Update the marker as each item lands; keep the one-line note current (date + +commit when done, or the blocker for `[!]`). Phases are roughly sequential but +P1.1 (the local refactor) is independent and can land first on its own. + +### Phase 0 — prerequisites +- [x] `optical` + `remote` both compiled into `rb-cli-mini` and the desktop + build, so the MiSTer can run `rb-cli serve` *and* drive the local optical + stack. *(done — the MiSTer optical PR.)* + +### Phase 1 — streaming ISO / BIN-CUE over the wire (no encode change) +- [x] **P1.1 — `OpticalSource` seam (local-only refactor, no behavior change).** + *(done 2026-06-27)* New `src/optical/source.rs` with the trait + `LocalCdReader`; + `rip_iso` / `rip_bin_cue` take `&dyn OpticalSource`; `run_rip` builds the source + via `open_optical_source`; `eject` moved behind the trait. `RipConfig` keeps + `device_path` for now — the `OpticalTarget` switch is deferred to P1.7 where the + remote dispatch needs it (avoids a `remote`-feature-gated enum variant with no + consumer yet). Verified: optical unit tests green, no behavior change. +- [x] **P1.2 — wire DTOs.** *(done 2026-06-27)* `WireToc` / `WireTrack` / + `WireSectorMode` / `WireRetryConfig` / `WireOpticalDrive` in `protocol.rs` + (always under `remote`); `From` conversions to/from the `cd-da-reader` types + gated behind `optical` (in a `optical_conv` submodule). Serde round-trip + + conversion unit tests green. +- [x] **P1.3 — protocol surface.** *(done 2026-06-27)* `CAP_FAMILY_O = 1 << 2`; + the six optical `Request` variants (`ListOpticalDrives` / `OpenOptical` / + `ReadToc` / `ReadOpticalSectors` / `EjectOptical` / `CloseOptical`) + the + `OpticalOpened` / `Toc` / `OpticalDrives` responses (sector data reuses + `FileBegin` + chunk stream). +- [x] **P1.4 — daemon handlers.** *(done 2026-06-27)* `server.rs` + `optical_server` module: a per-connection `OpticalState` that wraps a + `LocalCdReader` (reusing the P1.1 read/eject ops) + a **process-global** + `AtomicBool` busy guard (released on session drop / connection teardown, since + `cd-da-reader` holds a global handle); dispatch arms for all six verbs + (`not(optical)` builds reply "built without the optical feature"); `Hello` + advertises `CAP_FAMILY_O` under `optical`. Builds clean in optical / + remote-only configs. *Requires the elevated daemon (root) to open `/dev/sr0` + `O_RDWR`.* +- [x] **P1.5 — client methods.** *(done 2026-06-27)* `RemoteSession::{list_optical_drives, + open_optical, read_toc, read_optical_sectors, eject_optical, close_optical}` + in `client.rs` (return the Wire DTOs, so no `optical` gate), delegated through + `RemoteConnection` in `connection.rs`. +- [x] **P1.6 — `RemoteCdReader`.** *(done 2026-06-27)* `source.rs` (gated + `remote`) implements `OpticalSource` over an `Arc>`; + retry sent at open, `Drop` calls `close_optical` to free the daemon's slot. +- [x] **P1.7 — wire it up.** *(done 2026-06-27)* `OpticalTarget` (`Local` | + `Remote{conn, device_path}`, `Remote` arm `#[cfg(feature = "remote")]`) with a + manual `Debug` (hides the conn) + `resolve()` that parses an + `rb://host:port/dev/sr0` device arg into a remote connection; `RipConfig.device` + replaces `device_path`; `open_optical_source` branches Local/Remote. CLI + `optical rip --device rb://…` works (pulled the CLI URL-parse forward from + P3.3); GUI/CLI local call sites build `OpticalTarget::Local`. Builds clean in + optical+remote / optical-only / default(GUI); 14 optical unit tests green. +- [~] **P1.8 — validate.** *(plumbing verified 2026-06-27)* `tests/remote_optical.rs`: + loopback client↔daemon test green — the `optical`-built daemon handles + `ListOpticalDrives` (round-trips, doesn't disclaim the feature), and + `OpenOptical` of a bogus device errors cleanly **and** releases the + process-global busy guard (a second open fails at open, not with "busy"). + Remaining: the byte-identical rip-a-real-disc validation (desktop ↔ a daemon on + a Linux box with an actual drive) needs hardware — user's SuperStation / a + networked box. + +### Phase 2 — CHD compression on the desktop +- [x] **P2.1 — remote → CHD.** *(done 2026-06-27 with P3.2)* `rip_to_chd_worker` + takes an `OpticalTarget`, so a remote rip → local temp `.bin`/`.cue` → + `convert::to_chd` runs entirely on the desktop. No protocol change — the encode + was always caller-side, so a remote source "just works". Reachable from the GUI + (CHD output + a remote drive) and the CLI two-step (`optical rip --device + rb://… --format bincue` then `optical convert disc.cue disc.chd`). +- [~] **P2.2 — validate.** Needs hardware: confirm a remote-ripped CD-CHD opens + in `chdman info` / loads in MAME and the device CPU stays idle during + compression (spot-check `top`). User's device. + +### Phase 3 — unified device picker + CLI parity +- [x] **P3.1 — picker core.** *(done 2026-06-27)* `src/model/optical_devices.rs`: + `RipDevice` / `DeviceLocation` (`Local` | `Remote{conn,label}`, `Remote` gated + `remote`); `list_local_rip_devices` + `append_remote_rip_devices` (errors + swallowed → offline/non-optical daemons add nothing, which also capability-gates + the picker) + `list_rip_devices`; `picker_label` / `cli_device_arg` / + `into_target` helpers. Unit test (local label/arg/target) + loopback test + (`remote_rip_device_enumeration_over_loopback`) green. +- [x] **P3.2 — GUI.** *(done 2026-06-27, compile-validated; runtime UI pending a + user check)* `optical_tab` now holds a unified `rip_devices: Vec` + + `remote_daemons`; the drive combo lists local + remote drives by `picker_label`; + an "Add remote daemon…" modal connects on a worker thread + (`ConnectStatus`/`poll_add_remote`, non-freezing) and unlocks the Physical-drive + mode. Rip dispatches via `RipDevice::to_target()`; `start_rip_to_chd` / + `rip_to_chd_worker` take an `OpticalTarget` (encode still local). Remote drives + are rip-only — `get_browsable_path` returns `None` for them (disc-info/browse + open the device locally). +- [x] **P3.3 — CLI.** *(done 2026-06-27)* `optical drives --remote host:port` + (repeatable) lists local + each daemon's drives via the picker core, printing a + feedable `` (`rb://host:port/dev/sr0` for remote rows). + `optical rip --device rb://…` landed in P1.7. +- [x] **P3.4 — polish.** *(done 2026-06-27)* Capability gating falls out of + `append_remote_rip_devices` (a daemon without `optical` errors on + `list_optical_drives` → contributes no drives). Location-aware eject is + automatic via `OpticalSource::eject` (local shells out locally; remote sends + `EjectOptical` to the daemon); the GUI eject checkbox gained a hover note. +- [x] **P3.5 — MRU of daemon addresses.** *(done 2026-06-27, GUI-only)* + `UpdateConfig.recent_daemon_addrs` (in `config.json`) + `remember_daemon()` + (dedup, newest-first, capped at 8). On a successful connect the Optical tab + records the address; the "Add remote daemon" dialog shows a "Recent:" quick-pick + list (one click re-connects). Persists across sessions; unit-tested. Note: + it's a pick list, not auto-reconnect — avoids blocking startup on an offline + daemon. + +### Done criteria (cross-cutting) +- [x] **README + `docs/full_MiSTer_support_status.md` note remote ripping.** + *(done 2026-06-27)* README MiSTer build list gained a "Remote ripping + off-device" bullet; the support-status intro gained a "Remote optical ripping" + capability line. +- [x] **CLI/GUI parity.** Both surfaces list local + remote drives and rip + iso/bincue/chd from either; remote addressed identically (`rb://…` device arg + vs the GUI's "Add remote daemon"). +- [x] **`DISK_IMAGE_EXTS` / picker filters unaffected.** Remote ripping adds no + new container/file types (it reuses the optical formats), so nothing to update + there. + +> When actively working a phase, mirror its open items into the session task +> list (`TaskCreate`) for in-flight tracking; this doc stays the durable record. + +## File-by-file touch points + +| File | Change | +|---|---| +| `src/optical/source.rs` (new) | `OpticalSource` trait, `LocalCdReader`, `RemoteCdReader` | +| `src/optical/rip.rs` | `rip_iso`/`rip_bin_cue` take `&dyn OpticalSource`; `RipConfig.device: OpticalTarget`; `eject` via the trait | +| `src/remote/protocol.rs` | `CAP_FAMILY_O`; optical `Request`/`Response` variants; `WireToc`/`WireTrack`/`WireSectorMode`/`WireRetryConfig`/`WireOpticalDrive` + `From` impls | +| `src/remote/server.rs` | `OpticalHandle`; dispatch arms for the optical tier; one-session guard; advertise `CAP_FAMILY_O` | +| `src/remote/client.rs` | `RemoteSession`: `list_optical_drives`, `open_optical`, `read_toc`, `read_optical_sectors`, `eject_optical`, `close_optical` | +| `src/model/optical_devices.rs` (new) | `RipDevice`, `DeviceLocation`, `list_rip_devices` | +| `src/gui/optical_tab.rs` | unified picker; "Add remote daemon" dialog; `OpticalTarget` wiring (encode path unchanged) | +| `src/cli/verbs/optical.rs` | `optical drives --remote`; `optical rip --device rb://…` scheme parse | +| `docs/full_MiSTer_support_status.md`, `README.md` | note remote ripping once shipped | + +## Open questions / risks + +- **One-session serialization** — acceptable for one drive; revisit only if a + daemon ever fronts multiple drives (would need per-drive handles, which + `cd-da-reader`'s global handle blocks today). +- **Error fidelity** — `CdReaderError`/`ScsiError` collapse to a string over the + wire; the desktop loses the structured SCSI sense. Acceptable; log the message. +- **Connection reuse** — the GUI should reuse the connection it listed drives + with for the rip (don't reconnect). The CLI's `rb://` path connects fresh. +- **Eject semantics** — `EjectOptical` ejects the *remote* tray; make the GUI + button label/location-aware so it's obvious which machine ejects. +- **Capability gating** — only offer remote drives from daemons whose `Hello` + advertised `CAP_FAMILY_O`; older/non-optical daemons simply don't appear. + +## Reuse (what already exists) + +- `RemoteConnection::connect_shared` (`src/remote/connection.rs`) — shared, + brokered session; the model for `RemoteCdReader`'s transport. +- `RemoteBlockReader` (`src/remote/block_reader.rs`) — the structural template + (handle lifecycle, `Drop` closes the daemon handle). +- `write_control`/`read_control` + `FileBegin`/`ChunkWriter`/`read_chunks` + (`src/remote/protocol.rs`) — framing for the TOC (JSON) and sectors (bulk). +- `BackupTab::RemoteSourceState` (`src/gui/backup_tab.rs`) — the connect → list → + pick → worker-thread/poll GUI pattern. diff --git a/docs/todo_hfsplus_fork_growth.md b/docs/todo_hfsplus_fork_growth.md new file mode 100644 index 00000000..ed724513 --- /dev/null +++ b/docs/todo_hfsplus_fork_growth.md @@ -0,0 +1,387 @@ +# HFS+ B-tree fork grow-on-full (§4b) — implementation + macOS validation plan + +**Status:** TODO / not started. This is the one deferred step of +[`docs/hfsplus_btree_growth_plan.md`](hfsplus_btree_growth_plan.md) (§4b). P1–P5 +(variable-length index keys, blank catalog sizing, extents/attributes splitting, +defrag re-verification, and density rotation) all shipped; this doc is the +follow-on so the catalog / extents-overflow / attributes B-trees can **grow their +backing fork when they run out of nodes**, instead of returning +`DiskFull("no free B-tree nodes")`. + +The headline reason this is written as its own doc: **it is the part of the plan +that can't be validated against a real Apple implementation from inside the +repo.** macOS (through at least Sequoia) still reads, writes, mounts, and +`fsck_hfs`-checks **HFS+ / HFSX** volumes — only *classic* "HFS Standard" lost +its writable driver. So the acceptance bar for this work is not just our own +`hfsplus_fsck`: it is **`fsck_hfs -fy` clean and mountable on your Mac**. The +[macOS validation recipe](#7-macos-validation-recipe-the-real-acceptance-test) is +the point of the exercise; the Rust below exists to make that recipe pass. + +--- + +## 1. What fails today + +Every B-tree node allocation funnels through one function: + +- `hfs_common::btree_alloc_node` (`src/fs/hfs_common.rs:1071`) finds a clear bit + in the tree's **node-allocation bitmap** and returns the node index. When the + fork has no free nodes it returns + `Err(FilesystemError::DiskFull("no free B-tree nodes"))` + (`hfs_common.rs:1092`). + +That error propagates up through `btree_insert_full` → +`HfsPlusFilesystem::insert_catalog_record` / `insert_xattr_record` / +`insert_extents_overflow_record`. There is **no path that makes the fork +bigger** — neither for HFS+ *nor for classic HFS*. Both rely entirely on the fork +being pre-sized (P2 sizes the blank catalog from the volume; the defrag/clone +path over-sizes its target). So the gap §4b closes is: + +> a B-tree whose fork was sized **elsewhere** (e.g. a small catalog on a +> macOS-formatted volume, or any volume whose record count outgrew its 0.5% +> estimate) and is then filled past that size through live `put`s. + +`btree_alloc_node` only has the tree buffer; it can't allocate volume blocks or +touch the Volume Header. So **growth must happen one level up**, in the +`HfsPlusFilesystem` insert methods, which own `self.vh`, `self.bitmap`, +`self.reader`, and the three tree buffers. + +## 2. Why it was deferred, and the bar for picking it up + +Classic HFS ships with no grow path; we matched that. Doing §4b means writing +genuinely new on-disk-mutation machinery (allocate blocks, extend a fork, append +map nodes) that **must produce structures `fsck_hfs` accepts** — and the +node-allocation-bitmap / map-node area is exactly what historically produced the +`IndexSiblingLinkBroken` corruption (see +`PROMPT-hfs-catalog-incremental-put-packing.md`). That is why the deliverable is +"Mac says clean," not "we say clean." + +Pick this up when a real workload needs to fill a foreign/under-sized catalog in +place. Until then, P2 sizing covers `rb-cli new` and the clone path. + +## 3. Design overview + +Add one grow primitive and invoke it with retry-once from the three insert +methods. + +```text +insert_*_record(rec): + loop: + match btree_insert_full(buf, rec, kf, cmp): + Ok(()) => return Ok + Err(DiskFull) if !grown_yet => grow_btree_fork(kind)?; grown_yet = true; continue + Err(e) => return Err(e) +``` + +`grow_btree_fork(kind)` does, in order: + +1. **Decide the growth amount.** Round up to the fork's clump (`ForkData.clump_size`, + or fall back to `VH.data_clump_size` / a fixed e.g. 8-node minimum). Growing a + clump at a time amortises the work and matches Apple's allocator. +2. **Allocate volume blocks** via the existing `allocate_extents(blocks)` + (`hfsplus.rs:2294`) — it updates `self.bitmap`, `vh.free_blocks`, and + `vh.next_allocation`, and already handles multi-run allocation. *(But see + Phase A: we want a contiguous-tail allocation, not the largest-run-first one.)* +3. **Attach the new blocks to the fork** (`vh.catalog_file` / `vh.extents_file` / + `vh.attributes_file`, each a `ForkData` with 8 inline `ExtentDescriptor`s): + - merge into the fork's **last extent** if the new blocks are physically + contiguous with it (the common, fragmentation-free case — extent count + unchanged); + - else drop them into the next free inline extent slot; + - else (all 8 inline slots used) **spill to the extents-overflow B-tree** + (Phase B). + Bump `ForkData.total_blocks` and `ForkData.logical_size`. +4. **Grow the in-memory tree buffer.** Extend `self.catalog_data` + (`self.extents_overflow_data` / `self.attributes_data`) by + `new_nodes * node_size` zero bytes so later inserts can write into it. The + buffer length must stay `== total_nodes * node_size`. +5. **Publish the new nodes to the tree.** Read the `BTreeHeader`, set + `total_nodes += new_nodes` and `free_nodes += new_nodes`, write it back. The + new node indices must be addressable + marked **free** in the node-allocation + bitmap: + - within the header node's bitmap-record capacity + (`(node_size − 278) × 8` ≈ **30,544 nodes / 117 MiB at node_size 4096** — the + same cap P2's blank sizing clamps to) the bits already exist and read as 0 + (free), so there is nothing more to do — this is **Phase A**; + - past that cap, append **map nodes** to the bitmap chain — **Phase C**. +6. **Bump volume bookkeeping:** `vh.free_blocks` already decremented in step 2; + increment `vh.write_count`. On the next `sync_metadata()` the fork extents, + bitmap, tree buffer, and VH are all persisted (`write_catalog`, + `write_allocation_bitmap`, the VH serialize). + +`btree_alloc_node` is unchanged — once `total_nodes`/`free_nodes` and the bitmap +say there's room, it just works. + +## 4. Phasing + +Each phase is independently shippable and independently Mac-verifiable. **Phase A +alone is the high-value 90% case**; B and C are orthogonal extensions. + +### Phase 0 — testability prerequisite (CLI HFS+ create) + +Not part of grow itself, but needed to drive grow from `rb-cli` for the macOS +recipe (§7) and to honour CLAUDE.md's GUI/CLI-parity rule: add +`new --fs hfsplus [--case-sensitive] [--min-catalog BYTES]` wired to +`create_blank_hfsplus_sized`. `--min-catalog` (a deliberately *small* value) is +what makes the grow path easy to trigger — a few thousand `put`s instead of +hundreds of thousands. (The in-repo Phase-A unit test doesn't need this; it calls +`create_blank_hfsplus_sized` directly. Method 1 in §7 also works without it.) + +### Phase A — contiguous tail growth, ≤ 8 extents, within the node-bitmap cap + +The minimum viable grow, and the one that covers nearly every real volume. + +- New allocator helper `allocate_contiguous_after(last_block, blocks) -> + Option` that tries to claim the run *immediately following* + the fork's current last extent. If those blocks are free, the fork's last + extent simply gets longer — **extent count stays the same, write path + unchanged**. Fall back to a free inline slot (when < 8 used) via the existing + `allocate_extents`. +- Stay under the **header-node bitmap cap** (≈30,544 nodes @ 4096). If a grow + would cross it, return `DiskFull` for now (Phase C lifts this). A 117 MiB + catalog is millions of records — fine for everything this tool targets. +- `write_catalog` (`hfsplus.rs:2258`) → `write_fork_data` already walks the 8 + inline extents, so **no write-path change** is needed. +- **Touchpoints:** `insert_catalog_record` / `insert_xattr_record` / + `insert_extents_overflow_record` (wrap with retry); new `grow_btree_fork` + + `allocate_contiguous_after`; `ForkData` update; `BTreeHeader` total/free bump. + +Gate: a catalog that starts at N nodes and is filled to ≫ N records succeeds, our +`hfsplus_fsck` is clean, **and `fsck_hfs` on macOS is clean** (recipe §7). + +### Phase B — overflow extents (fork needs a 9th extent) + +When the fork is fragmented enough to exhaust all 8 inline `ExtentDescriptor`s +and contiguous tail growth isn't possible: + +- Insert an extents-overflow record for the **special file** being grown — + `fileID = kHFSCatalogFileID (4)` / `kHFSExtentsFileID (3)` / + `kHFSAttributesFileID (8)`, `forkType = 0`, `startBlock = fork-relative block + of the new extent` — via `insert_extents_overflow_record` (now splittable, + P3). **Recursion guard:** growing the catalog may need an overflow record, + which may need to grow the *extents-overflow* fork, which must therefore be + grown **first / separately** — never re-enter the same fork's grow. +- **Write path must learn overflow.** `write_catalog` / + `write_allocation_bitmap` currently write only the inline 8 extents. The read + side already consults overflow (`read_fork_with_overflow`, `hfsplus.rs:613`); + the write side needs the same so a > 8-extent fork is fully persisted. This is + the bulk of Phase B. +- Prefer to **avoid** this case: Phase A's contiguous-tail strategy keeps most + forks at 1–2 extents. Treat Phase B as the fragmented-volume fallback. + +### Phase C — map nodes past the header-bitmap cap + +When `total_nodes` would exceed `(node_size − 278) × 8`: + +- The node-allocation bitmap spills from the header node (record 2) into + dedicated **map nodes** (`BTNodeKind = 2`) chained off the header's `fLink`. + Reuse the existing, defrag-tested helpers: + `map_nodes_required` (`hfs_common.rs:961`), `init_map_node` + (`hfs_common.rs:1137`), `map_node_bitmap_bytes`, `btree_bitmap_segments`, + `btree_node_bitmap_range`. The defrag builder already materialises map nodes up + front (`hfsplus_defrag.rs:~1401`); Phase C does it **incrementally** mid-tree. +- A new map node itself consumes a node slot: allocate it from the freshly grown + space, mark it used, link it into the chain, then the nodes it covers become + addressable. +- **Highest-risk step** — this is the classic `IndexSiblingLinkBroken` neighbourhood. + Validate aggressively (a multi-hundred-MiB catalog) and lean on `fsck_hfs`. + +## 5. Risks & gotchas + +- **Write-path overflow (Phase B).** The single biggest trap: silently writing + only the first 8 extents of a grown fork truncates the tree on disk while it + looks fine in memory. Mirror `read_fork_with_overflow` on the write side and + unit-test a > 8-extent round-trip before trusting it. +- **Map-node bitmap chain (Phase C).** Off-by-one in the header-vs-map capacity + boundary corrupts node allocation. Note the two capacity formulas already in + the tree differ: `write_blank_btree_header_node` lays the header bitmap record + at bytes `270..node_size−8` (= `(node_size−278)×8` bits), while + `map_nodes_required` assumes a `(node_size−256)×8`-bit header. Pick one and use + it everywhere; P2 deliberately clamps the blank to the **smaller** (270-based) + value, so grow code should too. +- **Journaling.** macOS formats HFS+ as **journaled (HFS+J)** by default. We + refuse *dirty* journaled volumes (`prepare_for_edit`) and edit a *clean* one in + place without writing through the journal, relying on sync-boundary + consistency. Growth mutates VH + allocation bitmap + fork extents — all + journaled metadata. After our edit the on-disk state must be self-consistent so + macOS replays an empty journal and mounts cleanly. **Test the non-journaled + case first** (our blanks are non-journaled), then a macOS-made journaled + volume. +- **Atomicity / rollback.** `create_file` snapshots the whole FS for rollback on + error; a grow that half-allocated must leave `bitmap` / `free_blocks` / fork / + header consistent so the snapshot guard restores cleanly. `allocate_extents` + already rolls back partial allocations — follow that discipline. +- **Clump alignment.** Grow by a clump, not a single node, or a per-`put` + workload re-enters grow constantly and fragments the fork (defeating Phase A). +- **`next_allocation` / free-space accounting.** Keep `vh.free_blocks` and the + bitmap in lock-step; `fsck_hfs` recomputes the bitmap from all fork extents and + flags any mismatch (the "bitmap reports X but should be Y" error class). +- **Special-file extent records (Phase B).** Overflow extents for the catalog + itself are keyed by the reserved CNIDs (3/4/6/8), not user CNIDs — easy to get + wrong, and `fsck_hfs` checks them specifically. + +## 6. In-repo tests (write these alongside the code) + +These let you iterate without a Mac; §7 is the real cross-check. + +- **Phase A unit:** build a volume whose catalog is deliberately tiny (e.g. + `create_blank_hfsplus_sized(.., min_catalog_bytes = small)`), insert enough + records to force ≥ 2 grows, then assert: every insert succeeded, `total_nodes` + grew, `hfsplus_fsck` clean, every record still findable, and a reopen + (`HfsPlusFilesystem::open`) re-reads the grown fork (proves the fork extents + + header persisted). +- **Contiguity:** after Phase-A growth the catalog fork should still be 1–2 + extents (assert `vh.catalog_file` extent count stays low). +- **Phase B unit:** pre-fragment the volume bitmap so the fork is forced to a 9th + extent; grow; reopen; read the whole catalog back; `hfsplus_fsck` clean. (Reuse + the fragmentation trick from + `test_hfsplus_fragmented_file_splits_extents_overflow_btree_real_path`.) +- **Phase C unit:** a large enough catalog to cross the map-node boundary (sizing + the blank just under the cap, then growing past it); `hfsplus_fsck` clean; + walk + count all records. +- Keep `cargo test --lib`, `cargo build --all-targets` (zero warnings), and + `cargo clippy -- -D warnings` green (CONTRIBUTING hard rules). + +## 7. macOS validation recipe (the real acceptance test) + +Run on a Mac. **HFS+ / HFSX is still fully supported by macOS** (read, write, +mount, `diskutil`, `fsck_hfs`) — only *classic* "HFS Standard" became read-only +(High Sierra) and then unreadable. We are exercising HFS **Extended**, which is +fine through current macOS. + +> ### CLI reality check — two gaps to close first +> +> - **rb-cli can't *create* an HFS+ volume.** `new --fs` accepts only +> `{hfs, hfv, fat, efs, affs, ntfs}` (`src/cli/verbs/new.rs`); `hfs`/`hfv` are +> *classic* HFS. HFS+ blanks only exist internally (`create_blank_hfsplus*`, +> used by restore/clone/tests). So there is no `rb-cli new --fs hfsplus` to make +> a small-catalog test volume. +> - **`rb-cli put` copies a *host file*, not stdin** (`PutArgs.host_file: PathBuf`, +> or `--zero N` to zero-fill). `put IMG HOSTFILE DEST`. `mkdir`/`put` *do* edit +> an existing HFS+ image (they route through `open_editable_filesystem`). +> +> **Phase 0 (prerequisite, do this with the §4b work):** expose HFS+ creation on +> the CLI — `new --fs hfsplus [--case-sensitive] [--min-catalog BYTES]` wired to +> `create_blank_hfsplus_sized`, per the GUI/CLI-parity rule in CLAUDE.md. The +> `--min-catalog` knob (a *small* value) is what lets you build a volume whose +> catalog is intentionally too small, so a few thousand `put`s force the grow path +> on demand instead of needing hundreds of thousands of files. Until Phase 0 +> lands, use **method 1** (a test emits the image) for grow coverage. + +### Method 1 — a Rust test emits a grown image; macOS judges it (works today) + +The Phase-A/B/C unit tests (§6) build a small-catalog HFS+ volume entirely +in-process and grow it. Have the test write its final image to a file: + +```rust +// at the end of the grow test, after our own hfsplus_fsck is clean: +fs.sync_metadata().unwrap(); // flush VH / bitmap / fork extents / tree buffer to the image +std::fs::write("/tmp/grown-hfsplus.img", fs.reader.get_ref()).unwrap(); +``` + +Then on the Mac: + +```sh +# raw single-volume HFS+ image (no partition map) -> attach without mounting +hdiutil attach -nomount -imagekey diskimage-class=CRawDiskImage /tmp/grown-hfsplus.img +# -> prints e.g. /dev/disk7 +fsck_hfs -f -n /dev/rdisk7 # -n = check, never modify; must say "appears to be OK" +fsck_hfs -d -f -n /dev/rdisk7 # verbose, if the first reports anything +hdiutil detach /dev/disk7 + +# mount read-only and walk every record +hdiutil attach -readonly /tmp/grown-hfsplus.img # /Volumes/ +find "/Volumes/" -type f | wc -l # must equal the test's record count +diskutil verifyVolume /dev/disk7 +hdiutil detach /dev/disk7 +``` + +If the test produces an APM/GPT-wrapped image instead of a bare volume, +`hdiutil attach -nomount` exposes the `Apple_HFS` slice as `/dev/disk7s2` — run +`fsck_hfs` against that slice. + +### Method 2 — round-trip a *macOS-formatted* HFS+ volume (the foreign-catalog case) + +This is the scenario §4b actually targets: a catalog built by Apple, then grown +by us. Needs Phase 0 (so `rb-cli` can edit it) and enough files to outgrow the +catalog macOS reserved. + +```sh +# 1) macOS builds a journaled HFS+ volume and seeds it. +hdiutil create -size 400m -fs HFS+ -volname Grow -type UDIF macgrow.dmg +hdiutil attach macgrow.dmg # /Volumes/Grow +mkdir -p /Volumes/Grow/seed +hdiutil detach /Volumes/Grow # leaves a CLEAN journal (we refuse dirty) + +# 2) Convert to a raw, read/write image rb-cli can edit in place. +hdiutil convert macgrow.dmg -format UDRW -o macgrow.img +RB=./target/release/rb-cli +printf x > /tmp/x # a tiny host file to copy in + +# 3) Fill past the catalog macOS reserved. Use img@1 if it has a partition map +# (rb-cli prints the partitions for `rb-cli inspect macgrow.img`); a bare HFS+ +# volume needs no @N. Make many small files across many dirs (worst case for +# node usage). This is slow one-at-a-time; scripts can batch with `rb-cli batch`. +for d in $(seq 0 199); do $RB mkdir macgrow.img "/seed/d$d" -q; done +for i in $(seq 1 40000); do + d=$(( i % 200 )) + $RB put macgrow.img /tmp/x "/seed/d$d/f$i.txt" -q || { echo "FAILED at $i"; break; } +done +$RB fsck macgrow.img # our own checker first + +# 4) The verdict: macOS fsck_hfs on the grown volume. +hdiutil attach -nomount macgrow.img +fsck_hfs -f -n /dev/rdiskN # MUST be clean — Apple-built catalog, grown by us +hdiutil attach -readonly macgrow.img && find /Volumes/Grow -type f | wc -l && hdiutil detach /dev/diskN +``` + +### What "pass" means + +- `fsck_hfs -f -n` reports the volume **appears to be OK** — no + `IndexSiblingLinkBroken`, no "Invalid node structure", no bitmap / free-block + mismatch, no key-order errors — on a volume that required **≥ 1 grow** (confirm + the catalog `total_blocks` is larger than the freshly-built size). +- The volume **mounts** and `find` enumerates every file written. +- **Phase C bonus:** push a catalog past ~117 MiB (a multi-GiB volume, + ≳ several-hundred-thousand small files) to exercise map-node appending, and + `fsck_hfs` that too. + +Paste the exact `fsck_hfs` output into the PR — *"`fsck_hfs` clean after N grows, +catalog grew from X to Y nodes"* is the deliverable, not our own `hfsplus_fsck`. + +## 8. Acceptance criteria + +```text +1. A volume whose catalog/extents/attributes fork is too small for its record + count grows in place through live puts; no spurious DiskFull while the VOLUME + still has free blocks. +2. hfsplus_fsck clean AND `fsck_hfs -fy` clean on macOS, after >= 1 grow (Phase A), + a > 8-extent fork (Phase B), and a past-the-cap catalog (Phase C). +3. The grown volume mounts on macOS and every record is readable; file/dir counts + match. +4. Reopening the image in rusty-backup re-reads the grown fork correctly (extents, + header counts, node bitmap) and further edits still succeed. +5. Classic HFS and the existing HFS+ paths are unaffected (P1–P5 tests stay green; + blanks that never hit the grow path are byte-identical to today). +``` + +## 9. Code pointers + +- `src/fs/hfs_common.rs` — `btree_alloc_node` (:1071, DiskFull at :1092); + `BTreeHeader` total/free nodes; node-bitmap helpers `btree_node_bitmap_range` + (:939), `map_node_bitmap_bytes` (:950), `map_nodes_required` (:961), + `init_map_node` (:1137), `btree_bitmap_segments`, `btree_bitmap_set`/`_clear`. +- `src/fs/hfsplus.rs` — the three insert methods (`insert_catalog_record`, + `insert_xattr_record`, `insert_extents_overflow_record`); `allocate_extents` + (:2294), `free_blocks` (:2356), `write_catalog` (:2258), + `write_allocation_bitmap` (:2269), `read_fork_with_overflow` (:613); + `ForkData` (:92, 8 inline extents); VH `catalog_file`/`extents_file`/ + `attributes_file` + `free_blocks`/`next_allocation`/`write_count`; + `build_blank_hfsplus_front` (node-bitmap cap `(node_size − 278) × 8`), + `create_blank_hfsplus_sized` (use for the tiny-catalog test fixtures). +- `src/fs/hfsplus_defrag.rs` — the *upfront* map-node + sized-fork builder + (`map_nodes_required` call site, ~:1401) — the reference for Phase C, just + applied incrementally. +- `src/fs/hfsplus_fsck.rs` — extend coverage so our own fsck validates grown + forks (node-bitmap accounting, map-node chain, overflow extents) before the + macOS cross-check. +``` diff --git a/src/cli/verbs/optical.rs b/src/cli/verbs/optical.rs index e3f1d5d1..e0c9522a 100644 --- a/src/cli/verbs/optical.rs +++ b/src/cli/verbs/optical.rs @@ -23,7 +23,7 @@ use crate::backup::LogMessage; use crate::cli::logging::log_stderr; use crate::optical::{ convert::{bincue_to_iso, chd_to_bincue, chd_to_iso, iso_to_bincue, to_chd, ConvertProgress}, - rip::{run_rip, RipConfig, RipFormat, RipProgress}, + rip::{run_rip, OpticalTarget, RipConfig, RipFormat, RipProgress}, }; use crate::partition::format_size; use crate::rbformats::chd_options::{ChdOptions, ChdProfile}; @@ -31,7 +31,7 @@ use crate::rbformats::chd_options::{ChdOptions, ChdProfile}; #[derive(Debug, Subcommand)] pub enum OpticalCommand { /// List connected physical optical drives and their device paths. - Drives, + Drives(DrivesArgs), /// Rip a physical CD/DVD drive to a disk image file. Rip(RipArgs), /// Re-encode an optical image into a different format. @@ -44,7 +44,7 @@ pub enum OpticalCommand { pub fn run(cmd: OpticalCommand) -> Result<()> { match cmd { - OpticalCommand::Drives => run_drives_verb(), + OpticalCommand::Drives(a) => run_drives_verb(a), OpticalCommand::Rip(a) => run_rip_verb(a), OpticalCommand::Convert(a) => run_convert_verb(a), OpticalCommand::Browse(a) => run_browse_verb(a), @@ -54,19 +54,42 @@ pub fn run(cmd: OpticalCommand) -> Result<()> { // ---------------- drives ---------------- -/// List physical optical drives, mirroring the GUI Optical tab's drive picker -/// (`opticaldiscs::drives::list_drives`). Prints one drive per line to stdout as -/// ` ` so the path can be fed to `optical rip -/// --device`. ASCII only (no glyphs) per the project's terminal-output rule. -fn run_drives_verb() -> Result<()> { - let drives = opticaldiscs::drives::list_drives(); - if drives.is_empty() { +#[derive(Debug, Args)] +pub struct DrivesArgs { + /// Also query these daemons for their optical drives (repeatable), e.g. + /// `--remote mister.local:7341`. Remote rows print an `rb://...` device arg + /// you can pass straight to `optical rip --device`. + #[arg(long = "remote", value_name = "HOST:PORT")] + pub remotes: Vec, +} + +/// List optical drives via the unified picker core: local drives, plus the +/// drives of each `--remote` daemon. Prints one drive per line to stdout as +/// ` ` — `` is a local path or an +/// `rb://host:port/dev/sr0` URL, feedable straight to `optical rip --device`. +/// ASCII only (no glyphs) per the project's terminal-output rule. +fn run_drives_verb(args: DrivesArgs) -> Result<()> { + // `devices` is only mutated by the remote arm below. + #[cfg_attr(not(feature = "remote"), allow(unused_mut))] + let mut devices = crate::model::optical_devices::list_local_rip_devices(); + #[cfg(feature = "remote")] + for addr in &args.remotes { + let conn = crate::remote::connection::RemoteConnection::connect_shared(addr) + .with_context(|| format!("connecting to {addr}"))?; + crate::model::optical_devices::append_remote_rip_devices(&mut devices, &conn); + } + #[cfg(not(feature = "remote"))] + if !args.remotes.is_empty() { + bail!("--remote needs the `remote` feature; this binary was built without it"); + } + + if devices.is_empty() { log_stderr("No optical drives found."); return Ok(()); } - log_stderr(format!("Found {} optical drive(s):", drives.len())); - for d in &drives { - println!("{} {}", d.device_path.display(), d.display_name); + log_stderr(format!("Found {} optical drive(s):", devices.len())); + for d in &devices { + println!("{} {}", d.cli_device_arg(), d.display_name); } Ok(()) } @@ -92,7 +115,10 @@ impl From for RipFormat { #[derive(Debug, Args)] pub struct RipArgs { - /// Source drive (e.g. `/dev/sr0`, `disk6`, `\\.\E:`). See `rb-cli show devices`. + /// Source drive: a local path (e.g. `/dev/sr0`, `disk6`, `\\.\E:`) or a + /// remote daemon's drive as `rb://host:port/dev/sr0` (the daemon issues the + /// SCSI reads; this side does the encoding). `rb-cli optical drives` lists + /// local drives. #[arg(long)] pub device: PathBuf, @@ -116,7 +142,7 @@ fn run_rip_verb(args: RipArgs) -> Result<()> { args.format )); let config = RipConfig { - device_path: args.device, + device: OpticalTarget::resolve(&args.device.to_string_lossy())?, output_path: args.output, format: args.format.into(), eject_after: args.eject, @@ -134,6 +160,7 @@ fn run_rip_verb(args: RipArgs) -> Result<()> { fn drain_rip(progress: Arc>) -> Result<()> { let mut last_op = String::new(); let mut last_pct: i32 = -1; + let mut tracker = crate::model::rate_tracker::RateTracker::default(); loop { std::thread::sleep(Duration::from_millis(250)); let (logs, op, cur, total, finished, error) = match progress.lock() { @@ -153,6 +180,9 @@ fn drain_rip(progress: Arc>) -> Result<()> { for LogMessage { level, message } in logs { log_stderr(format!("[{level:?}] {message}")); } + // Sample every tick (the stage label resets the window between phases), + // even though we only print every 5%, so the rate/ETA is warm by then. + tracker.record(cur, &op); if op != last_op { log_stderr(format!("status: {op}")); last_op = op; @@ -162,9 +192,10 @@ fn drain_rip(progress: Arc>) -> Result<()> { let pct = ((cur as f64 / total as f64) * 100.0) as i32; if pct / 5 != last_pct / 5 { log_stderr(format!( - " progress: {pct:>3}% ({}/{})", + " progress: {pct:>3}% ({}/{}){}", format_size(cur), - format_size(total) + format_size(total), + tracker.suffix(cur, total), )); last_pct = pct; } @@ -253,6 +284,7 @@ fn run_convert_verb(args: ConvertArgs) -> Result<()> { fn drain_convert(progress: Arc>) -> Result<()> { let mut last_op = String::new(); let mut last_pct: i32 = -1; + let mut tracker = crate::model::rate_tracker::RateTracker::default(); loop { std::thread::sleep(Duration::from_millis(250)); let (logs, op, cur, total, finished, error) = match progress.lock() { @@ -272,6 +304,9 @@ fn drain_convert(progress: Arc>) -> Result<()> { for LogMessage { level, message } in logs { log_stderr(format!("[{level:?}] {message}")); } + // Sample every tick (the stage label resets the window between phases), + // even though we only print every 5%, so the rate/ETA is warm by then. + tracker.record(cur, &op); if op != last_op { log_stderr(format!("status: {op}")); last_op = op; @@ -281,9 +316,10 @@ fn drain_convert(progress: Arc>) -> Result<()> { let pct = ((cur as f64 / total as f64) * 100.0) as i32; if pct / 5 != last_pct / 5 { log_stderr(format!( - " progress: {pct:>3}% ({}/{})", + " progress: {pct:>3}% ({}/{}){}", format_size(cur), - format_size(total) + format_size(total), + tracker.suffix(cur, total), )); last_pct = pct; } diff --git a/src/fs/filesystem.rs b/src/fs/filesystem.rs index 1a8ddffe..b7c79a6c 100644 --- a/src/fs/filesystem.rs +++ b/src/fs/filesystem.rs @@ -315,6 +315,23 @@ pub enum ResourceForkSource { /// to flush changes to disk. This enables batching multiple edits into a single atomic /// write. pub trait EditableFilesystem: Filesystem { + /// Enter "bulk" editing mode for a batch of operations whose atomicity the + /// *caller* guarantees as a whole (e.g. an importer that discards every + /// change if any single op fails). Implementations may use this to skip + /// per-operation rollback bookkeeping — for filesystems that snapshot the + /// whole metadata on each edit (classic HFS clones its multi-megabyte + /// catalog per `create_file`), that bookkeeping dominates a large import. + /// + /// Contract: while in bulk mode a *failed* edit may leave in-memory state + /// inconsistent, so the caller MUST abandon the volume (not sync/commit) on + /// any error. Always pair with [`end_bulk`](Self::end_bulk). The default is + /// a no-op, so filesystems whose edits are already cheap need do nothing. + fn begin_bulk(&mut self) {} + + /// Leave bulk editing mode (see [`begin_bulk`](Self::begin_bulk)). Default + /// is a no-op. + fn end_bulk(&mut self) {} + /// Create a file in the given parent directory. /// /// `data` is a reader providing the file contents; `data_len` is the total size. diff --git a/src/fs/hfs.rs b/src/fs/hfs.rs index 2e85e1a6..de0e77f0 100644 --- a/src/fs/hfs.rs +++ b/src/fs/hfs.rs @@ -8,7 +8,7 @@ use super::filesystem::{ }; use super::hfs_common::{ self, bitmap_clear_bit_be, bitmap_find_clear_run_be, bitmap_set_bit_be, btree_free_node, - btree_insert_record, btree_remove_record, BTreeHeader, + btree_remove_record, BTreeHeader, }; use super::CompactResult; @@ -425,6 +425,17 @@ pub struct HfsFilesystem { /// Volume (create, modify, backup) dates staged by `set_volume_dates`. /// When Some, overrides the automatic `hfs_now()` stamp in `sync_metadata`. pending_volume_dates: Option<(u32, u32, u32)>, + /// When set (via `begin_bulk` / `end_bulk`), per-operation rollback + /// snapshots are skipped: `snapshot()` returns `None`. The caller takes + /// responsibility for batch-level atomicity (discarding all changes if any + /// operation fails). Used by `tar_import` so a large import doesn't clone + /// the whole multi-megabyte catalog on every single `create_file`. + bulk_mode: bool, + /// Cache for `ensure_catalog_initialized`: once both B-trees are confirmed + /// live, this short-circuits the check. Without it every edit re-reads the + /// (multi-megabyte) extents-overflow fork from disk just to test whether + /// it's blank — death by a thousand reads on a large import. + btrees_initialized: bool, } impl HfsFilesystem { @@ -459,6 +470,8 @@ impl HfsFilesystem { bitmap: None, pending_boot_blocks: None, pending_volume_dates: None, + bulk_mode: false, + btrees_initialized: false, }) } @@ -803,35 +816,60 @@ impl HfsFilesystem { /// Find a catalog record by (parent_id, name). /// Returns Some((node_idx, rec_idx, absolute_offset_in_catalog_data)) if found. + /// + /// Descends the B-tree index to the single leaf that would hold the key and + /// scans only that leaf — O(log n), not a walk of the whole leaf chain. In + /// a consistent catalog a key lives in exactly one leaf (the one the search + /// path reaches), so this is exact. It matters at scale: `create_file` / + /// `create_directory` / `rename` call this for duplicate detection on every + /// entry, so a linear walk here made bulk imports (`untar`) O(n^2). The + /// index is kept consistent by the incremental inserter + /// ([`hfs_common::btree_insert_full`]); on a structurally broken catalog + /// (pre-fsck) prefer repairing first — editing assumes a healthy tree. fn find_catalog_record_by_name( &self, parent_id: u32, name: &[u8], ) -> Option<(u32, usize, usize)> { let search_key = Self::build_catalog_key(parent_id, name); - let node_size = BigEndian::read_u16(&self.catalog_data[32..34]) as usize; - let first_leaf = BigEndian::read_u32(&self.catalog_data[24..28]); + let header = BTreeHeader::read(&self.catalog_data); + let node_size = header.node_size as usize; + if node_size == 0 || self.catalog_data.len() < node_size { + return None; + } - hfs_common::walk_leaf_records( + let (leaf_idx, _) = hfs_common::btree_find_insert_leaf( &self.catalog_data, - first_leaf, - node_size, - |node_idx, rec_idx, abs_off, rec| { - if rec.len() < 7 { - return None; - } - let key_len = rec[0] as usize; - let key_end = 1 + key_len; - if key_end > rec.len() { - return None; - } - if Self::catalog_compare(&rec[..key_end], &search_key) == Ordering::Equal { - Some((node_idx, rec_idx, abs_off)) - } else { - None - } - }, - ) + &header, + &search_key, + &Self::catalog_compare, + ); + + let off = leaf_idx as usize * node_size; + if off + node_size > self.catalog_data.len() { + return None; + } + let node = &self.catalog_data[off..off + node_size]; + let num_records = BigEndian::read_u16(&node[10..12]) as usize; + for rec_idx in 0..num_records { + let (start, end) = hfs_common::btree_record_range(node, node_size, rec_idx); + if start >= end || end > node_size { + continue; + } + let rec = &node[start..end]; + if rec.len() < 7 { + continue; + } + let key_len = rec[0] as usize; + let key_end = 1 + key_len; + if key_end > rec.len() { + continue; + } + if Self::catalog_compare(&rec[..key_end], &search_key) == Ordering::Equal { + return Some((leaf_idx, rec_idx, off + start)); + } + } + None } /// Find a thread record by CNID (thread key: parent_id=cnid, name=""). @@ -888,62 +926,29 @@ impl HfsFilesystem { } /// Insert a catalog record into the B-tree, handling splits and growth. + /// + /// Delegates to the shared incremental inserter + /// ([`hfs_common::btree_insert_full`]) — the same find-leaf / + /// split-with-insert / grow-root-or-insert-into-parent machinery that HFS+ + /// and the streamed defrag builder use. It maintains the index nodes + /// incrementally (O(log n) per insert) rather than rebuilding the entire + /// index after every leaf split. + /// + /// The previous per-split `rebuild_index_nodes` approach was O(n) per split + /// (O(n^2) over a bulk import like `untar`) and, worse, corrupted the tree + /// once it grew past the header node's 2048-bit allocation bitmap: the + /// rebuild's free-all-index-nodes pass was capped at that bitmap, so index + /// nodes living in map-node-covered segments leaked. That drained the free + /// nodes until a rebuild ran out mid-flight and left the index with broken + /// sibling links — surfacing as the spurious "disk full: no free B-tree + /// nodes" / `IndexSiblingLinkBroken` failure at ~7.4k catalog records. fn insert_catalog_record(&mut self, key_record: &[u8]) -> Result<(), FilesystemError> { - let header = BTreeHeader::read(&self.catalog_data); - let node_size = header.node_size as usize; - - // Find the correct leaf node - let (leaf_idx, _parent_chain) = hfs_common::btree_find_insert_leaf( - &self.catalog_data, - &header, + hfs_common::btree_insert_full( + &mut self.catalog_data, key_record, + &hfs_common::BTreeKeyFormat::CLASSIC_CATALOG, &Self::catalog_compare, - ); - - // Try to insert into the leaf - let offset = leaf_idx as usize * node_size; - let node = &mut self.catalog_data[offset..offset + node_size]; - match btree_insert_record(node, node_size, key_record, &Self::catalog_compare) { - Ok(_) => { - let mut h = BTreeHeader::read(&self.catalog_data); - h.leaf_records += 1; - h.write(&mut self.catalog_data); - Ok(()) - } - Err(_) => { - // Leaf full — split-and-insert atomically. The merged - // partition uses a byte-based split point, which is robust - // to uneven catalog record sizes (split-then-insert with a - // count-based split could leave the target half too packed - // for the new record). - let mut h = BTreeHeader::read(&self.catalog_data); - let _ = hfs_common::btree_split_leaf_with_insert( - &mut self.catalog_data, - node_size, - leaf_idx, - &mut h, - key_record, - &Self::catalog_compare, - )?; - - h.leaf_records += 1; - h.write(&mut self.catalog_data); - - // Full index rebuild replaces incremental parent chain insertion - let mut dummy_report = super::fsck::RepairReport { - fixes_applied: Vec::new(), - fixes_failed: Vec::new(), - unrepairable_count: 0, - }; - super::hfs_fsck::rebuild_index_nodes( - &mut self.catalog_data, - node_size, - &mut dummy_report, - ); - - Ok(()) - } - } + ) } /// Remove a catalog record from a leaf node. @@ -1021,19 +1026,36 @@ impl HfsFilesystem { } /// Capture a snapshot of all mutable in-memory state for rollback. - fn snapshot(&self) -> (Vec, Option>, HfsMasterDirectoryBlock) { - ( + /// + /// Returns `None` in bulk mode (see [`HfsFilesystem::bulk_mode`]): the + /// clone of the whole catalog is the dominant per-operation cost during a + /// large import, and the caller guarantees batch-level atomicity there. + /// `restore_snapshot(None)` is then a no-op, so the existing + /// `let snap = self.snapshot(); ... restore_snapshot(snap)` call sites need + /// no changes. + fn snapshot(&self) -> Option<(Vec, Option>, HfsMasterDirectoryBlock)> { + if self.bulk_mode { + return None; + } + Some(( self.catalog_data.clone(), self.bitmap.clone(), self.mdb.clone(), - ) + )) } - /// Restore in-memory state from a previously captured snapshot. - fn restore_snapshot(&mut self, snap: (Vec, Option>, HfsMasterDirectoryBlock)) { - self.catalog_data = snap.0; - self.bitmap = snap.1; - self.mdb = snap.2; + /// Restore in-memory state from a previously captured snapshot. A `None` + /// snapshot (bulk mode) restores nothing — the caller owns rollback. + fn restore_snapshot( + &mut self, + snap: Option<(Vec, Option>, HfsMasterDirectoryBlock)>, + ) { + let Some((catalog_data, bitmap, mdb)) = snap else { + return; + }; + self.catalog_data = catalog_data; + self.bitmap = bitmap; + self.mdb = mdb; } pub(crate) fn mdb(&self) -> &HfsMasterDirectoryBlock { @@ -1910,6 +1932,12 @@ impl HfsFilesystem { /// them to disk. Subsequent edits can then use the normal insert paths /// instead of panicking on an empty B-tree header. fn ensure_catalog_initialized(&mut self) -> Result<(), FilesystemError> { + // Once confirmed live, never re-check: the extents read below hits disk + // and would otherwise repeat on every single edit (e.g. per file in a + // large `untar`). + if self.btrees_initialized { + return Ok(()); + } let catalog_blank = is_catalog_uninitialized(&self.catalog_data); // Read the extents B-tree bytes so we can detect a blank one. let extents_data = if self.mdb.extents_file_size > 0 { @@ -1929,9 +1957,12 @@ impl HfsFilesystem { .map(is_catalog_uninitialized) .unwrap_or(false); if !catalog_blank && !extents_blank { + self.btrees_initialized = true; return Ok(()); } - self.initialize_empty_btrees(catalog_blank, extents_blank) + self.initialize_empty_btrees(catalog_blank, extents_blank)?; + self.btrees_initialized = true; + Ok(()) } fn initialize_empty_btrees( @@ -2021,12 +2052,17 @@ impl HfsFilesystem { /// Classic Mac OS expects HFS B-tree node size = 512 bytes. While the on-disk /// format technically allows larger node sizes, Apple's reference tools and /// CiderPress2 both hard-code 512, and an emulated Quadra produced "error -/// type -127" when fed a catalog with 1024-byte nodes. Always use 512. +/// type -127" when fed a catalog with 1024-byte nodes. Always use 512 — this +/// is a hard mountability constraint, not a sizing convenience, so do *not* be +/// tempted to scale it up for large catalogs. /// -/// The header node's bitmap covers `(512 - 256) * 8 = 2048` nodes; trees that -/// need more nodes require additional MAP nodes (NodeType=2) chained off the -/// header — not yet implemented here. Callers should size their B-tree files -/// to ≤ 1 MB (2048 nodes × 512 bytes) until that's wired up. +/// The header node's bitmap covers only `(512 - 256) * 8 = 2048` nodes, but +/// that is no longer a ceiling: B-tree files larger than 2048 nodes chain MAP +/// nodes (NodeType=2) off the header to extend the allocation bitmap (see +/// [`hfs_map_nodes_required`] / [`hfs_common::init_map_node`] and the +/// segment-aware bitmap helpers). Catalogs of tens of thousands of nodes are +/// fully supported at node_size=512; the index is maintained incrementally so +/// there is no per-split rebuild to choke on the larger node count. fn pick_btree_node_size(_target_bytes: u64) -> u16 { 512 } @@ -2795,6 +2831,14 @@ impl Filesystem for HfsFilesystem { const CATALOG_DIR_THREAD: i8 = 3; impl EditableFilesystem for HfsFilesystem { + fn begin_bulk(&mut self) { + self.bulk_mode = true; + } + + fn end_bulk(&mut self) { + self.bulk_mode = false; + } + fn create_file( &mut self, parent: &FileEntry, @@ -2893,9 +2937,10 @@ impl EditableFilesystem for HfsFilesystem { // No file thread record: classic HFS file threads are optional // (Inside Macintosh: Files), and Finder / CiderPress2 don't emit // them. Skipping them roughly halves catalog size for file-heavy - // volumes and lets dense sources (~5000 entries) fit within the - // 2048-node header-bitmap cap. File CNID lookups fall back to a - // leaf scan in `locate_record_data` / `find_file_record_offset_by_cnid`. + // volumes (map nodes now lift the old 2048-node header-bitmap limit, + // so this is a size/compat choice, not a hard cap workaround). File + // CNID lookups fall back to a leaf scan in `locate_record_data` / + // `find_file_record_offset_by_cnid`. // Update parent valence self.update_parent_valence(parent_id, 1)?; @@ -4667,6 +4712,497 @@ mod tests { assert_fsck_clean(&mut fs); } + /// Random-order bulk inserts must pack the catalog densely *and* stay + /// fsck-clean. The greedy "pack-left-full" split used to split off only the + /// overflow on a middle insert, leaving a trail of ~1-record leaves: a + /// random import packed ~1.6 records/node, roughly half what classic Mac OS + /// writes (a real MacPack disk sits near 3.0). The B*-style sibling rotation + /// (redistribute into a neighbour before splitting, at both the leaf and the + /// index level) lifts random-order packing to ~3.0 records/node while + /// leaving sequential growth fully packed (4 records/leaf). + #[test] + fn test_hfs_random_insert_packs_densely_and_fsck_clean() { + use super::hfs_common::BTreeHeader; + type Fs = HfsFilesystem>>; + let block_size = 32 * 1024u32; + let img = create_blank_hfs_sized(64 * 1024 * 1024, block_size, "Pack", 0, 16 * 1024 * 1024) + .unwrap(); + let mut fs = Fs::open(Cursor::new(img), 0).unwrap(); + let root_id = fs.root().unwrap().location as u32; + + // Pseudo-random insertion order (multiplicative hash), de-duplicated. + let n = 16000u32; + let mut seen = std::collections::HashSet::new(); + let order: Vec = (0..n) + .map(|i| i.wrapping_mul(2654435761) % n) + .filter(|x| seen.insert(*x)) + .collect(); + let count = order.len() as u32; + fs.begin_bulk(); + for &i in &order { + let name = format!("f{:05}", i); + let mut kr = Fs::build_catalog_key(root_id, name.as_bytes()); + kr.extend_from_slice(&Fs::build_file_record( + 16 + i, + 0, + 0, + 0, + 0, + 0, + 0, + &[0u8; 4], + &[0u8; 4], + block_size, + )); + fs.insert_catalog_record(&kr).unwrap(); + } + fs.end_bulk(); + fs.mdb.file_count = count; + fs.mdb.next_catalog_id = 16 + n; + fs.update_parent_valence(root_id, count as i32).unwrap(); + + // Density: well above the ~1.6 the old split produced, and now close to + // the ~3.2 of a fully-packed sequential build thanks to sibling rotation. + let h = BTreeHeader::read(fs.catalog_data()); + let used = h.total_nodes - h.free_nodes; + let rec_per_node = count as f64 / used as f64; + assert!( + rec_per_node > 2.8, + "random-order packing too sparse: {count} records in {used} nodes ({rec_per_node:.2}/node)" + ); + + let result = fs.fsck().unwrap(); + assert!( + result.errors.is_empty(), + "fsck errors after random import: {:?}", + result + .errors + .iter() + .take(8) + .map(|e| &e.message) + .collect::>() + ); + } + + /// Bulk mode (`begin_bulk` / `end_bulk`) skips the per-operation catalog + /// snapshot for speed — the dominant per-file cost on a large import once + /// the duplicate check is index-based. A successful batch must still leave + /// a fsck-clean volume identical to the per-op path (on the success path no + /// rollback ever happens, so the only difference is the skipped clone). + /// This is the path `tar_import` drives. + #[test] + fn test_hfs_bulk_mode_import_fsck_clean() { + let block_size = 32 * 1024u32; + let img = create_blank_hfs_sized( + 64 * 1024 * 1024, + block_size, + "BulkImport", + 0, + 8 * 1024 * 1024, + ) + .expect("create blank volume"); + let mut fs = HfsFilesystem::open(Cursor::new(img), 0).unwrap(); + let root = fs.root().unwrap(); + let root_id = root.location as u32; + let options = CreateFileOptions::default(); + + fs.begin_bulk(); + for i in 0..9000 { + let name = format!("f{:05}.txt", i); + let mut empty = Cursor::new(Vec::new()); + fs.create_file(&root, &name, &mut empty, 0, &options) + .unwrap_or_else(|e| panic!("bulk create_file #{i}: {e}")); + } + fs.end_bulk(); + + assert_eq!(fs.mdb.file_count, 9000); + let result = fs.fsck().unwrap(); + assert!( + result.errors.is_empty(), + "bulk-mode import left fsck errors: {:?}", + result + .errors + .iter() + .take(8) + .map(|e| &e.message) + .collect::>() + ); + // Records inserted under skipped snapshots are still correct/findable. + assert!(fs + .find_catalog_record_by_name(root_id, b"f04500.txt") + .is_some()); + assert_eq!(fs.list_directory(&root).unwrap().len(), 9000); + + // After end_bulk, normal snapshotting resumes: a duplicate create is + // rejected and rolls back cleanly, leaving the volume fsck-clean. + let mut empty = Cursor::new(Vec::new()); + assert!(fs + .create_file(&root, "f00000.txt", &mut empty, 0, &options) + .is_err()); + assert_fsck_clean(&mut fs); + } + + /// Regression for the classic-HFS catalog B-tree scaling bug + /// (PROMPT-hfs-catalog-btree-scaling): importing many files used to die at + /// ~7.4k records with "disk full: no free B-tree nodes" and fsck reported + /// widespread `IndexSiblingLinkBroken`. Two writer defects combined — the + /// per-split index rebuild leaked index nodes past the 2048-node header + /// bitmap (exhausting free nodes), and the incremental index split never + /// maintained fLink/bLink sibling links. Drive the B-tree well past that + /// boundary on a production-shaped volume (node_size 512) and assert it is + /// fsck-clean. + /// + /// Uses the low-level inserter directly so the test stays fast (the + /// higher-level `create_file` does an O(n) duplicate scan per call); the + /// end-to-end path is covered by + /// `test_hfs_create_file_multilevel_index_fsck_clean`. + #[test] + fn test_hfs_catalog_scales_past_header_bitmap_fsck_clean() { + use super::hfs_common::BTreeHeader; + type Fs = HfsFilesystem>>; + let block_size = 32 * 1024u32; + // Empty files cost no data blocks, so a 64 MiB volume with a generous + // 10 MiB catalog (20,480 nodes @ 512) holds the records and forces the + // catalog past the header bitmap's 2048-node window into map nodes. + let img = create_blank_hfs_sized( + 64 * 1024 * 1024, + block_size, + "Scale20k", + 0, + 10 * 1024 * 1024, + ) + .expect("create blank volume"); + let mut fs = Fs::open(Cursor::new(img), 0).unwrap(); + let root = fs.root().unwrap(); + let root_id = root.location as u32; + + let n: u32 = 22_000; + for i in 0..n { + let name = format!("f{:05}.txt", i); + let mut key_record = Fs::build_catalog_key(root_id, name.as_bytes()); + let file_rec = + Fs::build_file_record(16 + i, 0, 0, 0, 0, 0, 0, &[0u8; 4], &[0u8; 4], block_size); + key_record.extend_from_slice(&file_rec); + fs.insert_catalog_record(&key_record) + .unwrap_or_else(|e| panic!("insert #{i} ({name}): {e}")); + } + + // Mirror the MDB bookkeeping the create path maintains so fsck's + // cross-checks (file_count, next_catalog_id, root valence) pass. + fs.mdb.file_count = n; + fs.mdb.next_catalog_id = 16 + n; + fs.update_parent_valence(root_id, n as i32).unwrap(); + + // Confirm the catalog really grew a multi-level index beyond the single + // header bitmap — otherwise the test wouldn't exercise the bug at all. + let header = BTreeHeader::read(fs.catalog_data()); + assert!( + header.depth >= 3, + "expected a multi-level index, depth = {}", + header.depth + ); + let used = header.total_nodes - header.free_nodes; + assert!( + used > 2048, + "expected > 2048 catalog nodes in use, got {used}" + ); + + let result = fs.fsck().unwrap(); + assert!( + result.errors.is_empty(), + "fsck found {} errors after {n}-record import: {:?}", + result.errors.len(), + result + .errors + .iter() + .take(8) + .map(|e| &e.message) + .collect::>() + ); + + // After all those splits, records must still be findable by descending + // the index (locates the correct leaf), including across the old + // failure point (~7386) and the map-node boundary. + for i in [0u32, 1, 7386, 12345, n - 1] { + let name = format!("f{:05}.txt", i); + assert!( + fs.find_catalog_record_by_name(root_id, name.as_bytes()) + .is_some(), + "record {name} not findable after bulk insert" + ); + } + } + + /// Per-`put`-shaped regression (PROMPT-hfs-catalog-incremental-put-packing): + /// 20k records inserted ONE AT A TIME in non-sequential (shuffled) key + /// order, spread across many parent directories — exactly the shape of a + /// `rb-cli put`-driven image build — must pack the catalog nearly as tightly + /// as a sequential bulk import and stay fsck-clean to the volume's real + /// limit. Before the B*-style sibling rotation (leaf + index), a shuffled + /// per-record build left leaves at the classic ~69% B-tree occupancy and the + /// index far worse, exhausting the node budget ~2x sooner ("disk full: no + /// free B-tree nodes") and leaving `IndexSiblingLinkBroken` at the ceiling. + #[test] + fn test_hfs_incremental_shuffled_multidir_packs_dense_fsck_clean() { + use super::hfs_common::BTreeHeader; + type Fs = HfsFilesystem>>; + let block_size = 32 * 1024u32; + // ~6 MiB catalog (12,288 nodes @ 512). With the rotation fix 20k records + // pack into well under that; the pre-fix ~1.6 rec/node packing would have + // needed far more leaf nodes alone and exhausted it. + let img = create_blank_hfs_sized( + 128 * 1024 * 1024, + block_size, + "PutShuffle", + 0, + 6 * 1024 * 1024, + ) + .expect("create blank volume"); + let mut fs = Fs::open(Cursor::new(img), 0).unwrap(); + + // Real directories so threads / valence / counts stay fsck-correct. + const DIRS: u32 = 200; + let root = fs.root().unwrap(); + let mut dir_ids = Vec::with_capacity(DIRS as usize); + for d in 0..DIRS { + let name = format!("dir{d:03}"); + let de = fs + .create_directory(&root, &name, &Default::default()) + .unwrap(); + dir_ids.push(de.location as u32); + } + let base_cnid = fs.mdb.next_catalog_id; + + // 20k files in shuffled key order (multiplicative hash is a bijection + // mod 2^32, so names are unique and land mid-leaf, not appended), + // sprayed across the directories. + let n: u32 = 20_000; + let mut per_dir = vec![0i32; DIRS as usize]; + for i in 0..n { + let hash = i.wrapping_mul(2_654_435_761); + let d = (hash % DIRS) as usize; + let name = format!("f{hash:08x}"); + let mut kr = Fs::build_catalog_key(dir_ids[d], name.as_bytes()); + kr.extend_from_slice(&Fs::build_file_record( + base_cnid + i, + 0, + 0, + 0, + 0, + 0, + 0, + &[0u8; 4], + &[0u8; 4], + block_size, + )); + fs.insert_catalog_record(&kr) + .unwrap_or_else(|e| panic!("insert #{i} into dir{d}: {e}")); + per_dir[d] += 1; + } + // Mirror the bookkeeping the create path would have maintained so fsck's + // cross-checks (file_count, next_catalog_id, per-dir valence) pass. + fs.mdb.file_count += n; + fs.mdb.next_catalog_id = base_cnid + n; + for (d, &cnt) in per_dir.iter().enumerate() { + fs.update_parent_valence(dir_ids[d], cnt).unwrap(); + } + + // Density: the whole catalog (leaves + index) averages well above the + // pre-fix ~1.6 rec/node — proof the incremental build packed tight and + // didn't run away into a sparse, over-deep tree. + let h = BTreeHeader::read(fs.catalog_data()); + let used = h.total_nodes - h.free_nodes; + let rec_per_node = h.leaf_records as f64 / used as f64; + assert!( + rec_per_node > 2.6, + "shuffled multi-dir packing too sparse: {} records in {used} nodes ({rec_per_node:.2}/node)", + h.leaf_records + ); + + // No IndexSiblingLinkBroken or other structural damage at the ceiling. + let result = fs.fsck().unwrap(); + assert!( + result.errors.is_empty(), + "fsck found {} errors after 20k shuffled multi-dir put: {:?}", + result.errors.len(), + result + .errors + .iter() + .take(8) + .map(|e| &e.message) + .collect::>() + ); + + // Records remain findable by index descent across the keyspace. + for i in [0u32, 7, 9_999, n - 1] { + let hash = i.wrapping_mul(2_654_435_761); + let parent = dir_ids[(hash % DIRS) as usize]; + let name = format!("f{hash:08x}"); + assert!( + fs.find_catalog_record_by_name(parent, name.as_bytes()) + .is_some(), + "record {name} not findable after shuffled multi-dir insert" + ); + } + } + + /// The delete / fsck-repair path rebuilds the catalog index wholesale via + /// `rebuild_index_nodes`. Its free-all-index-nodes pass used to be capped + /// at the 2048-bit header bitmap, leaking every index node beyond it; once + /// a catalog crosses that line a rebuild must still round-trip fsck-clean. + #[test] + fn test_rebuild_index_nodes_past_header_bitmap_fsck_clean() { + use super::hfs_common::BTreeHeader; + type Fs = HfsFilesystem>>; + let block_size = 32 * 1024u32; + let img = + create_blank_hfs_sized(64 * 1024 * 1024, block_size, "Rebuild", 0, 10 * 1024 * 1024) + .expect("create blank volume"); + let mut fs = Fs::open(Cursor::new(img), 0).unwrap(); + let root_id = fs.root().unwrap().location as u32; + + let n: u32 = 12_000; + for i in 0..n { + let name = format!("f{:05}.txt", i); + let mut key_record = Fs::build_catalog_key(root_id, name.as_bytes()); + let file_rec = + Fs::build_file_record(16 + i, 0, 0, 0, 0, 0, 0, &[0u8; 4], &[0u8; 4], block_size); + key_record.extend_from_slice(&file_rec); + fs.insert_catalog_record(&key_record).unwrap(); + } + fs.mdb.file_count = n; + fs.mdb.next_catalog_id = 16 + n; + fs.update_parent_valence(root_id, n as i32).unwrap(); + + let node_size = BTreeHeader::read(fs.catalog_data()).node_size as usize; + let used_before = { + let h = BTreeHeader::read(fs.catalog_data()); + h.total_nodes - h.free_nodes + }; + assert!( + used_before > 2048, + "test needs > 2048 nodes, got {used_before}" + ); + + // Force a full index rebuild and require it to succeed without leaking. + let mut report = crate::fs::fsck::RepairReport { + fixes_applied: Vec::new(), + fixes_failed: Vec::new(), + unrepairable_count: 0, + }; + crate::fs::hfs_fsck::rebuild_index_nodes(&mut fs.catalog_data, node_size, &mut report); + assert!( + report.fixes_failed.is_empty(), + "rebuild reported failures: {:?}", + report.fixes_failed + ); + + let result = fs.fsck().unwrap(); + assert!( + result.errors.is_empty(), + "fsck after rebuild: {:?}", + result + .errors + .iter() + .take(8) + .map(|e| &e.message) + .collect::>() + ); + } + + /// End-to-end guard through the real `create_file` API: enough files to + /// grow a multi-level catalog index must stay fsck-clean. ~2000 files at + /// node_size 512 yields ~500 leaves and tens of index nodes — enough to + /// split index nodes, the exact spot the incremental inserter used to + /// leave sibling links broken. + #[test] + fn test_hfs_create_file_multilevel_index_fsck_clean() { + use super::hfs_common::BTreeHeader; + let block_size = 32 * 1024u32; + let img = + create_blank_hfs_sized(32 * 1024 * 1024, block_size, "MultiIdx", 0, 4 * 1024 * 1024) + .expect("create blank volume"); + let mut fs = HfsFilesystem::open(Cursor::new(img), 0).unwrap(); + let root = fs.root().unwrap(); + let options = CreateFileOptions::default(); + for i in 0..2000 { + let name = format!("f{:05}.txt", i); + let mut empty = Cursor::new(Vec::new()); + fs.create_file(&root, &name, &mut empty, 0, &options) + .unwrap_or_else(|e| panic!("create_file #{i}: {e}")); + } + let header = BTreeHeader::read(fs.catalog_data()); + assert!( + header.depth >= 2, + "expected an index level, depth = {}", + header.depth + ); + let result = fs.fsck().unwrap(); + assert!( + result.errors.is_empty(), + "fsck errors: {:?}", + result + .errors + .iter() + .take(8) + .map(|e| &e.message) + .collect::>() + ); + assert_eq!(fs.mdb.file_count, 2000); + } + + /// Lock in lossless round-tripping of filenames carrying raw control bytes + /// (e.g. 0x7F / DEL, as in the Macintosh Garden "IPNetRouter" + /// title that triggered the related JSON-escaping bug downstream). Mac + /// Roman maps bytes < 0x80 straight through, so create then list must + /// preserve them byte-for-byte. + #[test] + fn test_hfs_control_char_filename_roundtrip() { + let img = make_editable_hfs_image(); + let mut fs = HfsFilesystem::open(Cursor::new(img), 0).unwrap(); + let root = fs.root().unwrap(); + let options = CreateFileOptions::default(); + let name = "IPNetRouter\u{7f}\u{7f}_154_68k"; + let mut data = Cursor::new(b"x".to_vec()); + fs.create_file(&root, name, &mut data, 1, &options) + .expect("create_file with raw 0x7F bytes in the name"); + let entries = fs.list_directory(&root).unwrap(); + assert!( + entries.iter().any(|e| e.name == name), + "0x7F-laden name not preserved; got {:?}", + entries.iter().map(|e| &e.name).collect::>() + ); + assert_fsck_clean(&mut fs); + } + + /// A null byte in a catalog name is valid on classic HFS (only the colon is + /// forbidden) — a leading null is an old Finder trick to sort a file to the + /// top. Real Apple disks carry such names (3 on the MacPack boot volume), so + /// fsck must treat them as an informational *warning*, not an error. + #[test] + fn test_hfs_null_byte_name_is_warning_not_error() { + let img = make_editable_hfs_image(); + let mut fs = HfsFilesystem::open(Cursor::new(img), 0).unwrap(); + let root = fs.root().unwrap(); + let options = CreateFileOptions::default(); + let mut data = Cursor::new(b"x".to_vec()); + fs.create_file(&root, "\u{0}Sorts First", &mut data, 1, &options) + .expect("create_file with a leading null byte in the name"); + let result = fs.fsck().unwrap(); + assert!( + result.errors.is_empty(), + "a null-byte name must not be a fsck error: {:?}", + result.errors.iter().map(|e| &e.message).collect::>() + ); + assert!( + result + .warnings + .iter() + .any(|w| w.code == "UnusualCatalogName"), + "expected an UnusualCatalogName warning for the null-byte name" + ); + } + #[test] fn test_fsck_stress_many_files() { let img = make_editable_hfs_image(); diff --git a/src/fs/hfs_common.rs b/src/fs/hfs_common.rs index 2e9260bf..daa5d7cb 100644 --- a/src/fs/hfs_common.rs +++ b/src/fs/hfs_common.rs @@ -7,7 +7,6 @@ use byteorder::{BigEndian, ByteOrder}; use std::cmp::Ordering; use std::sync::OnceLock; -use unicode_normalization::UnicodeNormalization; use super::filesystem::FilesystemError; @@ -368,43 +367,26 @@ pub fn decode_ostype(code: &[u8; 4]) -> String { // B-tree key comparison // --------------------------------------------------------------------------- -/// Code points that Apple's `FastUnicodeCompare` treats as **ignorable** -/// during case-folded comparison — they're skipped entirely, as if absent. -/// -/// Source: Apple's `FastUnicodeCompare.c` `gLowerCaseTable`. The table -/// maps zero-width / format / variation-selector ranges to 0x0000 (the -/// algorithm's "skip" sentinel). Latin-letter casing is covered by -/// `char::to_uppercase` below. -/// -/// **Note**: U+0000 (NUL) is NOT in this list. Apple's table maps NUL to -/// 0xFFFF — i.e. NUL is the highest-sorting character, not ignorable. -/// That's how the on-disk `"\0\0\0\0HFS+ Private Data"` directory ends -/// up sorting *last* among its parent's children rather than first. -/// `fold_hfsplus_name_for_compare` rewrites NUL → U+FFFF before the -/// final uppercase pass to match. -fn is_hfsplus_ignored(c: char) -> bool { - matches!( - c as u32, - 0x00AD - | 0x034F - | 0x070F - | 0x180B..=0x180E - | 0x200B..=0x200F - | 0x202A..=0x202E - | 0x206A..=0x206F - | 0xFE00..=0xFE0F - | 0xFEFF - ) -} - -fn fold_hfsplus_name_for_compare(name: &[u16]) -> Vec { - let s = String::from_utf16_lossy(name); - let nfd: String = s.nfd().collect(); - nfd.chars() - .filter(|c| !is_hfsplus_ignored(*c)) - .map(|c| if c == '\0' { '\u{FFFF}' } else { c }) - .flat_map(|c| c.to_uppercase()) - .collect() +/// Normalize an HFS+ catalog name into Apple's `FastUnicodeCompare` form: +/// canonically decompose each UTF-16 unit (Apple's TN1150 decomposition, NOT +/// Rust NFD — the two differ in the ranges Apple excludes), then case-fold each +/// resulting unit through Apple's table, dropping units that fold to 0 +/// (Apple's "ignored" set: zero-width / format / variation selectors). NUL +/// folds to 0xFFFF, so a leading-NUL name like the `HFS+ Private Data` +/// directory sorts *last*. Comparing the folded `u16` sequences lexicographically +/// reproduces a real Mac OS volume's ordering exactly (see [`hfs_unicode`]). +fn fold_hfsplus_name_for_compare(name: &[u16]) -> Vec { + let mut out = Vec::with_capacity(name.len()); + let mut buf = [0u16; 3]; + for &unit in name { + for &d in super::hfs_unicode::decompose(unit, &mut buf) { + let folded = super::hfs_unicode::case_fold(d); + if folded != 0 { + out.push(folded); + } + } + } + out } /// Compare two HFS+ catalog keys (case-insensitive by default). @@ -435,35 +417,51 @@ pub fn compare_hfsplus_keys( folded_a.cmp(&folded_b) } -#[allow(dead_code)] -/// Mac Roman case-insensitive uppercase table (bytes 0x00-0xFF). -/// Characters that have uppercase equivalents are mapped; all others map to themselves. -static MAC_ROMAN_UPPER: [u8; 256] = { - let mut table = [0u8; 256]; - let mut i = 0u16; - while i < 256 { - table[i as usize] = i as u8; - i += 1; - } - // ASCII lowercase → uppercase - let mut c = b'a'; - while c <= b'z' { - table[c as usize] = c - 32; - c += 1; - } - // Mac Roman specific mappings (lowercase → uppercase) - table[0x87] = 0x83; // á → É? No: 0x87=á, uppercase is 0xE7=Á... Let me use the standard HFS approach - // HFS uses a simpler approach: the Finder's case folding table. - // For correctness, we uppercase ASCII and leave high bytes as-is for comparison. - // The HFS spec defines a specific case-folding table, but for practical purposes - // the critical case is ASCII letters. High-byte Mac Roman diacritics are compared as-is. - table -}; - -#[allow(dead_code)] -/// Compare two HFS (classic) catalog keys (case-insensitive Mac Roman). +/// Mac Roman catalog-name collation order (byte -> sort rank), as used by +/// classic Mac OS HFS. +/// +/// Classic HFS compares catalog names with the File Manager's `_RelString` +/// trap (`CMKeyCmp` in Apple's source: `OS/HFS/CMMAINT.a`), i.e. the system +/// string comparison — **case-insensitive, diacritical-sensitive**, with a +/// specific Mac Roman ordering. This is NOT a plain ASCII uppercase: lowercase +/// folds onto its uppercase *rank*, accented letters sort right after their +/// base letter (so "Bézier" < "Bind"), and punctuation / quotes / spaces +/// (incl. high-byte curly quotes 0xD2-0xD5, en/em dashes, and the non-breaking +/// space 0xCA, which sorts as a space) get their own early ranks rather than +/// landing after 'Z' by raw byte value. /// -/// Keys are: parent_cnid (u32) + name (Mac Roman bytes). +/// The table is the well-known `hfs_charorder` reproduction (hfsutils +/// `libhfs/data.c`); the Linux kernel's `caseorder` (`fs/hfs/string.c`) is a +/// second, independent copy — they use different absolute values but induce an +/// identical ordering, and this table reproduces real Apple-formatted volumes +/// fsck-clean (verified against a MacPack boot disk). The previous table only +/// folded ASCII and left high bytes raw, which mis-sorted any name with an +/// accent or curly quote. +static HFS_CHARORDER: [u8; 256] = [ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x20, 0x22, 0x23, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, + 0x37, 0x38, 0x39, 0x3a, 0x3b, 0x3c, 0x3d, 0x3e, 0x3f, 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, + 0x47, 0x48, 0x58, 0x5a, 0x5e, 0x60, 0x67, 0x69, 0x6b, 0x6d, 0x73, 0x75, 0x77, 0x79, 0x7b, 0x7f, + 0x8d, 0x8f, 0x91, 0x93, 0x96, 0x98, 0x9f, 0xa1, 0xa3, 0xa5, 0xa8, 0xaa, 0xab, 0xac, 0xad, 0xae, + 0x54, 0x48, 0x58, 0x5a, 0x5e, 0x60, 0x67, 0x69, 0x6b, 0x6d, 0x73, 0x75, 0x77, 0x79, 0x7b, 0x7f, + 0x8d, 0x8f, 0x91, 0x93, 0x96, 0x98, 0x9f, 0xa1, 0xa3, 0xa5, 0xa8, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, + 0x4c, 0x50, 0x5c, 0x62, 0x7d, 0x81, 0x9a, 0x55, 0x4a, 0x56, 0x4c, 0x4e, 0x50, 0x5c, 0x62, 0x64, + 0x65, 0x66, 0x6f, 0x70, 0x71, 0x72, 0x7d, 0x89, 0x8a, 0x8b, 0x81, 0x83, 0x9c, 0x9d, 0x9e, 0x9a, + 0xb4, 0xb5, 0xb6, 0xb7, 0xb8, 0xb9, 0xba, 0x95, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf, 0xc0, 0x52, 0x85, + 0xc1, 0xc2, 0xc3, 0xc4, 0xc5, 0xc6, 0xc7, 0xc8, 0xc9, 0xca, 0xcb, 0x57, 0x8c, 0xcc, 0x52, 0x85, + 0xcd, 0xce, 0xcf, 0xd0, 0xd1, 0xd2, 0xd3, 0x26, 0x27, 0xd4, 0x20, 0x4a, 0x4e, 0x83, 0x87, 0x87, + 0xd5, 0xd6, 0x24, 0x25, 0x2d, 0x2e, 0xd7, 0xd8, 0xa7, 0xd9, 0xda, 0xdb, 0xdc, 0xdd, 0xde, 0xdf, + 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe6, 0xe7, 0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef, + 0xf0, 0xf1, 0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf7, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd, 0xfe, 0xff, +]; + +/// Compare two HFS (classic) catalog keys the way the Mac OS File Manager does. +/// +/// Keys are: parent_cnid (u32) + name (Mac Roman bytes). Parent IDs compare +/// numerically; names compare through [`HFS_CHARORDER`] byte-by-byte, with the +/// shorter name sorting first on a common prefix (matching hfsutils +/// `d_relstring` / the Linux kernel `hfs_strcmp`). pub fn compare_hfs_keys(parent_a: u32, name_a: &[u8], parent_b: u32, name_b: &[u8]) -> Ordering { match parent_a.cmp(&parent_b) { Ordering::Equal => {} @@ -471,8 +469,8 @@ pub fn compare_hfs_keys(parent_a: u32, name_a: &[u8], parent_b: u32, name_b: &[u } let len = name_a.len().min(name_b.len()); for i in 0..len { - let a = MAC_ROMAN_UPPER[name_a[i] as usize]; - let b = MAC_ROMAN_UPPER[name_b[i] as usize]; + let a = HFS_CHARORDER[name_a[i] as usize]; + let b = HFS_CHARORDER[name_b[i] as usize]; match a.cmp(&b) { Ordering::Equal => {} other => return other, @@ -516,6 +514,158 @@ pub fn normalize_catalog_index_key(key: &[u8]) -> Vec { result } +/// `kBTBigKeysMask` — the B-tree's key-length prefix is a 2-byte (`u16`) field. +/// Set on every HFS+ tree (catalog, extents-overflow, attributes); clear on +/// classic HFS, whose key length is a single byte. +pub const KBT_BIG_KEYS_MASK: u32 = 0x0000_0002; +/// `kBTVariableIndexKeysMask` — index records carry the child's variable-length +/// key verbatim. Set on the HFS+ catalog and attributes trees; clear on the +/// extents-overflow tree and classic HFS, whose index keys are padded to a +/// fixed `max_key_len`. +pub const KBT_VARIABLE_INDEX_KEYS_MASK: u32 = 0x0000_0004; + +/// Describes how a B-tree encodes its keys, so the shared split / grow / rotate +/// machinery below can serve both the classic-HFS 1-byte-key catalog and the +/// HFS+ 2-byte ("big key") trees without forking the algorithm. +/// +/// The classic path threads [`BTreeKeyFormat::CLASSIC_CATALOG`] and is +/// byte-identical to the pre-descriptor code (the fixed 0x25 index key produced +/// by [`normalize_catalog_index_key`]); HFS+ threads the matching `HFSPLUS_*` +/// constant, which is what unlocks variable-length index keys so an HFS+ catalog +/// can split past a single leaf level without corrupting. +/// +/// Derived from the BTHeaderRec `attributes` bitfield + `maxKeyLength`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct BTreeKeyFormat { + /// `kBTBigKeysMask`: key-length prefix is 2 bytes (HFS+) vs 1 byte (classic). + pub big_keys: bool, + /// `kBTVariableIndexKeysMask`: index records store the child's + /// variable-length key verbatim. When clear, index keys are padded to + /// `max_key_len`. + pub variable_index_keys: bool, + /// Maximum key length — the fixed index-key width when `variable_index_keys` + /// is clear (classic catalog: 0x25; HFS+ extents-overflow: 10). + pub max_key_len: u16, +} + +impl BTreeKeyFormat { + /// Classic HFS catalog: 1-byte key length, fixed 0x25-byte index keys. + pub const CLASSIC_CATALOG: BTreeKeyFormat = BTreeKeyFormat { + big_keys: false, + variable_index_keys: false, + max_key_len: HFS_CAT_MAX_KEY_LEN as u16, + }; + + /// HFS+/HFSX catalog: 2-byte key length, variable-length index keys + /// (`kHFSPlusCatalogKeyMaximumLength` = 516). + pub const HFSPLUS_CATALOG: BTreeKeyFormat = BTreeKeyFormat { + big_keys: true, + variable_index_keys: true, + max_key_len: 516, + }; + + /// HFS+ extents-overflow: 2-byte key length, fixed 10-byte index keys + /// (`kHFSPlusExtentKeyMaximumLength` = 10). + pub const HFSPLUS_EXTENTS: BTreeKeyFormat = BTreeKeyFormat { + big_keys: true, + variable_index_keys: false, + max_key_len: 10, + }; + + /// HFS+ attributes: 2-byte key length, variable-length index keys + /// (`kHFSPlusAttrKeyMaximumLength` = 264). + pub const HFSPLUS_ATTRIBUTES: BTreeKeyFormat = BTreeKeyFormat { + big_keys: true, + variable_index_keys: true, + max_key_len: 264, + }; + + /// Build a descriptor from a BTHeaderRec `attributes` bitfield and + /// `maxKeyLength`. The `attributes` field on hand-built volumes can be 0 + /// (no flags) even for HFS+ trees, so callers that know the tree's identity + /// should prefer the `HFSPLUS_*` / `CLASSIC_CATALOG` constants; this helper + /// is for code paths that only have the raw header bytes. + pub fn from_header(attributes: u32, max_key_len: u16) -> Self { + BTreeKeyFormat { + big_keys: attributes & KBT_BIG_KEYS_MASK != 0, + variable_index_keys: attributes & KBT_VARIABLE_INDEX_KEYS_MASK != 0, + max_key_len, + } + } + + /// The BTHeaderRec `attributes` bits implied by this format. + pub fn header_attributes(&self) -> u32 { + let mut a = 0; + if self.big_keys { + a |= KBT_BIG_KEYS_MASK; + } + if self.variable_index_keys { + a |= KBT_VARIABLE_INDEX_KEYS_MASK; + } + a + } + + /// Bytes occupied by the key-length prefix (1 classic, 2 big-key). + #[inline] + pub fn len_prefix(&self) -> usize { + if self.big_keys { + 2 + } else { + 1 + } + } + + /// Read the key length stored at the front of `record`. + #[inline] + pub fn key_len(&self, record: &[u8]) -> usize { + if self.big_keys { + if record.len() < 2 { + 0 + } else { + BigEndian::read_u16(&record[0..2]) as usize + } + } else if record.is_empty() { + 0 + } else { + record[0] as usize + } + } + + /// The key portion of a record — the length prefix plus `key_len` key + /// bytes. This is exactly the slice an index separator must carry, and what + /// the split helpers extract from a leaf/child's first record. + pub fn key_portion(&self, record: &[u8]) -> Vec { + let end = (self.len_prefix() + self.key_len(record)).min(record.len()); + record[..end].to_vec() + } + + /// Build the key bytes of an index record (length prefix + key) from a + /// child node's first-key portion. + /// + /// - Classic HFS: the fixed 0x25 normalized key, identical to + /// [`normalize_catalog_index_key`]. + /// - HFS+ variable-index-key trees (catalog, attributes): the key portion + /// verbatim (2-byte length + key bytes). + /// - HFS+ fixed-index-key trees (extents-overflow): the key bytes padded to + /// `max_key_len`, length field forced to `max_key_len`. + pub fn make_index_key(&self, first_key: &[u8]) -> Vec { + if !self.big_keys { + return normalize_catalog_index_key(first_key); + } + if self.variable_index_keys { + return first_key.to_vec(); + } + let max = self.max_key_len as usize; + let copy = self.key_len(first_key).min(max); + let mut result = vec![0u8; 2 + max]; + BigEndian::write_u16(&mut result[0..2], max as u16); + if first_key.len() >= 2 + copy { + result[2..2 + copy].copy_from_slice(&first_key[2..2 + copy]); + } + result + } +} + /// Read the B-tree header record from catalog_data (node 0, record 0 at offset 14). /// Returns (depth, root_node, leaf_records, first_leaf, last_leaf, node_size, /// max_key_len, total_nodes, free_nodes). @@ -1055,6 +1205,7 @@ pub fn btree_split_leaf_with_insert( node_idx: u32, header: &mut BTreeHeader, new_record: &[u8], + kf: &BTreeKeyFormat, compare_fn: &F, ) -> Result)>, FilesystemError> where @@ -1083,48 +1234,83 @@ where } records.insert(insert_pos, new_record.to_vec()); - // Greedy partition by bytes. Per-node usage: 14 (descriptor) + sum(rec_lens) - // + 2 * count + 2 (free pointer). Single record larger than node payload - // is a structural impossibility (HFS+ catalog key+record max is well under - // node_size); reject up front rather than loop forever. + // Per-node usage: 14 (descriptor) + sum(rec_lens) + 2 * count + 2 (free + // pointer); `payload` accounts for the fixed overhead. A single record + // larger than the payload is a structural impossibility (HFS/HFS+ key+record + // max is well under node_size) — reject up front rather than loop forever. let payload = node_size.saturating_sub(16); - let mut chunks: Vec>> = Vec::new(); - let mut cur: Vec> = Vec::new(); - let mut cur_bytes: usize = 0; - for rec in records.into_iter() { - let cost = rec.len() + 2; - if cost > payload { + for rec in &records { + if rec.len() + 2 > payload { return Err(FilesystemError::InvalidData(format!( "btree_split_leaf_with_insert: record of {} bytes exceeds node payload {}", rec.len(), payload ))); } - if cur_bytes + cost > payload { - chunks.push(std::mem::take(&mut cur)); + } + + // Greedy "pack-left-full" partition by bytes. `cuts` holds the record index + // at which each chunk after the first begins (chunk count = cuts.len() + 1). + let mut cuts: Vec = Vec::new(); + let mut cur_bytes: usize = 0; + for (i, rec) in records.iter().enumerate() { + let cost = rec.len() + 2; + if i > 0 && cur_bytes + cost > payload { + cuts.push(i); cur_bytes = 0; } cur_bytes += cost; - cur.push(rec); - } - if !cur.is_empty() { - chunks.push(cur); - } - // Caller's contract: "I tried inserting and it failed, allocate me a - // new node." `btree_insert_record`'s free-space check is 2 bytes more - // conservative than the actual packing limit (it deducts the new - // offset slot from `free` AND adds it to `total_needed`), so the - // merged record set can occasionally fit in a single node by 1–2 - // bytes. We still must produce ≥ 1 new node here — force a midpoint - // 2-way split if the greedy packer collapsed everything into one - // chunk. Both halves trivially fit since the whole set fits a node. - if chunks.len() < 2 { - let mut all = chunks.pop().unwrap_or_default(); - let mid = all.len() / 2; - let right = all.split_off(mid.max(1)); - chunks.push(all); - chunks.push(right); } + // Caller's contract: "I tried inserting and it failed, allocate me a new + // node." `btree_insert_record`'s free-space check is 2 bytes more + // conservative than the actual packing limit, so the merged set can fit a + // single node by 1–2 bytes. We still must produce ≥ 1 new node — force a + // 2-way split at the midpoint when greedy yielded a single chunk. + if cuts.is_empty() { + cuts.push( + (records.len() / 2) + .max(1) + .min(records.len().saturating_sub(1)), + ); + } + + // Density tuning. Greedy pack-left is ideal for *sequential* (append) + // inserts — the left node stays full — but for a *random* insert into the + // middle of a full leaf it splits off only the overflow (often a single + // record), leaving a trail of ~1-record leaves; a catalog built that way + // runs ~2x larger than Mac OS writes. When the new record landed in the + // middle and the whole set fits two nodes, re-split at the most balanced + // *valid* point instead, so both leaves keep room to fill (random inserts + // then converge to ~69% full, matching a real Mac volume). Sequential growth + // keeps the dense greedy path. + let is_append = insert_pos == old_num; + if cuts.len() == 1 && !is_append { + let total: usize = records.iter().map(|r| r.len() + 2).sum(); + let mut prefix = 0usize; + let mut best_k = cuts[0]; + let mut best_diff = usize::MAX; + for k in 1..records.len() { + prefix += records[k - 1].len() + 2; + let right = total - prefix; + if prefix <= payload && right <= payload { + let diff = prefix.abs_diff(total / 2); + if diff < best_diff { + best_diff = diff; + best_k = k; + } + } + } + cuts[0] = best_k; + } + + // Materialize chunks by draining `records` at the cut points (front to back). + let mut chunks: Vec>> = Vec::with_capacity(cuts.len() + 1); + let mut prev = 0usize; + for &cut in &cuts { + chunks.push(records.drain(0..cut - prev).collect()); + prev = cut; + } + chunks.push(records); // Allocate new nodes (one per extra chunk beyond the first, which reuses old_idx). let new_count = chunks.len() - 1; @@ -1187,18 +1373,364 @@ where header.last_leaf_node = last_idx; } - // Collect separator keys: first record of each new chunk. + // Collect separator keys: first record of each new chunk. The key portion + // (length prefix + key bytes) is format-dependent — 1-byte length for + // classic HFS, 2-byte for HFS+ — so read it through `kf` rather than + // assuming `first[0]` is the whole length. let mut results: Vec<(u32, Vec)> = Vec::with_capacity(new_count); for (i, idx) in new_indices.iter().enumerate() { - let first = &chunks[i + 1][0]; - let key_len = first[0] as usize; - let key_end = 1 + key_len; - let split_key = first[..key_end.min(first.len())].to_vec(); + let split_key = kf.key_portion(&chunks[i + 1][0]); results.push((*idx, split_key)); } Ok(results) } +/// Overwrite the record area of node `idx` with `recs` (in key order), +/// preserving the node descriptor (sibling links, kind, height). Mirrors the +/// rebuild step of [`btree_insert_record`] but takes a ready-made record list — +/// used by the sibling-rotation path, which rewrites both nodes of a pair at +/// once. The caller guarantees `recs` fits within the node payload. +fn btree_write_node_records(data: &mut [u8], node_size: usize, idx: u32, recs: &[Vec]) { + let off = idx as usize * node_size; + let node = &mut data[off..off + node_size]; + let mut write_pos = 14usize; + for (i, rec) in recs.iter().enumerate() { + node[write_pos..write_pos + rec.len()].copy_from_slice(rec); + let opos = node_size - 2 * (i + 1); + BigEndian::write_u16(&mut node[opos..opos + 2], write_pos as u16); + write_pos += rec.len(); + } + let fpos = node_size - 2 * (recs.len() + 1); + BigEndian::write_u16(&mut node[fpos..fpos + 2], write_pos as u16); + if write_pos < fpos { + node[write_pos..fpos].fill(0); + } + BigEndian::write_u16(&mut node[10..12], recs.len() as u16); +} + +/// True when index node `parent_idx` holds a record whose trailing 4-byte +/// child pointer equals `child` — i.e. `child` is a direct child of that index +/// node, so the two share `parent_idx` as a parent. +fn index_node_has_child(data: &[u8], node_size: usize, parent_idx: u32, child: u32) -> bool { + if parent_idx == 0 { + return false; + } + let off = parent_idx as usize * node_size; + if off + node_size > data.len() { + return false; + } + let num = BigEndian::read_u16(&data[off + 10..off + 12]) as usize; + for i in 0..num { + let (s, e) = btree_record_range(&data[off..off + node_size], node_size, i); + if e < s + 4 || e > node_size { + continue; + } + if BigEndian::read_u32(&data[off + e - 4..off + e]) == child { + return true; + } + } + false +} + +/// In index node `parent_idx`, rewrite (in place) the separator key of the +/// record pointing at `child` so it equals `new_first_key` (a record's key +/// portion: the `key_len` byte plus key bytes). Catalog index keys are a fixed +/// 0x25-length normalized key, so the record length never changes and the +/// offset table / sibling records are untouched. +/// +/// Returns `true` only when the separator was found *and* the in-place rewrite +/// was valid (the existing key portion is the same length as the normalized +/// key). Returns `false` — writing nothing — when `child` isn't a direct child +/// of `parent_idx`, or when the existing separator is a different length (a +/// real variable-length HFS+ index key, which can't be patched in place). The +/// caller uses the `false` result to abandon a rotation and split instead, so a +/// non-classic index is never left with a stale separator. +#[must_use] +fn btree_update_index_separator( + data: &mut [u8], + node_size: usize, + parent_idx: u32, + child: u32, + new_first_key: &[u8], + kf: &BTreeKeyFormat, +) -> bool { + let off = parent_idx as usize * node_size; + if off + node_size > data.len() { + return false; + } + let num = BigEndian::read_u16(&data[off + 10..off + 12]) as usize; + for i in 0..num { + let (s, e) = btree_record_range(&data[off..off + node_size], node_size, i); + if e < s + 4 || e > node_size { + continue; + } + if BigEndian::read_u32(&data[off + e - 4..off + e]) == child { + // The replacement separator must occupy the same number of bytes as + // the existing one — only then is an in-place patch valid (the + // offset table and sibling records stay put). Classic HFS index keys + // are a fixed 0x25 length so this always holds; an HFS+ variable key + // matches only when the new first key has the same length as the old + // separator. When it differs we return `false` and the caller + // abandons the rotation and splits instead, so a stale separator is + // never left behind. + let new_key = kf.make_index_key(new_first_key); + let key_bytes = e - s - 4; + if new_key.len() != key_bytes { + return false; + } + data[off + s..off + s + key_bytes].copy_from_slice(&new_key); + return true; + } + } + false +} + +/// Try to absorb `new_record` into a full leaf by redistributing records with +/// an adjacent sibling that shares the same parent index node and still has +/// room — a B*-tree rotation — instead of splitting. +/// +/// A plain 50/50 leaf split leaves non-sequential inserts (the per-`put` +/// workload, where keys land mid-leaf rather than appending) at the classic +/// ~69% B-tree occupancy, so a catalog grown one record at a time exhausts its +/// node budget ~1.4-2.7x sooner than a bulk/sequential build and can hit the +/// file's node ceiling far below the volume's real capacity. Rotating into a +/// neighbour before splitting lifts random-insert occupancy to ~88%, so the +/// incremental path packs nearly as tightly as the sequential one. +/// +/// Returns `true` when the record was absorbed: no node is allocated and the +/// tree shape is unchanged — only the two nodes' records and one parent +/// separator key move. Returns `false` when no sibling can take the overflow +/// and the caller must fall back to [`btree_split_leaf_with_insert`]. +/// +/// Safety of the in-place separator update: we only ever rewrite the separator +/// of the pair's *right* node, and only when the pair's *left* node keeps its +/// original first key (guarded below). The right node is never the parent's +/// first child (a right sibling sorts after `leaf_idx`; a left sibling sharing +/// the parent means `leaf_idx` isn't first), so the parent's minimum key — and +/// the grandparent invariant that mirrors it — is never disturbed. +fn btree_try_rotate_leaf( + data: &mut [u8], + node_size: usize, + leaf_idx: u32, + parent_idx: u32, + new_record: &[u8], + kf: &BTreeKeyFormat, + cmp: &F, +) -> bool +where + F: Fn(&[u8], &[u8]) -> Ordering, +{ + let payload = node_size.saturating_sub(16); + let cost = |r: &Vec| r.len() + 2; + let first_key = |r: &[u8]| -> Vec { kf.key_portion(r) }; + let read_records = |data: &[u8], idx: u32| -> Vec> { + let off = idx as usize * node_size; + let num = BigEndian::read_u16(&data[off + 10..off + 12]) as usize; + let mut v = Vec::with_capacity(num); + for i in 0..num { + let (s, e) = btree_record_range(&data[off..off + node_size], node_size, i); + v.push(data[off + s..off + e].to_vec()); + } + v + }; + + let loff = leaf_idx as usize * node_size; + let rsib = BigEndian::read_u32(&data[loff..loff + 4]); + let lsib = BigEndian::read_u32(&data[loff + 4..loff + 8]); + let leaf_recs = read_records(data, leaf_idx); + + // Prefer the right sibling, then the left. For each candidate, pool the two + // siblings' records plus `new_record` in key order, then redistribute them + // balanced across the pair so both nodes keep headroom. + for (sib_is_right, sib) in [(true, rsib), (false, lsib)] { + if sib == 0 || !index_node_has_child(data, node_size, parent_idx, sib) { + continue; + } + let sib_recs = read_records(data, sib); + let (left_idx, right_idx, left_recs, right_recs) = if sib_is_right { + (leaf_idx, sib, &leaf_recs, &sib_recs) + } else { + (sib, leaf_idx, &sib_recs, &leaf_recs) + }; + let Some(left_first) = left_recs.first() else { + continue; + }; + // Build the merged, key-ordered record set across the pair. + let mut pool: Vec> = left_recs.iter().chain(right_recs.iter()).cloned().collect(); + let mut p = pool.len(); + for (i, r) in pool.iter().enumerate() { + if cmp(r, new_record) == Ordering::Greater { + p = i; + break; + } + } + pool.insert(p, new_record.to_vec()); + // Skip if the left node's minimum would change — its parent separator + // (which we don't touch) would then be stale. Happens only when the new + // record sorts before the whole pair; rare, and a normal split handles + // it correctly. + if cmp(&pool[0], left_first) != Ordering::Equal { + continue; + } + let total: usize = pool.iter().map(cost).sum(); + if total > 2 * payload { + continue; // neighbour is too full to absorb the overflow + } + // Balanced split point: both halves within payload, closest to total/2. + let mut prefix = 0usize; + let mut best_p = 0usize; + let mut best_diff = usize::MAX; + for cut in 1..pool.len() { + prefix += cost(&pool[cut - 1]); + let right = total - prefix; + if prefix <= payload && right <= payload { + let diff = prefix.abs_diff(total / 2); + if diff < best_diff { + best_diff = diff; + best_p = cut; + } + } + } + if best_p == 0 { + continue; + } + let left: Vec> = pool[..best_p].to_vec(); + let right: Vec> = pool[best_p..].to_vec(); + let right_first = first_key(&right[0]); + // Patch the parent separator first: if it can't be updated in place + // (a variable-length, non-normalized index key — i.e. a real HFS+ + // catalog), abandon the rotation untouched and let the caller split, + // rather than leave the moved records behind a stale separator. + if !btree_update_index_separator(data, node_size, parent_idx, right_idx, &right_first, kf) { + continue; + } + btree_write_node_records(data, node_size, left_idx, &left); + btree_write_node_records(data, node_size, right_idx, &right); + return true; + } + false +} + +/// Split a full index node while inserting `new_record` (a normalized index +/// record: a 0x25-length key followed by a 4-byte child pointer), append-aware. +/// +/// The previous index split was a fixed 50/50 of the existing records, then the +/// new record went into one half. For *sequential* separator inserts — which is +/// what every leaf split produces when files arrive in key order (a bulk +/// `untar`, or `put`ting a sorted batch) — that froze each non-rightmost index +/// node at ~50%, doubling the index node count and adding a whole tree level. +/// Packing the left node full on an append (and rebalancing only a genuine +/// middle insert), exactly as the leaf split does, keeps the index dense. +/// +/// Returns `(new_node_idx, separator_key)` to install in the grandparent. Index +/// records are uniform and small, so a 2-way split always suffices (an overflow +/// is at most one record over a node). +fn btree_split_index_with_insert( + catalog_data: &mut [u8], + node_size: usize, + node_idx: u32, + header: &mut BTreeHeader, + new_record: &[u8], + kf: &BTreeKeyFormat, + compare_fn: &F, +) -> Result<(u32, Vec), FilesystemError> +where + F: Fn(&[u8], &[u8]) -> Ordering, +{ + let old_offset = node_idx as usize * node_size; + let old_num = BigEndian::read_u16(&catalog_data[old_offset + 10..old_offset + 12]) as usize; + let old_height = catalog_data[old_offset + 9]; + + // Existing records merged with new_record, in key order. + let mut records: Vec> = Vec::with_capacity(old_num + 1); + for i in 0..old_num { + let (s, e) = btree_record_range( + &catalog_data[old_offset..old_offset + node_size], + node_size, + i, + ); + records.push(catalog_data[old_offset + s..old_offset + e].to_vec()); + } + let mut insert_pos = records.len(); + for (i, rec) in records.iter().enumerate() { + if compare_fn(rec, new_record) == Ordering::Greater { + insert_pos = i; + break; + } + } + records.insert(insert_pos, new_record.to_vec()); + + let payload = node_size.saturating_sub(16); + let cost = |r: &Vec| r.len() + 2; + let total: usize = records.iter().map(cost).sum(); + + // Greedy pack-left (dense for appends). `cut` = count kept in the old node. + let mut cut = { + let mut acc = 0usize; + let mut k = 1usize; + for (i, r) in records.iter().enumerate() { + let c = cost(r); + if i > 0 && acc + c > payload { + break; + } + acc += c; + k = i + 1; + } + k.clamp(1, records.len() - 1) + }; + // A genuine middle insert rebalances so both halves keep room (matches the + // leaf split's density tuning); a tail append keeps the dense pack-left. + let is_append = insert_pos == old_num; + if !is_append { + let mut prefix = 0usize; + let mut best = cut; + let mut best_diff = usize::MAX; + for k in 1..records.len() { + prefix += cost(&records[k - 1]); + let right = total - prefix; + if prefix <= payload && right <= payload { + let diff = prefix.abs_diff(total / 2); + if diff < best_diff { + best_diff = diff; + best = k; + } + } + } + cut = best; + } + + let new_idx = btree_alloc_node(catalog_data, node_size, header.total_nodes)?; + header.free_nodes -= 1; + init_node( + catalog_data, + node_size, + new_idx, + BTREE_INDEX_NODE, + old_height, + ); + + let left: Vec> = records[..cut].to_vec(); + let right: Vec> = records[cut..].to_vec(); + btree_write_node_records(catalog_data, node_size, node_idx, &left); + btree_write_node_records(catalog_data, node_size, new_idx, &right); + + // Splice the index-level sibling chain: node_idx <-> new_idx <-> old_next. + // `btree_write_node_records` preserved node_idx's old forward link. + let old_next = BigEndian::read_u32(&catalog_data[old_offset..old_offset + 4]); + let new_offset = new_idx as usize * node_size; + BigEndian::write_u32(&mut catalog_data[old_offset..old_offset + 4], new_idx); + BigEndian::write_u32(&mut catalog_data[new_offset + 4..new_offset + 8], node_idx); + BigEndian::write_u32(&mut catalog_data[new_offset..new_offset + 4], old_next); + if old_next != 0 { + let next_off = old_next as usize * node_size; + BigEndian::write_u32(&mut catalog_data[next_off + 4..next_off + 8], new_idx); + } + + // Separator = first key portion (length prefix + key bytes) of the new node. + let split_key = kf.key_portion(&right[0]); + Ok((new_idx, split_key)) +} + /// Insert a separator key into a parent index node, pointing to child_node. /// /// For index nodes, each record is: key_bytes + child_node_ptr (4 bytes BE). @@ -1214,17 +1746,20 @@ pub fn btree_insert_into_index( child_node: u32, split_key: &[u8], header: &mut BTreeHeader, + kf: &BTreeKeyFormat, compare_fn: &F, parent_chain: &[(u32, u32)], // [(node_idx, parent_idx), ...] ) -> Result<(), FilesystemError> where F: Fn(&[u8], &[u8]) -> Ordering, { - // Build index record: normalized_key + child_pointer - // Mac OS forces all catalog index keys to length 0x25, zero-padded. - // With key_len=0x25, HFS_RECKEYSKIP = (1+37+1)&~1 = 38, so data starts - // at offset 38 (even), and the full record is always 42 bytes. - let mut index_record = normalize_catalog_index_key(split_key); + // Build the index record: index key + child_pointer. For classic HFS the + // index key is the fixed 0x25-length normalized key (record always 42 + // bytes); for an HFS+ variable-index-key tree it is the child's first key + // verbatim (2-byte length + key bytes). `make_index_key` picks the right + // shape per `kf`. Catalog/attributes keys are even-length, so the trailing + // child pointer always lands on an even offset. + let mut index_record = kf.make_index_key(split_key); let mut ptr = [0u8; 4]; BigEndian::write_u32(&mut ptr, child_node); index_record.extend_from_slice(&ptr); @@ -1236,20 +1771,45 @@ where match btree_insert_record(node, node_size, &index_record, compare_fn) { Ok(_) => Ok(()), Err(_) => { - // Index node is full — split it - let (new_idx, new_split_key) = - split_index_node(catalog_data, node_size, index_node_idx, header)?; - - // Insert the record into whichever half it belongs - // Compare split_key with new_split_key to decide - let target = if compare_fn(split_key, &new_split_key) == Ordering::Less { - index_node_idx - } else { - new_idx - }; - let t_offset = target as usize * node_size; - let t_node = &mut catalog_data[t_offset..t_offset + node_size]; - btree_insert_record(t_node, node_size, &index_record, compare_fn)?; + // Index node is full. Before allocating a node, try a B*-style + // rotation into an index sibling that shares this node's parent and + // has room — the same density win applied at the leaf level, so a + // deep or sequential catalog doesn't accumulate half-full index + // nodes (and the extra level they imply). The root has no parent and + // grows instead; `btree_try_rotate_leaf` is record-agnostic and + // updates the parent separator in place, so it serves index nodes + // just as it does leaves. + if let Some(&(_, parent_idx)) = parent_chain + .iter() + .find(|&&(nidx, _)| nidx == index_node_idx) + { + if parent_idx != 0 + && btree_try_rotate_leaf( + catalog_data, + node_size, + index_node_idx, + parent_idx, + &index_record, + kf, + compare_fn, + ) + { + return Ok(()); + } + } + + // No sibling could help — split (append-aware), placing the new + // record in the appropriate half, then install the new node's + // separator in the parent. + let (new_idx, new_split_key) = btree_split_index_with_insert( + catalog_data, + node_size, + index_node_idx, + header, + &index_record, + kf, + compare_fn, + )?; // Now insert new_split_key into this index node's parent // Find our parent from the chain @@ -1266,6 +1826,7 @@ where index_node_idx, new_idx, &new_split_key, + kf, )?; } else { // Recurse to parent index node @@ -1276,6 +1837,7 @@ where new_idx, &new_split_key, header, + kf, compare_fn, parent_chain, )?; @@ -1289,6 +1851,7 @@ where index_node_idx, new_idx, &new_split_key, + kf, )?; } Ok(()) @@ -1296,89 +1859,6 @@ where } } -/// Split an index node, similar to leaf splitting but for index records. -fn split_index_node( - catalog_data: &mut [u8], - node_size: usize, - node_idx: u32, - header: &mut BTreeHeader, -) -> Result<(u32, Vec), FilesystemError> { - let new_idx = btree_alloc_node(catalog_data, node_size, header.total_nodes)?; - header.free_nodes -= 1; - - let old_offset = node_idx as usize * node_size; - let new_offset = new_idx as usize * node_size; - - let old_num = BigEndian::read_u16(&catalog_data[old_offset + 10..old_offset + 12]) as usize; - let old_height = catalog_data[old_offset + 9]; - let split_point = old_num / 2; - - init_node( - catalog_data, - node_size, - new_idx, - BTREE_INDEX_NODE, - old_height, - ); - - // Collect records - let mut records: Vec> = Vec::with_capacity(old_num); - for i in 0..old_num { - let (start, end) = btree_record_range( - &catalog_data[old_offset..old_offset + node_size], - node_size, - i, - ); - records.push(catalog_data[old_offset + start..old_offset + end].to_vec()); - } - - // Rebuild old node with first half - { - let node = &mut catalog_data[old_offset..old_offset + node_size]; - BigEndian::write_u16(&mut node[10..12], 0); - BigEndian::write_u16(&mut node[node_size - 2..node_size], 14); - let mut write_pos = 14usize; - for (i, rec) in records[..split_point].iter().enumerate() { - node[write_pos..write_pos + rec.len()].copy_from_slice(rec); - let opos = node_size - 2 * (i + 1); - BigEndian::write_u16(&mut node[opos..opos + 2], write_pos as u16); - write_pos += rec.len(); - } - let fpos = node_size - 2 * (split_point + 1); - BigEndian::write_u16(&mut node[fpos..fpos + 2], write_pos as u16); - BigEndian::write_u16(&mut node[10..12], split_point as u16); - } - - // Build new node with second half - { - let node = &mut catalog_data[new_offset..new_offset + node_size]; - let count = old_num - split_point; - let mut write_pos = 14usize; - for (i, rec) in records[split_point..].iter().enumerate() { - node[write_pos..write_pos + rec.len()].copy_from_slice(rec); - let opos = node_size - 2 * (i + 1); - BigEndian::write_u16(&mut node[opos..opos + 2], write_pos as u16); - write_pos += rec.len(); - } - let fpos = node_size - 2 * (count + 1); - BigEndian::write_u16(&mut node[fpos..fpos + 2], write_pos as u16); - BigEndian::write_u16(&mut node[10..12], count as u16); - } - - // The separator key is just the key portion (key_len byte + key data) of the - // first record of the new node, without pad or child pointer. - let full_rec = &records[split_point]; - let split_key = if !full_rec.is_empty() { - let kl = full_rec[0] as usize; - let key_end = (1 + kl).min(full_rec.len()); - full_rec[..key_end].to_vec() - } else { - full_rec.clone() - }; - - Ok((new_idx, split_key)) -} - /// Grow the B-tree root: create a new root node at height+1 with two children. pub fn btree_grow_root( catalog_data: &mut [u8], @@ -1387,6 +1867,7 @@ pub fn btree_grow_root( old_root: u32, new_sibling: u32, split_key: &[u8], + kf: &BTreeKeyFormat, ) -> Result<(), FilesystemError> { let new_root = btree_alloc_node(catalog_data, node_size, header.total_nodes)?; header.free_nodes -= 1; @@ -1408,26 +1889,25 @@ pub fn btree_grow_root( // 1. First child (old_root): use the first key of old_root + pointer to old_root // 2. Second child (new_sibling): split_key + pointer to new_sibling - // Get first key of old_root (just the key portion, not the full record) - let (first_rec_start, _first_rec_end) = btree_record_range( + // Get the first key portion of old_root (length prefix + key bytes, read + // through `kf` so the 1-byte vs 2-byte length is handled correctly). + let (first_rec_start, first_rec_end) = btree_record_range( &catalog_data[old_root_offset..old_root_offset + node_size], node_size, 0, ); - let key_len = catalog_data[old_root_offset + first_rec_start] as usize; - let key_end = first_rec_start + 1 + key_len; - let first_key = - catalog_data[old_root_offset + first_rec_start..old_root_offset + key_end].to_vec(); - - // Record 1: normalized first_key + old_root pointer - // Mac OS forces all catalog index keys to 0x25 length. - let mut rec1 = normalize_catalog_index_key(&first_key); + let first_key = kf.key_portion( + &catalog_data[old_root_offset + first_rec_start..old_root_offset + first_rec_end], + ); + + // Record 1: index key for first_key + old_root pointer. + let mut rec1 = kf.make_index_key(&first_key); let mut ptr1 = [0u8; 4]; BigEndian::write_u32(&mut ptr1, old_root); rec1.extend_from_slice(&ptr1); - // Record 2: normalized split_key + new_sibling pointer - let mut rec2 = normalize_catalog_index_key(split_key); + // Record 2: index key for split_key + new_sibling pointer. + let mut rec2 = kf.make_index_key(split_key); let mut ptr2 = [0u8; 4]; BigEndian::write_u32(&mut ptr2, new_sibling); rec2.extend_from_slice(&ptr2); @@ -1642,6 +2122,7 @@ where pub fn btree_insert_full( data: &mut [u8], key_record: &[u8], + kf: &BTreeKeyFormat, cmp: &F, ) -> Result<(), super::filesystem::FilesystemError> where @@ -1667,8 +2148,29 @@ where } Err(_) => { let mut h = BTreeHeader::read(data); - let splits = - btree_split_leaf_with_insert(data, node_size, leaf_idx, &mut h, key_record, cmp)?; + // Before allocating a node, try a B*-style rotation into an adjacent + // sibling that shares this leaf's parent and still has room. This + // keeps the catalog densely packed for non-sequential (per-`put`) + // inserts; only when no sibling can help do we split. Needs a parent + // index node, so skip it while the root is still the lone leaf. + if h.depth > 1 { + if let Some(&(_, parent_idx)) = + parent_chain.iter().find(|&&(nidx, _)| nidx == leaf_idx) + { + if parent_idx != 0 + && btree_try_rotate_leaf( + data, node_size, leaf_idx, parent_idx, key_record, kf, cmp, + ) + { + h.leaf_records += 1; + h.write(data); + return Ok(()); + } + } + } + let splits = btree_split_leaf_with_insert( + data, node_size, leaf_idx, &mut h, key_record, kf, cmp, + )?; h.leaf_records += 1; // First new node's separator goes via either grow_root (depth==1) // or insert_into_index. Any additional separators (N-way splits @@ -1676,7 +2178,7 @@ where // root depth is already > 1 once the first separator installed. let first = &splits[0]; if h.depth == 1 { - btree_grow_root(data, node_size, &mut h, leaf_idx, first.0, &first.1)?; + btree_grow_root(data, node_size, &mut h, leaf_idx, first.0, &first.1, kf)?; } else if let Some(&(_, parent_idx)) = parent_chain.iter().find(|&&(nidx, _)| nidx == leaf_idx) { @@ -1687,11 +2189,12 @@ where first.0, &first.1, &mut h, + kf, cmp, &parent_chain, )?; } else { - btree_grow_root(data, node_size, &mut h, leaf_idx, first.0, &first.1)?; + btree_grow_root(data, node_size, &mut h, leaf_idx, first.0, &first.1, kf)?; } for split in &splits[1..] { // After the first separator installed, the leaf's parent is @@ -1715,6 +2218,7 @@ where split.0, &split.1, &mut h, + kf, cmp, &fresh_chain, )?; @@ -1921,6 +2425,57 @@ mod tests { assert_eq!(compare_hfsplus_keys(2, &a, 2, &b, false), Ordering::Equal); } + /// HFS+ case-folds to LOWERCASE (Apple `FastUnicodeCompare` / TN1150), so a + /// character between the uppercase and lowercase ASCII blocks — notably the + /// underscore `_` (0x5F) — sorts BEFORE letters, not after the uppercased + /// ones. These exact pairs come from a real Mac OS 9.2.2 HFS+ volume that + /// the previous uppercase fold flagged as `KeyOutOfOrder`. + #[test] + fn test_compare_hfsplus_keys_underscore_collation() { + let k = |s: &str| -> Vec { s.encode_utf16().collect() }; + for (a, b) in [ + ("as_OpenURL", "asVers.htm"), + ("wr_single_pixel.gif", "wrApp.gif"), + ("high_priority.gif", "highest_priority.gif"), + ("not_valid.gif", "note.gif"), + ] { + assert_eq!( + compare_hfsplus_keys(2, &k(a), 2, &k(b), false), + Ordering::Less, + "{a:?} must sort before {b:?} (lowercase fold)" + ); + } + // Still case-insensitive. + assert_eq!( + compare_hfsplus_keys(2, &k("README"), 2, &k("readme"), false), + Ordering::Equal + ); + } + + /// Apple's TN1150 tables differ from Rust's `to_lowercase` + NFD on exotic + /// code points; these cases pin the table-faithful behavior. + #[test] + fn test_hfsplus_unicode_tables_match_apple() { + use super::super::hfs_unicode::decompose_str; + let k = |s: &str| -> Vec { s.encode_utf16().collect() }; + + // ß (0xDF) folds 1:1 — it is NOT expanded to "ss" (as `to_uppercase` + // would), so "straße" and "strasse" are different names. + assert_ne!( + compare_hfsplus_keys(2, &k("stra\u{df}e"), 2, &k("strasse"), false), + Ordering::Equal + ); + + // Apple does NOT decompose the OHM sign U+2126 (TN1150 excludes + // U+2000..U+2FFF), unlike Unicode NFD which maps it to Greek capital + // Omega U+03A9. + assert_eq!(decompose_str("\u{2126}"), vec![0x2126]); + // It DOES canonically decompose precomposed Latin and Hangul. + assert_eq!(decompose_str("\u{e9}"), vec![0x0065, 0x0301]); // é + assert_eq!(decompose_str("\u{ac00}"), vec![0x1100, 0x1161]); // 가 + assert_eq!(decompose_str("\u{d4db}"), vec![0x1111, 0x1171, 0x11b6]); // 3-jamo Hangul + } + #[test] fn test_compare_hfsplus_keys_case_sensitive() { let a: Vec = "Hello".encode_utf16().collect(); @@ -1931,9 +2486,11 @@ mod tests { #[test] fn test_compare_hfsplus_keys_nfd() { - // é (U+00E9, precomposed) vs e + combining acute (U+0065 U+0301) + // é (U+00E9, precomposed) vs e + combining acute (U+0065 U+0301). + // Apple's decomposition folds the precomposed form to the decomposed + // one, so the two keys must compare equal. let a: Vec = "é".encode_utf16().collect(); - let b: Vec = "é".nfd().collect::().encode_utf16().collect(); + let b: Vec = vec![0x0065, 0x0301]; assert_eq!(compare_hfsplus_keys(2, &a, 2, &b, false), Ordering::Equal); } @@ -1995,6 +2552,34 @@ mod tests { assert_eq!(compare_hfs_keys(2, b"abc", 2, b"ABD"), Ordering::Less); } + /// Mac Roman collation must match classic Mac OS (`_RelString`), not raw + /// byte order — the cases below were lifted straight from a real + /// Apple-formatted MacPack disk that rusty-backup's old (ASCII-only) table + /// flagged as `KeysOutOfOrder`. é/â/ü etc. sort next to their base letter, + /// curly quotes / dashes / the non-breaking space sort as punctuation, and + /// the comparison stays case-insensitive but diacritical-sensitive. + #[test] + fn test_compare_hfs_keys_mac_roman_collation() { + // Accented letter sorts right after its base letter (é after e), + // so "Bézier" < "Bind" (é < i), not after 'Z' as a raw 0x8e would. + let bezier = b"B\x8ezier Text"; // 0x8e = é + assert_eq!(compare_hfs_keys(2, bezier, 2, b"Bind"), Ordering::Less); + // Leading curly double-quote (0xD2) sorts as punctuation, before 'E'. + let curly = b"\xd2Extension List\xd3 Docs"; + assert_eq!( + compare_hfs_keys(2, curly, 2, b"Extension List 1.0.2"), + Ordering::Less + ); + // Non-breaking space (0xCA) collates like a normal space, before 'b'. + let nbsp = b"\xcaRead Me\xca"; + assert_eq!(compare_hfs_keys(2, nbsp, 2, b"bas.rl"), Ordering::Less); + // Diacritical-sensitive: base < accented < next base ('A' < 'Ä' < 'B'). + assert_eq!(compare_hfs_keys(2, b"A", 2, b"\x80"), Ordering::Less); // 0x80 = Ä + assert_eq!(compare_hfs_keys(2, b"\x80", 2, b"B"), Ordering::Less); + // Case-insensitive across the high range too: Ä (0x80) == ä (0x8a). + assert_eq!(compare_hfs_keys(2, b"\x80", 2, b"\x8a"), Ordering::Equal); + } + #[test] fn test_compare_hfs_keys_parent_ordering() { assert_eq!(compare_hfs_keys(1, b"test", 2, b"test"), Ordering::Less); @@ -2144,9 +2729,16 @@ mod tests { let mut new_rec = vec![0u8; 45]; new_rec[0] = 5; let mut header = BTreeHeader::read(&data); - let splits = - btree_split_leaf_with_insert(&mut data, node_size, 1, &mut header, &new_rec, &compare) - .unwrap(); + let splits = btree_split_leaf_with_insert( + &mut data, + node_size, + 1, + &mut header, + &new_rec, + &BTreeKeyFormat::CLASSIC_CATALOG, + &compare, + ) + .unwrap(); assert_eq!(splits.len(), 1, "30-byte records: 2-way split is enough"); let new_idx = splits[0].0; assert_eq!(new_idx, 2); diff --git a/src/fs/hfs_fsck/btree.rs b/src/fs/hfs_fsck/btree.rs index 97ce55ba..67a43817 100644 --- a/src/fs/hfs_fsck/btree.rs +++ b/src/fs/hfs_fsck/btree.rs @@ -1151,13 +1151,20 @@ pub(crate) fn rebuild_index_nodes( use super::super::hfs_common::{btree_alloc_node, btree_free_node, init_node}; let header = BTreeHeader::read(catalog_data); - let (_bmp_off, bmp_size) = btree_node_bitmap_range(catalog_data, node_size); let max_nodes = catalog_data.len() / node_size; - // Free all existing index nodes + // Free all existing index nodes across the whole node range — not just the + // header node's bitmap window. Once a tree grows past the header bitmap's + // capacity ((node_size - 256) * 8 = 2048 nodes at node_size=512), its index + // nodes live in map-node-covered bitmap segments too. Capping this loop at + // the header bitmap (the old behaviour) left those index nodes allocated + // forever: every rebuild leaked them, draining free_nodes until a rebuild + // ran out of nodes mid-flight and left the index with broken sibling links + // ("no free B-tree nodes" / IndexSiblingLinkBroken past ~7.4k records). + // `btree_bitmap_test` / `btree_free_node` are segment-aware and walk the + // map-node chain transparently; bits with no backing segment read as free. let mut freed = 0u32; - let bmp_max_bit = (bmp_size as u32) * 8; - for node_idx in 1..(max_nodes as u32).min(bmp_max_bit) { + for node_idx in 1..max_nodes as u32 { if !btree_bitmap_test(catalog_data, node_size, node_idx) { continue; } diff --git a/src/fs/hfs_fsck/catalog.rs b/src/fs/hfs_fsck/catalog.rs index beaf854c..a50e9fce 100644 --- a/src/fs/hfs_fsck/catalog.rs +++ b/src/fs/hfs_fsck/catalog.rs @@ -16,7 +16,7 @@ use super::super::hfs::{ mac_roman_to_utf8, HfsExtDescriptor, HfsMasterDirectoryBlock, CATALOG_DIR, CATALOG_FILE, }; use super::super::hfs_common::{btree_record_range, BTreeHeader}; -use super::mdb::validate_hfs_name; +use super::mdb::{unusual_hfs_name, validate_hfs_name}; use super::{hfs_issue, HfsFsckCode, CATALOG_DIR_THREAD, CATALOG_FILE_THREAD}; /// Parsed catalog key for consistency checking. @@ -205,6 +205,11 @@ pub(super) fn check_catalog_consistency( HfsFsckCode::InvalidCatalogName, format!("directory CNID {}: {}", dir_id, problem), )); + } else if let Some(note) = unusual_hfs_name(&key.name) { + warnings.push(hfs_issue( + HfsFsckCode::UnusualCatalogName, + format!("directory CNID {}: {}", dir_id, note), + )); } } *directories_checked += 1; @@ -240,6 +245,11 @@ pub(super) fn check_catalog_consistency( HfsFsckCode::InvalidCatalogName, format!("file CNID {}: {}", file_id, problem), )); + } else if let Some(note) = unusual_hfs_name(&key.name) { + warnings.push(hfs_issue( + HfsFsckCode::UnusualCatalogName, + format!("file CNID {}: {}", file_id, note), + )); } // Gap 1: LEOF > PEOF — logical size must not exceed physical size if *data_size > *data_physical_size { diff --git a/src/fs/hfs_fsck/mdb.rs b/src/fs/hfs_fsck/mdb.rs index ee17529c..18c33f71 100644 --- a/src/fs/hfs_fsck/mdb.rs +++ b/src/fs/hfs_fsck/mdb.rs @@ -12,8 +12,14 @@ use super::super::fsck::FsckIssue; use super::super::hfs::HfsMasterDirectoryBlock; use super::{hfs_issue, HfsFsckCode}; -/// Validate an HFS catalog name (Mac Roman bytes). -/// Returns `Some(problem)` if invalid, `None` if ok. +/// Validate an HFS catalog name (Mac Roman bytes) for *structural* validity. +/// +/// Returns `Some(problem)` only for genuinely invalid names: empty, longer than +/// the 31-byte HFS limit, or containing a colon (0x3A, the path separator that +/// classic Mac OS forbids). Null and other control bytes are deliberately NOT +/// rejected — classic HFS permits any byte except the colon, and real disks +/// carry such names (a leading null forces a file to the top of the sort, an +/// old Finder trick). Those surface separately via [`unusual_hfs_name`]. pub(super) fn validate_hfs_name(name: &[u8]) -> Option { if name.is_empty() { return Some("name is empty".into()); @@ -24,15 +30,23 @@ pub(super) fn validate_hfs_name(name: &[u8]) -> Option { name.len() )); } - if name.contains(&0x00) { - return Some("name contains null byte".into()); - } if name.contains(&0x3A) { return Some("name contains colon (HFS path separator)".into()); } None } +/// A structurally-valid catalog name that is unusual enough to surface as a +/// *warning* (not an error). Returns `Some(note)` when the name embeds a null +/// byte — valid on classic HFS, but worth flagging since most tooling assumes +/// C-string names. +pub(super) fn unusual_hfs_name(name: &[u8]) -> Option { + if name.contains(&0x00) { + return Some("name contains a null byte (valid on classic HFS, but unusual)".into()); + } + None +} + /// Validate the alternate MDB against the primary MDB. pub(super) fn check_alternate_mdb( mdb: &HfsMasterDirectoryBlock, diff --git a/src/fs/hfs_fsck/mod.rs b/src/fs/hfs_fsck/mod.rs index 3fe98b41..46569c46 100644 --- a/src/fs/hfs_fsck/mod.rs +++ b/src/fs/hfs_fsck/mod.rs @@ -80,6 +80,11 @@ pub(super) enum HfsFsckCode { InvalidCnidRange, ReservedFieldNonZero, InvalidCatalogName, + /// Warning: a structurally valid name that is nonetheless unusual (e.g. + /// embeds null/control bytes). Classic Mac OS permits any byte except the + /// colon, and real disks carry such names (leading nulls to force sort + /// order), so this is informational, not an error. + UnusualCatalogName, ThreadNameNotEmpty, // Extents overflow B-tree structure ExtentsBtreeStructure, @@ -108,6 +113,7 @@ fn is_repairable(code: HfsFsckCode) -> bool { | HfsFsckCode::LeoFExceedsPeoF | HfsFsckCode::InvalidCnidRange | HfsFsckCode::InvalidCatalogName + | HfsFsckCode::UnusualCatalogName | HfsFsckCode::OffsetTableNotMonotonic | HfsFsckCode::OffsetTableOutOfBounds | HfsFsckCode::MissingParent diff --git a/src/fs/hfs_unicode.rs b/src/fs/hfs_unicode.rs new file mode 100644 index 00000000..2361b756 --- /dev/null +++ b/src/fs/hfs_unicode.rs @@ -0,0 +1,700 @@ +//! Apple HFS+ Unicode tables (TN1150): case-folding + canonical decomposition. +//! +//! Ported verbatim from the Linux kernel `fs/hfsplus/{tables.c,unicode.c}`, +//! which reproduces Apple's `FastUnicodeCompare`. These replace the previous +//! `char::to_lowercase` + Rust-`nfd()` approximation so HFS+ name comparison and +//! the canonical decomposition rusty-backup stores match real Mac OS exactly +//! (including the underscore-vs-letter ordering, 1:1 `ß` folding, NUL -> 0xFFFF, +//! the "ignored" code-point set, Hangul, and the decomposition-excluded ranges). +//! +//! `CASE_FOLD` is a 2-level table (256 high-byte offsets + subtables); a fold of +//! 0 marks an ignorable code point. `DECOMPOSE` is a 4-level (4-bit) trie whose +//! leaf packs `offset<<2 | size`; the decomposed sequence lives at `offset`. + +#[rustfmt::skip] +static CASE_FOLD: [u16; 2816] = [ + 0x0100, 0x0200, 0x0000, 0x0300, 0x0400, 0x0500, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0600, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0700, 0x0800, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0900, 0x0a00, 0xffff, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, + 0x0008, 0x0009, 0x000a, 0x000b, 0x000c, 0x000d, 0x000e, 0x000f, 0x0010, 0x0011, 0x0012, 0x0013, + 0x0014, 0x0015, 0x0016, 0x0017, 0x0018, 0x0019, 0x001a, 0x001b, 0x001c, 0x001d, 0x001e, 0x001f, + 0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, 0x0028, 0x0029, 0x002a, 0x002b, + 0x002c, 0x002d, 0x002e, 0x002f, 0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, + 0x0038, 0x0039, 0x003a, 0x003b, 0x003c, 0x003d, 0x003e, 0x003f, 0x0040, 0x0061, 0x0062, 0x0063, + 0x0064, 0x0065, 0x0066, 0x0067, 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, + 0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x005b, + 0x005c, 0x005d, 0x005e, 0x005f, 0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, + 0x0068, 0x0069, 0x006a, 0x006b, 0x006c, 0x006d, 0x006e, 0x006f, 0x0070, 0x0071, 0x0072, 0x0073, + 0x0074, 0x0075, 0x0076, 0x0077, 0x0078, 0x0079, 0x007a, 0x007b, 0x007c, 0x007d, 0x007e, 0x007f, + 0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087, 0x0088, 0x0089, 0x008a, 0x008b, + 0x008c, 0x008d, 0x008e, 0x008f, 0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097, + 0x0098, 0x0099, 0x009a, 0x009b, 0x009c, 0x009d, 0x009e, 0x009f, 0x00a0, 0x00a1, 0x00a2, 0x00a3, + 0x00a4, 0x00a5, 0x00a6, 0x00a7, 0x00a8, 0x00a9, 0x00aa, 0x00ab, 0x00ac, 0x00ad, 0x00ae, 0x00af, + 0x00b0, 0x00b1, 0x00b2, 0x00b3, 0x00b4, 0x00b5, 0x00b6, 0x00b7, 0x00b8, 0x00b9, 0x00ba, 0x00bb, + 0x00bc, 0x00bd, 0x00be, 0x00bf, 0x00c0, 0x00c1, 0x00c2, 0x00c3, 0x00c4, 0x00c5, 0x00e6, 0x00c7, + 0x00c8, 0x00c9, 0x00ca, 0x00cb, 0x00cc, 0x00cd, 0x00ce, 0x00cf, 0x00f0, 0x00d1, 0x00d2, 0x00d3, + 0x00d4, 0x00d5, 0x00d6, 0x00d7, 0x00f8, 0x00d9, 0x00da, 0x00db, 0x00dc, 0x00dd, 0x00fe, 0x00df, + 0x00e0, 0x00e1, 0x00e2, 0x00e3, 0x00e4, 0x00e5, 0x00e6, 0x00e7, 0x00e8, 0x00e9, 0x00ea, 0x00eb, + 0x00ec, 0x00ed, 0x00ee, 0x00ef, 0x00f0, 0x00f1, 0x00f2, 0x00f3, 0x00f4, 0x00f5, 0x00f6, 0x00f7, + 0x00f8, 0x00f9, 0x00fa, 0x00fb, 0x00fc, 0x00fd, 0x00fe, 0x00ff, 0x0100, 0x0101, 0x0102, 0x0103, + 0x0104, 0x0105, 0x0106, 0x0107, 0x0108, 0x0109, 0x010a, 0x010b, 0x010c, 0x010d, 0x010e, 0x010f, + 0x0111, 0x0111, 0x0112, 0x0113, 0x0114, 0x0115, 0x0116, 0x0117, 0x0118, 0x0119, 0x011a, 0x011b, + 0x011c, 0x011d, 0x011e, 0x011f, 0x0120, 0x0121, 0x0122, 0x0123, 0x0124, 0x0125, 0x0127, 0x0127, + 0x0128, 0x0129, 0x012a, 0x012b, 0x012c, 0x012d, 0x012e, 0x012f, 0x0130, 0x0131, 0x0133, 0x0133, + 0x0134, 0x0135, 0x0136, 0x0137, 0x0138, 0x0139, 0x013a, 0x013b, 0x013c, 0x013d, 0x013e, 0x0140, + 0x0140, 0x0142, 0x0142, 0x0143, 0x0144, 0x0145, 0x0146, 0x0147, 0x0148, 0x0149, 0x014b, 0x014b, + 0x014c, 0x014d, 0x014e, 0x014f, 0x0150, 0x0151, 0x0153, 0x0153, 0x0154, 0x0155, 0x0156, 0x0157, + 0x0158, 0x0159, 0x015a, 0x015b, 0x015c, 0x015d, 0x015e, 0x015f, 0x0160, 0x0161, 0x0162, 0x0163, + 0x0164, 0x0165, 0x0167, 0x0167, 0x0168, 0x0169, 0x016a, 0x016b, 0x016c, 0x016d, 0x016e, 0x016f, + 0x0170, 0x0171, 0x0172, 0x0173, 0x0174, 0x0175, 0x0176, 0x0177, 0x0178, 0x0179, 0x017a, 0x017b, + 0x017c, 0x017d, 0x017e, 0x017f, 0x0180, 0x0253, 0x0183, 0x0183, 0x0185, 0x0185, 0x0254, 0x0188, + 0x0188, 0x0256, 0x0257, 0x018c, 0x018c, 0x018d, 0x01dd, 0x0259, 0x025b, 0x0192, 0x0192, 0x0260, + 0x0263, 0x0195, 0x0269, 0x0268, 0x0199, 0x0199, 0x019a, 0x019b, 0x026f, 0x0272, 0x019e, 0x0275, + 0x01a0, 0x01a1, 0x01a3, 0x01a3, 0x01a5, 0x01a5, 0x01a6, 0x01a8, 0x01a8, 0x0283, 0x01aa, 0x01ab, + 0x01ad, 0x01ad, 0x0288, 0x01af, 0x01b0, 0x028a, 0x028b, 0x01b4, 0x01b4, 0x01b6, 0x01b6, 0x0292, + 0x01b9, 0x01b9, 0x01ba, 0x01bb, 0x01bd, 0x01bd, 0x01be, 0x01bf, 0x01c0, 0x01c1, 0x01c2, 0x01c3, + 0x01c6, 0x01c6, 0x01c6, 0x01c9, 0x01c9, 0x01c9, 0x01cc, 0x01cc, 0x01cc, 0x01cd, 0x01ce, 0x01cf, + 0x01d0, 0x01d1, 0x01d2, 0x01d3, 0x01d4, 0x01d5, 0x01d6, 0x01d7, 0x01d8, 0x01d9, 0x01da, 0x01db, + 0x01dc, 0x01dd, 0x01de, 0x01df, 0x01e0, 0x01e1, 0x01e2, 0x01e3, 0x01e5, 0x01e5, 0x01e6, 0x01e7, + 0x01e8, 0x01e9, 0x01ea, 0x01eb, 0x01ec, 0x01ed, 0x01ee, 0x01ef, 0x01f0, 0x01f3, 0x01f3, 0x01f3, + 0x01f4, 0x01f5, 0x01f6, 0x01f7, 0x01f8, 0x01f9, 0x01fa, 0x01fb, 0x01fc, 0x01fd, 0x01fe, 0x01ff, + 0x0300, 0x0301, 0x0302, 0x0303, 0x0304, 0x0305, 0x0306, 0x0307, 0x0308, 0x0309, 0x030a, 0x030b, + 0x030c, 0x030d, 0x030e, 0x030f, 0x0310, 0x0311, 0x0312, 0x0313, 0x0314, 0x0315, 0x0316, 0x0317, + 0x0318, 0x0319, 0x031a, 0x031b, 0x031c, 0x031d, 0x031e, 0x031f, 0x0320, 0x0321, 0x0322, 0x0323, + 0x0324, 0x0325, 0x0326, 0x0327, 0x0328, 0x0329, 0x032a, 0x032b, 0x032c, 0x032d, 0x032e, 0x032f, + 0x0330, 0x0331, 0x0332, 0x0333, 0x0334, 0x0335, 0x0336, 0x0337, 0x0338, 0x0339, 0x033a, 0x033b, + 0x033c, 0x033d, 0x033e, 0x033f, 0x0340, 0x0341, 0x0342, 0x0343, 0x0344, 0x0345, 0x0346, 0x0347, + 0x0348, 0x0349, 0x034a, 0x034b, 0x034c, 0x034d, 0x034e, 0x034f, 0x0350, 0x0351, 0x0352, 0x0353, + 0x0354, 0x0355, 0x0356, 0x0357, 0x0358, 0x0359, 0x035a, 0x035b, 0x035c, 0x035d, 0x035e, 0x035f, + 0x0360, 0x0361, 0x0362, 0x0363, 0x0364, 0x0365, 0x0366, 0x0367, 0x0368, 0x0369, 0x036a, 0x036b, + 0x036c, 0x036d, 0x036e, 0x036f, 0x0370, 0x0371, 0x0372, 0x0373, 0x0374, 0x0375, 0x0376, 0x0377, + 0x0378, 0x0379, 0x037a, 0x037b, 0x037c, 0x037d, 0x037e, 0x037f, 0x0380, 0x0381, 0x0382, 0x0383, + 0x0384, 0x0385, 0x0386, 0x0387, 0x0388, 0x0389, 0x038a, 0x038b, 0x038c, 0x038d, 0x038e, 0x038f, + 0x0390, 0x03b1, 0x03b2, 0x03b3, 0x03b4, 0x03b5, 0x03b6, 0x03b7, 0x03b8, 0x03b9, 0x03ba, 0x03bb, + 0x03bc, 0x03bd, 0x03be, 0x03bf, 0x03c0, 0x03c1, 0x03a2, 0x03c3, 0x03c4, 0x03c5, 0x03c6, 0x03c7, + 0x03c8, 0x03c9, 0x03aa, 0x03ab, 0x03ac, 0x03ad, 0x03ae, 0x03af, 0x03b0, 0x03b1, 0x03b2, 0x03b3, + 0x03b4, 0x03b5, 0x03b6, 0x03b7, 0x03b8, 0x03b9, 0x03ba, 0x03bb, 0x03bc, 0x03bd, 0x03be, 0x03bf, + 0x03c0, 0x03c1, 0x03c2, 0x03c3, 0x03c4, 0x03c5, 0x03c6, 0x03c7, 0x03c8, 0x03c9, 0x03ca, 0x03cb, + 0x03cc, 0x03cd, 0x03ce, 0x03cf, 0x03d0, 0x03d1, 0x03d2, 0x03d3, 0x03d4, 0x03d5, 0x03d6, 0x03d7, + 0x03d8, 0x03d9, 0x03da, 0x03db, 0x03dc, 0x03dd, 0x03de, 0x03df, 0x03e0, 0x03e1, 0x03e3, 0x03e3, + 0x03e5, 0x03e5, 0x03e7, 0x03e7, 0x03e9, 0x03e9, 0x03eb, 0x03eb, 0x03ed, 0x03ed, 0x03ef, 0x03ef, + 0x03f0, 0x03f1, 0x03f2, 0x03f3, 0x03f4, 0x03f5, 0x03f6, 0x03f7, 0x03f8, 0x03f9, 0x03fa, 0x03fb, + 0x03fc, 0x03fd, 0x03fe, 0x03ff, 0x0400, 0x0401, 0x0452, 0x0403, 0x0454, 0x0455, 0x0456, 0x0407, + 0x0458, 0x0459, 0x045a, 0x045b, 0x040c, 0x040d, 0x040e, 0x045f, 0x0430, 0x0431, 0x0432, 0x0433, + 0x0434, 0x0435, 0x0436, 0x0437, 0x0438, 0x0419, 0x043a, 0x043b, 0x043c, 0x043d, 0x043e, 0x043f, + 0x0440, 0x0441, 0x0442, 0x0443, 0x0444, 0x0445, 0x0446, 0x0447, 0x0448, 0x0449, 0x044a, 0x044b, + 0x044c, 0x044d, 0x044e, 0x044f, 0x0430, 0x0431, 0x0432, 0x0433, 0x0434, 0x0435, 0x0436, 0x0437, + 0x0438, 0x0439, 0x043a, 0x043b, 0x043c, 0x043d, 0x043e, 0x043f, 0x0440, 0x0441, 0x0442, 0x0443, + 0x0444, 0x0445, 0x0446, 0x0447, 0x0448, 0x0449, 0x044a, 0x044b, 0x044c, 0x044d, 0x044e, 0x044f, + 0x0450, 0x0451, 0x0452, 0x0453, 0x0454, 0x0455, 0x0456, 0x0457, 0x0458, 0x0459, 0x045a, 0x045b, + 0x045c, 0x045d, 0x045e, 0x045f, 0x0461, 0x0461, 0x0463, 0x0463, 0x0465, 0x0465, 0x0467, 0x0467, + 0x0469, 0x0469, 0x046b, 0x046b, 0x046d, 0x046d, 0x046f, 0x046f, 0x0471, 0x0471, 0x0473, 0x0473, + 0x0475, 0x0475, 0x0476, 0x0477, 0x0479, 0x0479, 0x047b, 0x047b, 0x047d, 0x047d, 0x047f, 0x047f, + 0x0481, 0x0481, 0x0482, 0x0483, 0x0484, 0x0485, 0x0486, 0x0487, 0x0488, 0x0489, 0x048a, 0x048b, + 0x048c, 0x048d, 0x048e, 0x048f, 0x0491, 0x0491, 0x0493, 0x0493, 0x0495, 0x0495, 0x0497, 0x0497, + 0x0499, 0x0499, 0x049b, 0x049b, 0x049d, 0x049d, 0x049f, 0x049f, 0x04a1, 0x04a1, 0x04a3, 0x04a3, + 0x04a5, 0x04a5, 0x04a7, 0x04a7, 0x04a9, 0x04a9, 0x04ab, 0x04ab, 0x04ad, 0x04ad, 0x04af, 0x04af, + 0x04b1, 0x04b1, 0x04b3, 0x04b3, 0x04b5, 0x04b5, 0x04b7, 0x04b7, 0x04b9, 0x04b9, 0x04bb, 0x04bb, + 0x04bd, 0x04bd, 0x04bf, 0x04bf, 0x04c0, 0x04c1, 0x04c2, 0x04c4, 0x04c4, 0x04c5, 0x04c6, 0x04c8, + 0x04c8, 0x04c9, 0x04ca, 0x04cc, 0x04cc, 0x04cd, 0x04ce, 0x04cf, 0x04d0, 0x04d1, 0x04d2, 0x04d3, + 0x04d4, 0x04d5, 0x04d6, 0x04d7, 0x04d8, 0x04d9, 0x04da, 0x04db, 0x04dc, 0x04dd, 0x04de, 0x04df, + 0x04e0, 0x04e1, 0x04e2, 0x04e3, 0x04e4, 0x04e5, 0x04e6, 0x04e7, 0x04e8, 0x04e9, 0x04ea, 0x04eb, + 0x04ec, 0x04ed, 0x04ee, 0x04ef, 0x04f0, 0x04f1, 0x04f2, 0x04f3, 0x04f4, 0x04f5, 0x04f6, 0x04f7, + 0x04f8, 0x04f9, 0x04fa, 0x04fb, 0x04fc, 0x04fd, 0x04fe, 0x04ff, 0x0500, 0x0501, 0x0502, 0x0503, + 0x0504, 0x0505, 0x0506, 0x0507, 0x0508, 0x0509, 0x050a, 0x050b, 0x050c, 0x050d, 0x050e, 0x050f, + 0x0510, 0x0511, 0x0512, 0x0513, 0x0514, 0x0515, 0x0516, 0x0517, 0x0518, 0x0519, 0x051a, 0x051b, + 0x051c, 0x051d, 0x051e, 0x051f, 0x0520, 0x0521, 0x0522, 0x0523, 0x0524, 0x0525, 0x0526, 0x0527, + 0x0528, 0x0529, 0x052a, 0x052b, 0x052c, 0x052d, 0x052e, 0x052f, 0x0530, 0x0561, 0x0562, 0x0563, + 0x0564, 0x0565, 0x0566, 0x0567, 0x0568, 0x0569, 0x056a, 0x056b, 0x056c, 0x056d, 0x056e, 0x056f, + 0x0570, 0x0571, 0x0572, 0x0573, 0x0574, 0x0575, 0x0576, 0x0577, 0x0578, 0x0579, 0x057a, 0x057b, + 0x057c, 0x057d, 0x057e, 0x057f, 0x0580, 0x0581, 0x0582, 0x0583, 0x0584, 0x0585, 0x0586, 0x0557, + 0x0558, 0x0559, 0x055a, 0x055b, 0x055c, 0x055d, 0x055e, 0x055f, 0x0560, 0x0561, 0x0562, 0x0563, + 0x0564, 0x0565, 0x0566, 0x0567, 0x0568, 0x0569, 0x056a, 0x056b, 0x056c, 0x056d, 0x056e, 0x056f, + 0x0570, 0x0571, 0x0572, 0x0573, 0x0574, 0x0575, 0x0576, 0x0577, 0x0578, 0x0579, 0x057a, 0x057b, + 0x057c, 0x057d, 0x057e, 0x057f, 0x0580, 0x0581, 0x0582, 0x0583, 0x0584, 0x0585, 0x0586, 0x0587, + 0x0588, 0x0589, 0x058a, 0x058b, 0x058c, 0x058d, 0x058e, 0x058f, 0x0590, 0x0591, 0x0592, 0x0593, + 0x0594, 0x0595, 0x0596, 0x0597, 0x0598, 0x0599, 0x059a, 0x059b, 0x059c, 0x059d, 0x059e, 0x059f, + 0x05a0, 0x05a1, 0x05a2, 0x05a3, 0x05a4, 0x05a5, 0x05a6, 0x05a7, 0x05a8, 0x05a9, 0x05aa, 0x05ab, + 0x05ac, 0x05ad, 0x05ae, 0x05af, 0x05b0, 0x05b1, 0x05b2, 0x05b3, 0x05b4, 0x05b5, 0x05b6, 0x05b7, + 0x05b8, 0x05b9, 0x05ba, 0x05bb, 0x05bc, 0x05bd, 0x05be, 0x05bf, 0x05c0, 0x05c1, 0x05c2, 0x05c3, + 0x05c4, 0x05c5, 0x05c6, 0x05c7, 0x05c8, 0x05c9, 0x05ca, 0x05cb, 0x05cc, 0x05cd, 0x05ce, 0x05cf, + 0x05d0, 0x05d1, 0x05d2, 0x05d3, 0x05d4, 0x05d5, 0x05d6, 0x05d7, 0x05d8, 0x05d9, 0x05da, 0x05db, + 0x05dc, 0x05dd, 0x05de, 0x05df, 0x05e0, 0x05e1, 0x05e2, 0x05e3, 0x05e4, 0x05e5, 0x05e6, 0x05e7, + 0x05e8, 0x05e9, 0x05ea, 0x05eb, 0x05ec, 0x05ed, 0x05ee, 0x05ef, 0x05f0, 0x05f1, 0x05f2, 0x05f3, + 0x05f4, 0x05f5, 0x05f6, 0x05f7, 0x05f8, 0x05f9, 0x05fa, 0x05fb, 0x05fc, 0x05fd, 0x05fe, 0x05ff, + 0x1000, 0x1001, 0x1002, 0x1003, 0x1004, 0x1005, 0x1006, 0x1007, 0x1008, 0x1009, 0x100a, 0x100b, + 0x100c, 0x100d, 0x100e, 0x100f, 0x1010, 0x1011, 0x1012, 0x1013, 0x1014, 0x1015, 0x1016, 0x1017, + 0x1018, 0x1019, 0x101a, 0x101b, 0x101c, 0x101d, 0x101e, 0x101f, 0x1020, 0x1021, 0x1022, 0x1023, + 0x1024, 0x1025, 0x1026, 0x1027, 0x1028, 0x1029, 0x102a, 0x102b, 0x102c, 0x102d, 0x102e, 0x102f, + 0x1030, 0x1031, 0x1032, 0x1033, 0x1034, 0x1035, 0x1036, 0x1037, 0x1038, 0x1039, 0x103a, 0x103b, + 0x103c, 0x103d, 0x103e, 0x103f, 0x1040, 0x1041, 0x1042, 0x1043, 0x1044, 0x1045, 0x1046, 0x1047, + 0x1048, 0x1049, 0x104a, 0x104b, 0x104c, 0x104d, 0x104e, 0x104f, 0x1050, 0x1051, 0x1052, 0x1053, + 0x1054, 0x1055, 0x1056, 0x1057, 0x1058, 0x1059, 0x105a, 0x105b, 0x105c, 0x105d, 0x105e, 0x105f, + 0x1060, 0x1061, 0x1062, 0x1063, 0x1064, 0x1065, 0x1066, 0x1067, 0x1068, 0x1069, 0x106a, 0x106b, + 0x106c, 0x106d, 0x106e, 0x106f, 0x1070, 0x1071, 0x1072, 0x1073, 0x1074, 0x1075, 0x1076, 0x1077, + 0x1078, 0x1079, 0x107a, 0x107b, 0x107c, 0x107d, 0x107e, 0x107f, 0x1080, 0x1081, 0x1082, 0x1083, + 0x1084, 0x1085, 0x1086, 0x1087, 0x1088, 0x1089, 0x108a, 0x108b, 0x108c, 0x108d, 0x108e, 0x108f, + 0x1090, 0x1091, 0x1092, 0x1093, 0x1094, 0x1095, 0x1096, 0x1097, 0x1098, 0x1099, 0x109a, 0x109b, + 0x109c, 0x109d, 0x109e, 0x109f, 0x10d0, 0x10d1, 0x10d2, 0x10d3, 0x10d4, 0x10d5, 0x10d6, 0x10d7, + 0x10d8, 0x10d9, 0x10da, 0x10db, 0x10dc, 0x10dd, 0x10de, 0x10df, 0x10e0, 0x10e1, 0x10e2, 0x10e3, + 0x10e4, 0x10e5, 0x10e6, 0x10e7, 0x10e8, 0x10e9, 0x10ea, 0x10eb, 0x10ec, 0x10ed, 0x10ee, 0x10ef, + 0x10f0, 0x10f1, 0x10f2, 0x10f3, 0x10f4, 0x10f5, 0x10c6, 0x10c7, 0x10c8, 0x10c9, 0x10ca, 0x10cb, + 0x10cc, 0x10cd, 0x10ce, 0x10cf, 0x10d0, 0x10d1, 0x10d2, 0x10d3, 0x10d4, 0x10d5, 0x10d6, 0x10d7, + 0x10d8, 0x10d9, 0x10da, 0x10db, 0x10dc, 0x10dd, 0x10de, 0x10df, 0x10e0, 0x10e1, 0x10e2, 0x10e3, + 0x10e4, 0x10e5, 0x10e6, 0x10e7, 0x10e8, 0x10e9, 0x10ea, 0x10eb, 0x10ec, 0x10ed, 0x10ee, 0x10ef, + 0x10f0, 0x10f1, 0x10f2, 0x10f3, 0x10f4, 0x10f5, 0x10f6, 0x10f7, 0x10f8, 0x10f9, 0x10fa, 0x10fb, + 0x10fc, 0x10fd, 0x10fe, 0x10ff, 0x2000, 0x2001, 0x2002, 0x2003, 0x2004, 0x2005, 0x2006, 0x2007, + 0x2008, 0x2009, 0x200a, 0x200b, 0x0000, 0x0000, 0x0000, 0x0000, 0x2010, 0x2011, 0x2012, 0x2013, + 0x2014, 0x2015, 0x2016, 0x2017, 0x2018, 0x2019, 0x201a, 0x201b, 0x201c, 0x201d, 0x201e, 0x201f, + 0x2020, 0x2021, 0x2022, 0x2023, 0x2024, 0x2025, 0x2026, 0x2027, 0x2028, 0x2029, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x202f, 0x2030, 0x2031, 0x2032, 0x2033, 0x2034, 0x2035, 0x2036, 0x2037, + 0x2038, 0x2039, 0x203a, 0x203b, 0x203c, 0x203d, 0x203e, 0x203f, 0x2040, 0x2041, 0x2042, 0x2043, + 0x2044, 0x2045, 0x2046, 0x2047, 0x2048, 0x2049, 0x204a, 0x204b, 0x204c, 0x204d, 0x204e, 0x204f, + 0x2050, 0x2051, 0x2052, 0x2053, 0x2054, 0x2055, 0x2056, 0x2057, 0x2058, 0x2059, 0x205a, 0x205b, + 0x205c, 0x205d, 0x205e, 0x205f, 0x2060, 0x2061, 0x2062, 0x2063, 0x2064, 0x2065, 0x2066, 0x2067, + 0x2068, 0x2069, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x2070, 0x2071, 0x2072, 0x2073, + 0x2074, 0x2075, 0x2076, 0x2077, 0x2078, 0x2079, 0x207a, 0x207b, 0x207c, 0x207d, 0x207e, 0x207f, + 0x2080, 0x2081, 0x2082, 0x2083, 0x2084, 0x2085, 0x2086, 0x2087, 0x2088, 0x2089, 0x208a, 0x208b, + 0x208c, 0x208d, 0x208e, 0x208f, 0x2090, 0x2091, 0x2092, 0x2093, 0x2094, 0x2095, 0x2096, 0x2097, + 0x2098, 0x2099, 0x209a, 0x209b, 0x209c, 0x209d, 0x209e, 0x209f, 0x20a0, 0x20a1, 0x20a2, 0x20a3, + 0x20a4, 0x20a5, 0x20a6, 0x20a7, 0x20a8, 0x20a9, 0x20aa, 0x20ab, 0x20ac, 0x20ad, 0x20ae, 0x20af, + 0x20b0, 0x20b1, 0x20b2, 0x20b3, 0x20b4, 0x20b5, 0x20b6, 0x20b7, 0x20b8, 0x20b9, 0x20ba, 0x20bb, + 0x20bc, 0x20bd, 0x20be, 0x20bf, 0x20c0, 0x20c1, 0x20c2, 0x20c3, 0x20c4, 0x20c5, 0x20c6, 0x20c7, + 0x20c8, 0x20c9, 0x20ca, 0x20cb, 0x20cc, 0x20cd, 0x20ce, 0x20cf, 0x20d0, 0x20d1, 0x20d2, 0x20d3, + 0x20d4, 0x20d5, 0x20d6, 0x20d7, 0x20d8, 0x20d9, 0x20da, 0x20db, 0x20dc, 0x20dd, 0x20de, 0x20df, + 0x20e0, 0x20e1, 0x20e2, 0x20e3, 0x20e4, 0x20e5, 0x20e6, 0x20e7, 0x20e8, 0x20e9, 0x20ea, 0x20eb, + 0x20ec, 0x20ed, 0x20ee, 0x20ef, 0x20f0, 0x20f1, 0x20f2, 0x20f3, 0x20f4, 0x20f5, 0x20f6, 0x20f7, + 0x20f8, 0x20f9, 0x20fa, 0x20fb, 0x20fc, 0x20fd, 0x20fe, 0x20ff, 0x2100, 0x2101, 0x2102, 0x2103, + 0x2104, 0x2105, 0x2106, 0x2107, 0x2108, 0x2109, 0x210a, 0x210b, 0x210c, 0x210d, 0x210e, 0x210f, + 0x2110, 0x2111, 0x2112, 0x2113, 0x2114, 0x2115, 0x2116, 0x2117, 0x2118, 0x2119, 0x211a, 0x211b, + 0x211c, 0x211d, 0x211e, 0x211f, 0x2120, 0x2121, 0x2122, 0x2123, 0x2124, 0x2125, 0x2126, 0x2127, + 0x2128, 0x2129, 0x212a, 0x212b, 0x212c, 0x212d, 0x212e, 0x212f, 0x2130, 0x2131, 0x2132, 0x2133, + 0x2134, 0x2135, 0x2136, 0x2137, 0x2138, 0x2139, 0x213a, 0x213b, 0x213c, 0x213d, 0x213e, 0x213f, + 0x2140, 0x2141, 0x2142, 0x2143, 0x2144, 0x2145, 0x2146, 0x2147, 0x2148, 0x2149, 0x214a, 0x214b, + 0x214c, 0x214d, 0x214e, 0x214f, 0x2150, 0x2151, 0x2152, 0x2153, 0x2154, 0x2155, 0x2156, 0x2157, + 0x2158, 0x2159, 0x215a, 0x215b, 0x215c, 0x215d, 0x215e, 0x215f, 0x2170, 0x2171, 0x2172, 0x2173, + 0x2174, 0x2175, 0x2176, 0x2177, 0x2178, 0x2179, 0x217a, 0x217b, 0x217c, 0x217d, 0x217e, 0x217f, + 0x2170, 0x2171, 0x2172, 0x2173, 0x2174, 0x2175, 0x2176, 0x2177, 0x2178, 0x2179, 0x217a, 0x217b, + 0x217c, 0x217d, 0x217e, 0x217f, 0x2180, 0x2181, 0x2182, 0x2183, 0x2184, 0x2185, 0x2186, 0x2187, + 0x2188, 0x2189, 0x218a, 0x218b, 0x218c, 0x218d, 0x218e, 0x218f, 0x2190, 0x2191, 0x2192, 0x2193, + 0x2194, 0x2195, 0x2196, 0x2197, 0x2198, 0x2199, 0x219a, 0x219b, 0x219c, 0x219d, 0x219e, 0x219f, + 0x21a0, 0x21a1, 0x21a2, 0x21a3, 0x21a4, 0x21a5, 0x21a6, 0x21a7, 0x21a8, 0x21a9, 0x21aa, 0x21ab, + 0x21ac, 0x21ad, 0x21ae, 0x21af, 0x21b0, 0x21b1, 0x21b2, 0x21b3, 0x21b4, 0x21b5, 0x21b6, 0x21b7, + 0x21b8, 0x21b9, 0x21ba, 0x21bb, 0x21bc, 0x21bd, 0x21be, 0x21bf, 0x21c0, 0x21c1, 0x21c2, 0x21c3, + 0x21c4, 0x21c5, 0x21c6, 0x21c7, 0x21c8, 0x21c9, 0x21ca, 0x21cb, 0x21cc, 0x21cd, 0x21ce, 0x21cf, + 0x21d0, 0x21d1, 0x21d2, 0x21d3, 0x21d4, 0x21d5, 0x21d6, 0x21d7, 0x21d8, 0x21d9, 0x21da, 0x21db, + 0x21dc, 0x21dd, 0x21de, 0x21df, 0x21e0, 0x21e1, 0x21e2, 0x21e3, 0x21e4, 0x21e5, 0x21e6, 0x21e7, + 0x21e8, 0x21e9, 0x21ea, 0x21eb, 0x21ec, 0x21ed, 0x21ee, 0x21ef, 0x21f0, 0x21f1, 0x21f2, 0x21f3, + 0x21f4, 0x21f5, 0x21f6, 0x21f7, 0x21f8, 0x21f9, 0x21fa, 0x21fb, 0x21fc, 0x21fd, 0x21fe, 0x21ff, + 0xfe00, 0xfe01, 0xfe02, 0xfe03, 0xfe04, 0xfe05, 0xfe06, 0xfe07, 0xfe08, 0xfe09, 0xfe0a, 0xfe0b, + 0xfe0c, 0xfe0d, 0xfe0e, 0xfe0f, 0xfe10, 0xfe11, 0xfe12, 0xfe13, 0xfe14, 0xfe15, 0xfe16, 0xfe17, + 0xfe18, 0xfe19, 0xfe1a, 0xfe1b, 0xfe1c, 0xfe1d, 0xfe1e, 0xfe1f, 0xfe20, 0xfe21, 0xfe22, 0xfe23, + 0xfe24, 0xfe25, 0xfe26, 0xfe27, 0xfe28, 0xfe29, 0xfe2a, 0xfe2b, 0xfe2c, 0xfe2d, 0xfe2e, 0xfe2f, + 0xfe30, 0xfe31, 0xfe32, 0xfe33, 0xfe34, 0xfe35, 0xfe36, 0xfe37, 0xfe38, 0xfe39, 0xfe3a, 0xfe3b, + 0xfe3c, 0xfe3d, 0xfe3e, 0xfe3f, 0xfe40, 0xfe41, 0xfe42, 0xfe43, 0xfe44, 0xfe45, 0xfe46, 0xfe47, + 0xfe48, 0xfe49, 0xfe4a, 0xfe4b, 0xfe4c, 0xfe4d, 0xfe4e, 0xfe4f, 0xfe50, 0xfe51, 0xfe52, 0xfe53, + 0xfe54, 0xfe55, 0xfe56, 0xfe57, 0xfe58, 0xfe59, 0xfe5a, 0xfe5b, 0xfe5c, 0xfe5d, 0xfe5e, 0xfe5f, + 0xfe60, 0xfe61, 0xfe62, 0xfe63, 0xfe64, 0xfe65, 0xfe66, 0xfe67, 0xfe68, 0xfe69, 0xfe6a, 0xfe6b, + 0xfe6c, 0xfe6d, 0xfe6e, 0xfe6f, 0xfe70, 0xfe71, 0xfe72, 0xfe73, 0xfe74, 0xfe75, 0xfe76, 0xfe77, + 0xfe78, 0xfe79, 0xfe7a, 0xfe7b, 0xfe7c, 0xfe7d, 0xfe7e, 0xfe7f, 0xfe80, 0xfe81, 0xfe82, 0xfe83, + 0xfe84, 0xfe85, 0xfe86, 0xfe87, 0xfe88, 0xfe89, 0xfe8a, 0xfe8b, 0xfe8c, 0xfe8d, 0xfe8e, 0xfe8f, + 0xfe90, 0xfe91, 0xfe92, 0xfe93, 0xfe94, 0xfe95, 0xfe96, 0xfe97, 0xfe98, 0xfe99, 0xfe9a, 0xfe9b, + 0xfe9c, 0xfe9d, 0xfe9e, 0xfe9f, 0xfea0, 0xfea1, 0xfea2, 0xfea3, 0xfea4, 0xfea5, 0xfea6, 0xfea7, + 0xfea8, 0xfea9, 0xfeaa, 0xfeab, 0xfeac, 0xfead, 0xfeae, 0xfeaf, 0xfeb0, 0xfeb1, 0xfeb2, 0xfeb3, + 0xfeb4, 0xfeb5, 0xfeb6, 0xfeb7, 0xfeb8, 0xfeb9, 0xfeba, 0xfebb, 0xfebc, 0xfebd, 0xfebe, 0xfebf, + 0xfec0, 0xfec1, 0xfec2, 0xfec3, 0xfec4, 0xfec5, 0xfec6, 0xfec7, 0xfec8, 0xfec9, 0xfeca, 0xfecb, + 0xfecc, 0xfecd, 0xfece, 0xfecf, 0xfed0, 0xfed1, 0xfed2, 0xfed3, 0xfed4, 0xfed5, 0xfed6, 0xfed7, + 0xfed8, 0xfed9, 0xfeda, 0xfedb, 0xfedc, 0xfedd, 0xfede, 0xfedf, 0xfee0, 0xfee1, 0xfee2, 0xfee3, + 0xfee4, 0xfee5, 0xfee6, 0xfee7, 0xfee8, 0xfee9, 0xfeea, 0xfeeb, 0xfeec, 0xfeed, 0xfeee, 0xfeef, + 0xfef0, 0xfef1, 0xfef2, 0xfef3, 0xfef4, 0xfef5, 0xfef6, 0xfef7, 0xfef8, 0xfef9, 0xfefa, 0xfefb, + 0xfefc, 0xfefd, 0xfefe, 0x0000, 0xff00, 0xff01, 0xff02, 0xff03, 0xff04, 0xff05, 0xff06, 0xff07, + 0xff08, 0xff09, 0xff0a, 0xff0b, 0xff0c, 0xff0d, 0xff0e, 0xff0f, 0xff10, 0xff11, 0xff12, 0xff13, + 0xff14, 0xff15, 0xff16, 0xff17, 0xff18, 0xff19, 0xff1a, 0xff1b, 0xff1c, 0xff1d, 0xff1e, 0xff1f, + 0xff20, 0xff41, 0xff42, 0xff43, 0xff44, 0xff45, 0xff46, 0xff47, 0xff48, 0xff49, 0xff4a, 0xff4b, + 0xff4c, 0xff4d, 0xff4e, 0xff4f, 0xff50, 0xff51, 0xff52, 0xff53, 0xff54, 0xff55, 0xff56, 0xff57, + 0xff58, 0xff59, 0xff5a, 0xff3b, 0xff3c, 0xff3d, 0xff3e, 0xff3f, 0xff40, 0xff41, 0xff42, 0xff43, + 0xff44, 0xff45, 0xff46, 0xff47, 0xff48, 0xff49, 0xff4a, 0xff4b, 0xff4c, 0xff4d, 0xff4e, 0xff4f, + 0xff50, 0xff51, 0xff52, 0xff53, 0xff54, 0xff55, 0xff56, 0xff57, 0xff58, 0xff59, 0xff5a, 0xff5b, + 0xff5c, 0xff5d, 0xff5e, 0xff5f, 0xff60, 0xff61, 0xff62, 0xff63, 0xff64, 0xff65, 0xff66, 0xff67, + 0xff68, 0xff69, 0xff6a, 0xff6b, 0xff6c, 0xff6d, 0xff6e, 0xff6f, 0xff70, 0xff71, 0xff72, 0xff73, + 0xff74, 0xff75, 0xff76, 0xff77, 0xff78, 0xff79, 0xff7a, 0xff7b, 0xff7c, 0xff7d, 0xff7e, 0xff7f, + 0xff80, 0xff81, 0xff82, 0xff83, 0xff84, 0xff85, 0xff86, 0xff87, 0xff88, 0xff89, 0xff8a, 0xff8b, + 0xff8c, 0xff8d, 0xff8e, 0xff8f, 0xff90, 0xff91, 0xff92, 0xff93, 0xff94, 0xff95, 0xff96, 0xff97, + 0xff98, 0xff99, 0xff9a, 0xff9b, 0xff9c, 0xff9d, 0xff9e, 0xff9f, 0xffa0, 0xffa1, 0xffa2, 0xffa3, + 0xffa4, 0xffa5, 0xffa6, 0xffa7, 0xffa8, 0xffa9, 0xffaa, 0xffab, 0xffac, 0xffad, 0xffae, 0xffaf, + 0xffb0, 0xffb1, 0xffb2, 0xffb3, 0xffb4, 0xffb5, 0xffb6, 0xffb7, 0xffb8, 0xffb9, 0xffba, 0xffbb, + 0xffbc, 0xffbd, 0xffbe, 0xffbf, 0xffc0, 0xffc1, 0xffc2, 0xffc3, 0xffc4, 0xffc5, 0xffc6, 0xffc7, + 0xffc8, 0xffc9, 0xffca, 0xffcb, 0xffcc, 0xffcd, 0xffce, 0xffcf, 0xffd0, 0xffd1, 0xffd2, 0xffd3, + 0xffd4, 0xffd5, 0xffd6, 0xffd7, 0xffd8, 0xffd9, 0xffda, 0xffdb, 0xffdc, 0xffdd, 0xffde, 0xffdf, + 0xffe0, 0xffe1, 0xffe2, 0xffe3, 0xffe4, 0xffe5, 0xffe6, 0xffe7, 0xffe8, 0xffe9, 0xffea, 0xffeb, + 0xffec, 0xffed, 0xffee, 0xffef, 0xfff0, 0xfff1, 0xfff2, 0xfff3, 0xfff4, 0xfff5, 0xfff6, 0xfff7, + 0xfff8, 0xfff9, 0xfffa, 0xfffb, 0xfffc, 0xfffd, 0xfffe, 0xffff, +]; + +#[rustfmt::skip] +static DECOMPOSE: [u16; 4218] = [ + 0x0010, 0x04c0, 0x0000, 0x06f0, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0xffff, 0xffff, + 0xffff, 0xffff, 0x0000, 0x07b0, 0x0020, 0x0070, 0x0160, 0x0190, 0x0230, 0x0000, 0x0000, 0x0000, + 0x0000, 0x02d0, 0x0340, 0x0360, 0x03b0, 0x03e0, 0x0400, 0x0430, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0030, 0x0040, 0x0050, 0x0060, + 0x2042, 0x204a, 0x2052, 0x205a, 0x2062, 0x206a, 0x0000, 0x2072, 0x207a, 0x2082, 0x208a, 0x2092, + 0x209a, 0x20a2, 0x20aa, 0x20b2, 0x0000, 0x20ba, 0x20c2, 0x20ca, 0x20d2, 0x20da, 0x20e2, 0x0000, + 0x0000, 0x20ea, 0x20f2, 0x20fa, 0x2102, 0x210a, 0x0000, 0x0000, 0x2112, 0x211a, 0x2122, 0x212a, + 0x2132, 0x213a, 0x0000, 0x2142, 0x214a, 0x2152, 0x215a, 0x2162, 0x216a, 0x2172, 0x217a, 0x2182, + 0x0000, 0x218a, 0x2192, 0x219a, 0x21a2, 0x21aa, 0x21b2, 0x0000, 0x0000, 0x21ba, 0x21c2, 0x21ca, + 0x21d2, 0x21da, 0x0000, 0x21e2, 0x0080, 0x0090, 0x00a0, 0x00b0, 0x00c0, 0x00d0, 0x00e0, 0x00f0, + 0x0000, 0x0000, 0x0100, 0x0110, 0x0120, 0x0130, 0x0140, 0x0150, 0x21ea, 0x21f2, 0x21fa, 0x2202, + 0x220a, 0x2212, 0x221a, 0x2222, 0x222a, 0x2232, 0x223a, 0x2242, 0x224a, 0x2252, 0x225a, 0x2262, + 0x0000, 0x0000, 0x226a, 0x2272, 0x227a, 0x2282, 0x228a, 0x2292, 0x229a, 0x22a2, 0x22aa, 0x22b2, + 0x22ba, 0x22c2, 0x22ca, 0x22d2, 0x22da, 0x22e2, 0x22ea, 0x22f2, 0x22fa, 0x2302, 0x0000, 0x0000, + 0x230a, 0x2312, 0x231a, 0x2322, 0x232a, 0x2332, 0x233a, 0x2342, 0x234a, 0x0000, 0x0000, 0x0000, + 0x2352, 0x235a, 0x2362, 0x236a, 0x0000, 0x2372, 0x237a, 0x2382, 0x238a, 0x2392, 0x239a, 0x0000, + 0x0000, 0x0000, 0x0000, 0x23a2, 0x23aa, 0x23b2, 0x23ba, 0x23c2, 0x23ca, 0x0000, 0x0000, 0x0000, + 0x23d2, 0x23da, 0x23e2, 0x23ea, 0x23f2, 0x23fa, 0x0000, 0x0000, 0x2402, 0x240a, 0x2412, 0x241a, + 0x2422, 0x242a, 0x2432, 0x243a, 0x2442, 0x244a, 0x2452, 0x245a, 0x2462, 0x246a, 0x2472, 0x247a, + 0x2482, 0x248a, 0x0000, 0x0000, 0x2492, 0x249a, 0x24a2, 0x24aa, 0x24b2, 0x24ba, 0x24c2, 0x24ca, + 0x24d2, 0x24da, 0x24e2, 0x24ea, 0x24f2, 0x24fa, 0x2502, 0x250a, 0x2512, 0x251a, 0x2522, 0x252a, + 0x2532, 0x253a, 0x2542, 0x0000, 0x254a, 0x2552, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x255a, 0x2562, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x256a, 0x2572, 0x257a, 0x2582, 0x258a, 0x2592, 0x259a, 0x25a2, 0x25ab, 0x25b7, 0x25c3, + 0x25cf, 0x25db, 0x25e7, 0x25f3, 0x25ff, 0x0000, 0x260b, 0x2617, 0x2623, 0x262f, 0x263a, 0x2642, + 0x0000, 0x0000, 0x264a, 0x2652, 0x265a, 0x2662, 0x266a, 0x2672, 0x267b, 0x2687, 0x2692, 0x269a, + 0x26a2, 0x0000, 0x0000, 0x0000, 0x26aa, 0x26b2, 0x0000, 0x0000, 0x0000, 0x0000, 0x26bb, 0x26c7, + 0x26d2, 0x26da, 0x26e2, 0x26ea, 0x0170, 0x0180, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x26f2, 0x26fa, 0x2702, 0x270a, + 0x2712, 0x271a, 0x2722, 0x272a, 0x2732, 0x273a, 0x2742, 0x274a, 0x2752, 0x275a, 0x2762, 0x276a, + 0x2772, 0x277a, 0x2782, 0x278a, 0x2792, 0x279a, 0x27a2, 0x27aa, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x01a0, 0x0000, 0x0000, 0x01b0, 0x0000, 0x0000, 0x01c0, + 0x01d0, 0x01e0, 0x01f0, 0x0200, 0x0210, 0x0220, 0x0000, 0x0000, 0x27b2, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x27b9, 0x27bd, 0x0000, 0x27c1, 0x27c6, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x27cd, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x27d1, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x27d6, 0x27de, 0x27e5, 0x27ea, 0x27f2, 0x27fa, 0x0000, 0x2802, 0x0000, 0x280a, 0x2812, + 0x281b, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x2826, 0x282e, 0x2836, 0x283e, 0x2846, 0x284e, 0x2857, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x2862, 0x286a, + 0x2872, 0x287a, 0x2882, 0x0000, 0x0000, 0x0000, 0x0000, 0x288a, 0x2892, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0240, 0x0250, 0x0000, 0x0260, + 0x0000, 0x0270, 0x0000, 0x0280, 0x0000, 0x0000, 0x0000, 0x0000, 0x0290, 0x02a0, 0x02b0, 0x02c0, + 0x0000, 0x289a, 0x0000, 0x28a2, 0x0000, 0x0000, 0x0000, 0x28aa, 0x0000, 0x0000, 0x0000, 0x0000, + 0x28b2, 0x0000, 0x28ba, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x28c2, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x28ca, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x28d2, 0x0000, 0x28da, 0x0000, 0x0000, 0x0000, 0x28e2, 0x0000, 0x0000, 0x0000, 0x0000, + 0x28ea, 0x0000, 0x28f2, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x28fa, 0x2902, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x290a, 0x2912, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x291a, 0x2922, 0x292a, 0x2932, 0x2939, 0x293d, 0x2942, 0x294a, 0x2951, 0x2955, 0x295a, 0x2962, + 0x296a, 0x2972, 0x297a, 0x2982, 0x2989, 0x298d, 0x2992, 0x299a, 0x29a2, 0x29aa, 0x29b2, 0x29ba, + 0x29c1, 0x29c5, 0x29ca, 0x29d2, 0x0000, 0x0000, 0x29da, 0x29e2, 0x29ea, 0x29f2, 0x29fa, 0x2a02, + 0x2a0a, 0x2a12, 0x0000, 0x0000, 0x2a1a, 0x2a22, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x02e0, 0x02f0, 0x0000, 0x0300, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0310, + 0x0320, 0x0330, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x2a2a, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x2a32, 0x0000, 0x0000, + 0x2a3a, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x2a42, 0x2a4a, 0x2a52, 0x2a5a, + 0x2a62, 0x2a6a, 0x2a72, 0x2a7a, 0x2a82, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x2a8a, 0x2a92, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x2a9a, 0x2aa2, 0x0000, 0x2aaa, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0350, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x2ab2, 0x2aba, 0x2ac2, 0x2aca, 0x0000, 0x2ad2, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0370, 0x0380, 0x0000, 0x0000, 0x0000, 0x0390, 0x0000, 0x0000, + 0x03a0, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x2ada, 0x0000, 0x0000, 0x2ae2, 0x2aea, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x2af2, 0x2afa, 0x0000, 0x2b02, + 0x0000, 0x0000, 0x0000, 0x0000, 0x2b0a, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x2b12, 0x2b1a, 0x2b22, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x03c0, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x03d0, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x2b2a, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x2b32, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x2b3a, + 0x2b42, 0x0000, 0x2b4a, 0x2b53, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x03f0, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x2b5e, 0x2b66, + 0x2b6e, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0410, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0420, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x2b76, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x2b7e, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0440, 0x0450, 0x0460, 0x0470, + 0x0480, 0x0490, 0x04a0, 0x04b0, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x2b86, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x2b8e, 0x0000, 0x0000, + 0x0000, 0x0000, 0x2b96, 0x0000, 0x0000, 0x0000, 0x0000, 0x2b9e, 0x0000, 0x0000, 0x0000, 0x0000, + 0x2ba6, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x2bae, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x2bb6, + 0x0000, 0x2bbe, 0x2bc6, 0x2bcf, 0x2bda, 0x2be3, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x2bee, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x2bf6, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x2bfe, 0x0000, 0x0000, 0x0000, 0x0000, 0x2c06, 0x0000, + 0x0000, 0x0000, 0x0000, 0x2c0e, 0x0000, 0x0000, 0x0000, 0x0000, 0x2c16, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x2c1e, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x04d0, 0x05e0, 0x04e0, 0x04f0, 0x0500, 0x0510, + 0x0520, 0x0530, 0x0540, 0x0550, 0x0560, 0x0570, 0x0580, 0x0590, 0x05a0, 0x05b0, 0x05c0, 0x05d0, + 0x2c26, 0x2c2e, 0x2c36, 0x2c3e, 0x2c46, 0x2c4e, 0x2c56, 0x2c5e, 0x2c67, 0x2c73, 0x2c7e, 0x2c86, + 0x2c8e, 0x2c96, 0x2c9e, 0x2ca6, 0x2cae, 0x2cb6, 0x2cbe, 0x2cc6, 0x2ccf, 0x2cdb, 0x2ce7, 0x2cf3, + 0x2cfe, 0x2d06, 0x2d0e, 0x2d16, 0x2d1f, 0x2d2b, 0x2d36, 0x2d3e, 0x2d46, 0x2d4e, 0x2d56, 0x2d5e, + 0x2d66, 0x2d6e, 0x2d76, 0x2d7e, 0x2d86, 0x2d8e, 0x2d96, 0x2d9e, 0x2da6, 0x2dae, 0x2db7, 0x2dc3, + 0x2dce, 0x2dd6, 0x2dde, 0x2de6, 0x2dee, 0x2df6, 0x2dfe, 0x2e06, 0x2e0f, 0x2e1b, 0x2e26, 0x2e2e, + 0x2e36, 0x2e3e, 0x2e46, 0x2e4e, 0x2e56, 0x2e5e, 0x2e66, 0x2e6e, 0x2e76, 0x2e7e, 0x2e86, 0x2e8e, + 0x2e96, 0x2e9e, 0x2ea6, 0x2eae, 0x2eb7, 0x2ec3, 0x2ecf, 0x2edb, 0x2ee7, 0x2ef3, 0x2eff, 0x2f0b, + 0x2f16, 0x2f1e, 0x2f26, 0x2f2e, 0x2f36, 0x2f3e, 0x2f46, 0x2f4e, 0x2f57, 0x2f63, 0x2f6e, 0x2f76, + 0x2f7e, 0x2f86, 0x2f8e, 0x2f96, 0x2f9f, 0x2fab, 0x2fb7, 0x2fc3, 0x2fcf, 0x2fdb, 0x2fe6, 0x2fee, + 0x2ff6, 0x2ffe, 0x3006, 0x300e, 0x3016, 0x301e, 0x3026, 0x302e, 0x3036, 0x303e, 0x3046, 0x304e, + 0x3057, 0x3063, 0x306f, 0x307b, 0x3086, 0x308e, 0x3096, 0x309e, 0x30a6, 0x30ae, 0x30b6, 0x30be, + 0x30c6, 0x30ce, 0x30d6, 0x30de, 0x30e6, 0x30ee, 0x30f6, 0x30fe, 0x3106, 0x310e, 0x3116, 0x311e, + 0x3126, 0x312e, 0x3136, 0x313e, 0x3146, 0x314e, 0x3156, 0x315e, 0x3166, 0x316e, 0x0000, 0x3176, + 0x0000, 0x0000, 0x0000, 0x0000, 0x317e, 0x3186, 0x318e, 0x3196, 0x319f, 0x31ab, 0x31b7, 0x31c3, + 0x31cf, 0x31db, 0x31e7, 0x31f3, 0x31ff, 0x320b, 0x3217, 0x3223, 0x322f, 0x323b, 0x3247, 0x3253, + 0x325f, 0x326b, 0x3277, 0x3283, 0x328e, 0x3296, 0x329e, 0x32a6, 0x32ae, 0x32b6, 0x32bf, 0x32cb, + 0x32d7, 0x32e3, 0x32ef, 0x32fb, 0x3307, 0x3313, 0x331f, 0x332b, 0x3336, 0x333e, 0x3346, 0x334e, + 0x3356, 0x335e, 0x3366, 0x336e, 0x3377, 0x3383, 0x338f, 0x339b, 0x33a7, 0x33b3, 0x33bf, 0x33cb, + 0x33d7, 0x33e3, 0x33ef, 0x33fb, 0x3407, 0x3413, 0x341f, 0x342b, 0x3437, 0x3443, 0x344f, 0x345b, + 0x3466, 0x346e, 0x3476, 0x347e, 0x3487, 0x3493, 0x349f, 0x34ab, 0x34b7, 0x34c3, 0x34cf, 0x34db, + 0x34e7, 0x34f3, 0x34fe, 0x3506, 0x350e, 0x3516, 0x351e, 0x3526, 0x352e, 0x3536, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x05f0, 0x0600, 0x0610, 0x0620, 0x0630, 0x0640, 0x0650, 0x0660, + 0x0670, 0x0680, 0x0690, 0x06a0, 0x06b0, 0x06c0, 0x06d0, 0x06e0, 0x353e, 0x3546, 0x354f, 0x355b, + 0x3567, 0x3573, 0x357f, 0x358b, 0x3596, 0x359e, 0x35a7, 0x35b3, 0x35bf, 0x35cb, 0x35d7, 0x35e3, + 0x35ee, 0x35f6, 0x35ff, 0x360b, 0x3617, 0x3623, 0x0000, 0x0000, 0x362e, 0x3636, 0x363f, 0x364b, + 0x3657, 0x3663, 0x0000, 0x0000, 0x366e, 0x3676, 0x367f, 0x368b, 0x3697, 0x36a3, 0x36af, 0x36bb, + 0x36c6, 0x36ce, 0x36d7, 0x36e3, 0x36ef, 0x36fb, 0x3707, 0x3713, 0x371e, 0x3726, 0x372f, 0x373b, + 0x3747, 0x3753, 0x375f, 0x376b, 0x3776, 0x377e, 0x3787, 0x3793, 0x379f, 0x37ab, 0x37b7, 0x37c3, + 0x37ce, 0x37d6, 0x37df, 0x37eb, 0x37f7, 0x3803, 0x0000, 0x0000, 0x380e, 0x3816, 0x381f, 0x382b, + 0x3837, 0x3843, 0x0000, 0x0000, 0x384e, 0x3856, 0x385f, 0x386b, 0x3877, 0x3883, 0x388f, 0x389b, + 0x0000, 0x38a6, 0x0000, 0x38af, 0x0000, 0x38bb, 0x0000, 0x38c7, 0x38d2, 0x38da, 0x38e3, 0x38ef, + 0x38fb, 0x3907, 0x3913, 0x391f, 0x392a, 0x3932, 0x393b, 0x3947, 0x3953, 0x395f, 0x396b, 0x3977, + 0x3982, 0x398a, 0x3992, 0x399a, 0x39a2, 0x39aa, 0x39b2, 0x39ba, 0x39c2, 0x39ca, 0x39d2, 0x39da, + 0x39e2, 0x39ea, 0x0000, 0x0000, 0x39f3, 0x39ff, 0x3a0c, 0x3a1c, 0x3a2c, 0x3a3c, 0x3a4c, 0x3a5c, + 0x3a6b, 0x3a77, 0x3a84, 0x3a94, 0x3aa4, 0x3ab4, 0x3ac4, 0x3ad4, 0x3ae3, 0x3aef, 0x3afc, 0x3b0c, + 0x3b1c, 0x3b2c, 0x3b3c, 0x3b4c, 0x3b5b, 0x3b67, 0x3b74, 0x3b84, 0x3b94, 0x3ba4, 0x3bb4, 0x3bc4, + 0x3bd3, 0x3bdf, 0x3bec, 0x3bfc, 0x3c0c, 0x3c1c, 0x3c2c, 0x3c3c, 0x3c4b, 0x3c57, 0x3c64, 0x3c74, + 0x3c84, 0x3c94, 0x3ca4, 0x3cb4, 0x3cc2, 0x3cca, 0x3cd3, 0x3cde, 0x3ce7, 0x0000, 0x3cf2, 0x3cfb, + 0x3d06, 0x3d0e, 0x3d16, 0x3d1e, 0x3d26, 0x0000, 0x3d2d, 0x0000, 0x0000, 0x3d32, 0x3d3b, 0x3d46, + 0x3d4f, 0x0000, 0x3d5a, 0x3d63, 0x3d6e, 0x3d76, 0x3d7e, 0x3d86, 0x3d8e, 0x3d96, 0x3d9e, 0x3da6, + 0x3dae, 0x3db6, 0x3dbf, 0x3dcb, 0x0000, 0x0000, 0x3dd6, 0x3ddf, 0x3dea, 0x3df2, 0x3dfa, 0x3e02, + 0x0000, 0x3e0a, 0x3e12, 0x3e1a, 0x3e22, 0x3e2a, 0x3e33, 0x3e3f, 0x3e4a, 0x3e52, 0x3e5a, 0x3e63, + 0x3e6e, 0x3e76, 0x3e7e, 0x3e86, 0x3e8e, 0x3e96, 0x3e9e, 0x3ea5, 0x0000, 0x0000, 0x3eab, 0x3eb6, + 0x3ebf, 0x0000, 0x3eca, 0x3ed3, 0x3ede, 0x3ee6, 0x3eee, 0x3ef6, 0x3efe, 0x3f05, 0x0000, 0x0000, + 0x0700, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0710, 0x0720, 0x0730, 0x0740, + 0x0000, 0x0750, 0x0760, 0x0770, 0x0780, 0x0790, 0x0000, 0x07a0, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x3f0a, 0x0000, 0x3f12, 0x0000, + 0x3f1a, 0x0000, 0x3f22, 0x0000, 0x3f2a, 0x0000, 0x3f32, 0x0000, 0x3f3a, 0x0000, 0x3f42, 0x0000, + 0x3f4a, 0x0000, 0x3f52, 0x0000, 0x3f5a, 0x0000, 0x3f62, 0x0000, 0x0000, 0x3f6a, 0x0000, 0x3f72, + 0x0000, 0x3f7a, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x3f82, 0x3f8a, 0x0000, 0x3f92, + 0x3f9a, 0x0000, 0x3fa2, 0x3faa, 0x0000, 0x3fb2, 0x3fba, 0x0000, 0x3fc2, 0x3fca, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x3fd2, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x3fda, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x3fe2, 0x0000, 0x3fea, 0x0000, 0x3ff2, 0x0000, 0x3ffa, 0x0000, + 0x4002, 0x0000, 0x400a, 0x0000, 0x4012, 0x0000, 0x401a, 0x0000, 0x4022, 0x0000, 0x402a, 0x0000, + 0x4032, 0x0000, 0x403a, 0x0000, 0x0000, 0x4042, 0x0000, 0x404a, 0x0000, 0x4052, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x405a, 0x4062, 0x0000, 0x406a, 0x4072, 0x0000, 0x407a, 0x4082, + 0x0000, 0x408a, 0x4092, 0x0000, 0x409a, 0x40a2, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x40aa, 0x0000, 0x0000, 0x40b2, 0x40ba, 0x40c2, 0x40ca, 0x0000, 0x0000, 0x0000, 0x40d2, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x07c0, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x07d0, 0x07e0, 0x07f0, 0x0800, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x40da, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x40e2, 0x40ea, + 0x40f3, 0x40ff, 0x410a, 0x4112, 0x411a, 0x4122, 0x412a, 0x4132, 0x413a, 0x4142, 0x414a, 0x0000, + 0x4152, 0x415a, 0x4162, 0x416a, 0x4172, 0x0000, 0x417a, 0x0000, 0x4182, 0x418a, 0x0000, 0x4192, + 0x419a, 0x0000, 0x41a2, 0x41aa, 0x41b2, 0x41ba, 0x41c2, 0x41ca, 0x41d2, 0x41da, 0x41e2, 0x0000, + 0x0041, 0x0300, 0x0041, 0x0301, 0x0041, 0x0302, 0x0041, 0x0303, 0x0041, 0x0308, 0x0041, 0x030a, + 0x0043, 0x0327, 0x0045, 0x0300, 0x0045, 0x0301, 0x0045, 0x0302, 0x0045, 0x0308, 0x0049, 0x0300, + 0x0049, 0x0301, 0x0049, 0x0302, 0x0049, 0x0308, 0x004e, 0x0303, 0x004f, 0x0300, 0x004f, 0x0301, + 0x004f, 0x0302, 0x004f, 0x0303, 0x004f, 0x0308, 0x0055, 0x0300, 0x0055, 0x0301, 0x0055, 0x0302, + 0x0055, 0x0308, 0x0059, 0x0301, 0x0061, 0x0300, 0x0061, 0x0301, 0x0061, 0x0302, 0x0061, 0x0303, + 0x0061, 0x0308, 0x0061, 0x030a, 0x0063, 0x0327, 0x0065, 0x0300, 0x0065, 0x0301, 0x0065, 0x0302, + 0x0065, 0x0308, 0x0069, 0x0300, 0x0069, 0x0301, 0x0069, 0x0302, 0x0069, 0x0308, 0x006e, 0x0303, + 0x006f, 0x0300, 0x006f, 0x0301, 0x006f, 0x0302, 0x006f, 0x0303, 0x006f, 0x0308, 0x0075, 0x0300, + 0x0075, 0x0301, 0x0075, 0x0302, 0x0075, 0x0308, 0x0079, 0x0301, 0x0079, 0x0308, 0x0041, 0x0304, + 0x0061, 0x0304, 0x0041, 0x0306, 0x0061, 0x0306, 0x0041, 0x0328, 0x0061, 0x0328, 0x0043, 0x0301, + 0x0063, 0x0301, 0x0043, 0x0302, 0x0063, 0x0302, 0x0043, 0x0307, 0x0063, 0x0307, 0x0043, 0x030c, + 0x0063, 0x030c, 0x0044, 0x030c, 0x0064, 0x030c, 0x0045, 0x0304, 0x0065, 0x0304, 0x0045, 0x0306, + 0x0065, 0x0306, 0x0045, 0x0307, 0x0065, 0x0307, 0x0045, 0x0328, 0x0065, 0x0328, 0x0045, 0x030c, + 0x0065, 0x030c, 0x0047, 0x0302, 0x0067, 0x0302, 0x0047, 0x0306, 0x0067, 0x0306, 0x0047, 0x0307, + 0x0067, 0x0307, 0x0047, 0x0327, 0x0067, 0x0327, 0x0048, 0x0302, 0x0068, 0x0302, 0x0049, 0x0303, + 0x0069, 0x0303, 0x0049, 0x0304, 0x0069, 0x0304, 0x0049, 0x0306, 0x0069, 0x0306, 0x0049, 0x0328, + 0x0069, 0x0328, 0x0049, 0x0307, 0x004a, 0x0302, 0x006a, 0x0302, 0x004b, 0x0327, 0x006b, 0x0327, + 0x004c, 0x0301, 0x006c, 0x0301, 0x004c, 0x0327, 0x006c, 0x0327, 0x004c, 0x030c, 0x006c, 0x030c, + 0x004e, 0x0301, 0x006e, 0x0301, 0x004e, 0x0327, 0x006e, 0x0327, 0x004e, 0x030c, 0x006e, 0x030c, + 0x004f, 0x0304, 0x006f, 0x0304, 0x004f, 0x0306, 0x006f, 0x0306, 0x004f, 0x030b, 0x006f, 0x030b, + 0x0052, 0x0301, 0x0072, 0x0301, 0x0052, 0x0327, 0x0072, 0x0327, 0x0052, 0x030c, 0x0072, 0x030c, + 0x0053, 0x0301, 0x0073, 0x0301, 0x0053, 0x0302, 0x0073, 0x0302, 0x0053, 0x0327, 0x0073, 0x0327, + 0x0053, 0x030c, 0x0073, 0x030c, 0x0054, 0x0327, 0x0074, 0x0327, 0x0054, 0x030c, 0x0074, 0x030c, + 0x0055, 0x0303, 0x0075, 0x0303, 0x0055, 0x0304, 0x0075, 0x0304, 0x0055, 0x0306, 0x0075, 0x0306, + 0x0055, 0x030a, 0x0075, 0x030a, 0x0055, 0x030b, 0x0075, 0x030b, 0x0055, 0x0328, 0x0075, 0x0328, + 0x0057, 0x0302, 0x0077, 0x0302, 0x0059, 0x0302, 0x0079, 0x0302, 0x0059, 0x0308, 0x005a, 0x0301, + 0x007a, 0x0301, 0x005a, 0x0307, 0x007a, 0x0307, 0x005a, 0x030c, 0x007a, 0x030c, 0x004f, 0x031b, + 0x006f, 0x031b, 0x0055, 0x031b, 0x0075, 0x031b, 0x0041, 0x030c, 0x0061, 0x030c, 0x0049, 0x030c, + 0x0069, 0x030c, 0x004f, 0x030c, 0x006f, 0x030c, 0x0055, 0x030c, 0x0075, 0x030c, 0x0055, 0x0308, + 0x0304, 0x0075, 0x0308, 0x0304, 0x0055, 0x0308, 0x0301, 0x0075, 0x0308, 0x0301, 0x0055, 0x0308, + 0x030c, 0x0075, 0x0308, 0x030c, 0x0055, 0x0308, 0x0300, 0x0075, 0x0308, 0x0300, 0x0041, 0x0308, + 0x0304, 0x0061, 0x0308, 0x0304, 0x0041, 0x0307, 0x0304, 0x0061, 0x0307, 0x0304, 0x00c6, 0x0304, + 0x00e6, 0x0304, 0x0047, 0x030c, 0x0067, 0x030c, 0x004b, 0x030c, 0x006b, 0x030c, 0x004f, 0x0328, + 0x006f, 0x0328, 0x004f, 0x0328, 0x0304, 0x006f, 0x0328, 0x0304, 0x01b7, 0x030c, 0x0292, 0x030c, + 0x006a, 0x030c, 0x0047, 0x0301, 0x0067, 0x0301, 0x0041, 0x030a, 0x0301, 0x0061, 0x030a, 0x0301, + 0x00c6, 0x0301, 0x00e6, 0x0301, 0x00d8, 0x0301, 0x00f8, 0x0301, 0x0041, 0x030f, 0x0061, 0x030f, + 0x0041, 0x0311, 0x0061, 0x0311, 0x0045, 0x030f, 0x0065, 0x030f, 0x0045, 0x0311, 0x0065, 0x0311, + 0x0049, 0x030f, 0x0069, 0x030f, 0x0049, 0x0311, 0x0069, 0x0311, 0x004f, 0x030f, 0x006f, 0x030f, + 0x004f, 0x0311, 0x006f, 0x0311, 0x0052, 0x030f, 0x0072, 0x030f, 0x0052, 0x0311, 0x0072, 0x0311, + 0x0055, 0x030f, 0x0075, 0x030f, 0x0055, 0x0311, 0x0075, 0x0311, 0x0306, 0x0307, 0x0300, 0x0301, + 0x0313, 0x0308, 0x030d, 0x02b9, 0x003b, 0x00a8, 0x030d, 0x0391, 0x030d, 0x00b7, 0x0395, 0x030d, + 0x0397, 0x030d, 0x0399, 0x030d, 0x039f, 0x030d, 0x03a5, 0x030d, 0x03a9, 0x030d, 0x03b9, 0x0308, + 0x030d, 0x0399, 0x0308, 0x03a5, 0x0308, 0x03b1, 0x030d, 0x03b5, 0x030d, 0x03b7, 0x030d, 0x03b9, + 0x030d, 0x03c5, 0x0308, 0x030d, 0x03b9, 0x0308, 0x03c5, 0x0308, 0x03bf, 0x030d, 0x03c5, 0x030d, + 0x03c9, 0x030d, 0x03d2, 0x030d, 0x03d2, 0x0308, 0x0415, 0x0308, 0x0413, 0x0301, 0x0406, 0x0308, + 0x041a, 0x0301, 0x0423, 0x0306, 0x0418, 0x0306, 0x0438, 0x0306, 0x0435, 0x0308, 0x0433, 0x0301, + 0x0456, 0x0308, 0x043a, 0x0301, 0x0443, 0x0306, 0x0474, 0x030f, 0x0475, 0x030f, 0x0416, 0x0306, + 0x0436, 0x0306, 0x0410, 0x0306, 0x0430, 0x0306, 0x0410, 0x0308, 0x0430, 0x0308, 0x00c6, 0x00e6, + 0x0415, 0x0306, 0x0435, 0x0306, 0x018f, 0x0259, 0x018f, 0x0308, 0x0259, 0x0308, 0x0416, 0x0308, + 0x0436, 0x0308, 0x0417, 0x0308, 0x0437, 0x0308, 0x01b7, 0x0292, 0x0418, 0x0304, 0x0438, 0x0304, + 0x0418, 0x0308, 0x0438, 0x0308, 0x041e, 0x0308, 0x043e, 0x0308, 0x019f, 0x0275, 0x019f, 0x0308, + 0x0275, 0x0308, 0x0423, 0x0304, 0x0443, 0x0304, 0x0423, 0x0308, 0x0443, 0x0308, 0x0423, 0x030b, + 0x0443, 0x030b, 0x0427, 0x0308, 0x0447, 0x0308, 0x042b, 0x0308, 0x044b, 0x0308, 0x0928, 0x093c, + 0x0930, 0x093c, 0x0933, 0x093c, 0x0915, 0x093c, 0x0916, 0x093c, 0x0917, 0x093c, 0x091c, 0x093c, + 0x0921, 0x093c, 0x0922, 0x093c, 0x092b, 0x093c, 0x092f, 0x093c, 0x09ac, 0x09bc, 0x09c7, 0x09be, + 0x09c7, 0x09d7, 0x09a1, 0x09bc, 0x09a2, 0x09bc, 0x09af, 0x09bc, 0x0a16, 0x0a3c, 0x0a17, 0x0a3c, + 0x0a1c, 0x0a3c, 0x0a21, 0x0a3c, 0x0a2b, 0x0a3c, 0x0b47, 0x0b56, 0x0b47, 0x0b3e, 0x0b47, 0x0b57, + 0x0b21, 0x0b3c, 0x0b22, 0x0b3c, 0x0b2f, 0x0b3c, 0x0b92, 0x0bd7, 0x0bc6, 0x0bbe, 0x0bc7, 0x0bbe, + 0x0bc6, 0x0bd7, 0x0c46, 0x0c56, 0x0cbf, 0x0cd5, 0x0cc6, 0x0cd5, 0x0cc6, 0x0cd6, 0x0cc6, 0x0cc2, + 0x0cc6, 0x0cc2, 0x0cd5, 0x0d46, 0x0d3e, 0x0d47, 0x0d3e, 0x0d46, 0x0d57, 0x0e4d, 0x0e32, 0x0ecd, + 0x0eb2, 0x0f42, 0x0fb7, 0x0f4c, 0x0fb7, 0x0f51, 0x0fb7, 0x0f56, 0x0fb7, 0x0f5b, 0x0fb7, 0x0f40, + 0x0fb5, 0x0f72, 0x0f71, 0x0f74, 0x0f71, 0x0fb2, 0x0f80, 0x0fb2, 0x0f80, 0x0f71, 0x0fb3, 0x0f80, + 0x0fb3, 0x0f80, 0x0f71, 0x0f80, 0x0f71, 0x0f92, 0x0fb7, 0x0f9c, 0x0fb7, 0x0fa1, 0x0fb7, 0x0fa6, + 0x0fb7, 0x0fab, 0x0fb7, 0x0f90, 0x0fb5, 0x0041, 0x0325, 0x0061, 0x0325, 0x0042, 0x0307, 0x0062, + 0x0307, 0x0042, 0x0323, 0x0062, 0x0323, 0x0042, 0x0331, 0x0062, 0x0331, 0x0043, 0x0327, 0x0301, + 0x0063, 0x0327, 0x0301, 0x0044, 0x0307, 0x0064, 0x0307, 0x0044, 0x0323, 0x0064, 0x0323, 0x0044, + 0x0331, 0x0064, 0x0331, 0x0044, 0x0327, 0x0064, 0x0327, 0x0044, 0x032d, 0x0064, 0x032d, 0x0045, + 0x0304, 0x0300, 0x0065, 0x0304, 0x0300, 0x0045, 0x0304, 0x0301, 0x0065, 0x0304, 0x0301, 0x0045, + 0x032d, 0x0065, 0x032d, 0x0045, 0x0330, 0x0065, 0x0330, 0x0045, 0x0327, 0x0306, 0x0065, 0x0327, + 0x0306, 0x0046, 0x0307, 0x0066, 0x0307, 0x0047, 0x0304, 0x0067, 0x0304, 0x0048, 0x0307, 0x0068, + 0x0307, 0x0048, 0x0323, 0x0068, 0x0323, 0x0048, 0x0308, 0x0068, 0x0308, 0x0048, 0x0327, 0x0068, + 0x0327, 0x0048, 0x032e, 0x0068, 0x032e, 0x0049, 0x0330, 0x0069, 0x0330, 0x0049, 0x0308, 0x0301, + 0x0069, 0x0308, 0x0301, 0x004b, 0x0301, 0x006b, 0x0301, 0x004b, 0x0323, 0x006b, 0x0323, 0x004b, + 0x0331, 0x006b, 0x0331, 0x004c, 0x0323, 0x006c, 0x0323, 0x004c, 0x0323, 0x0304, 0x006c, 0x0323, + 0x0304, 0x004c, 0x0331, 0x006c, 0x0331, 0x004c, 0x032d, 0x006c, 0x032d, 0x004d, 0x0301, 0x006d, + 0x0301, 0x004d, 0x0307, 0x006d, 0x0307, 0x004d, 0x0323, 0x006d, 0x0323, 0x004e, 0x0307, 0x006e, + 0x0307, 0x004e, 0x0323, 0x006e, 0x0323, 0x004e, 0x0331, 0x006e, 0x0331, 0x004e, 0x032d, 0x006e, + 0x032d, 0x004f, 0x0303, 0x0301, 0x006f, 0x0303, 0x0301, 0x004f, 0x0303, 0x0308, 0x006f, 0x0303, + 0x0308, 0x004f, 0x0304, 0x0300, 0x006f, 0x0304, 0x0300, 0x004f, 0x0304, 0x0301, 0x006f, 0x0304, + 0x0301, 0x0050, 0x0301, 0x0070, 0x0301, 0x0050, 0x0307, 0x0070, 0x0307, 0x0052, 0x0307, 0x0072, + 0x0307, 0x0052, 0x0323, 0x0072, 0x0323, 0x0052, 0x0323, 0x0304, 0x0072, 0x0323, 0x0304, 0x0052, + 0x0331, 0x0072, 0x0331, 0x0053, 0x0307, 0x0073, 0x0307, 0x0053, 0x0323, 0x0073, 0x0323, 0x0053, + 0x0301, 0x0307, 0x0073, 0x0301, 0x0307, 0x0053, 0x030c, 0x0307, 0x0073, 0x030c, 0x0307, 0x0053, + 0x0323, 0x0307, 0x0073, 0x0323, 0x0307, 0x0054, 0x0307, 0x0074, 0x0307, 0x0054, 0x0323, 0x0074, + 0x0323, 0x0054, 0x0331, 0x0074, 0x0331, 0x0054, 0x032d, 0x0074, 0x032d, 0x0055, 0x0324, 0x0075, + 0x0324, 0x0055, 0x0330, 0x0075, 0x0330, 0x0055, 0x032d, 0x0075, 0x032d, 0x0055, 0x0303, 0x0301, + 0x0075, 0x0303, 0x0301, 0x0055, 0x0304, 0x0308, 0x0075, 0x0304, 0x0308, 0x0056, 0x0303, 0x0076, + 0x0303, 0x0056, 0x0323, 0x0076, 0x0323, 0x0057, 0x0300, 0x0077, 0x0300, 0x0057, 0x0301, 0x0077, + 0x0301, 0x0057, 0x0308, 0x0077, 0x0308, 0x0057, 0x0307, 0x0077, 0x0307, 0x0057, 0x0323, 0x0077, + 0x0323, 0x0058, 0x0307, 0x0078, 0x0307, 0x0058, 0x0308, 0x0078, 0x0308, 0x0059, 0x0307, 0x0079, + 0x0307, 0x005a, 0x0302, 0x007a, 0x0302, 0x005a, 0x0323, 0x007a, 0x0323, 0x005a, 0x0331, 0x007a, + 0x0331, 0x0068, 0x0331, 0x0074, 0x0308, 0x0077, 0x030a, 0x0079, 0x030a, 0x017f, 0x0307, 0x0041, + 0x0323, 0x0061, 0x0323, 0x0041, 0x0309, 0x0061, 0x0309, 0x0041, 0x0302, 0x0301, 0x0061, 0x0302, + 0x0301, 0x0041, 0x0302, 0x0300, 0x0061, 0x0302, 0x0300, 0x0041, 0x0302, 0x0309, 0x0061, 0x0302, + 0x0309, 0x0041, 0x0302, 0x0303, 0x0061, 0x0302, 0x0303, 0x0041, 0x0323, 0x0302, 0x0061, 0x0323, + 0x0302, 0x0041, 0x0306, 0x0301, 0x0061, 0x0306, 0x0301, 0x0041, 0x0306, 0x0300, 0x0061, 0x0306, + 0x0300, 0x0041, 0x0306, 0x0309, 0x0061, 0x0306, 0x0309, 0x0041, 0x0306, 0x0303, 0x0061, 0x0306, + 0x0303, 0x0041, 0x0323, 0x0306, 0x0061, 0x0323, 0x0306, 0x0045, 0x0323, 0x0065, 0x0323, 0x0045, + 0x0309, 0x0065, 0x0309, 0x0045, 0x0303, 0x0065, 0x0303, 0x0045, 0x0302, 0x0301, 0x0065, 0x0302, + 0x0301, 0x0045, 0x0302, 0x0300, 0x0065, 0x0302, 0x0300, 0x0045, 0x0302, 0x0309, 0x0065, 0x0302, + 0x0309, 0x0045, 0x0302, 0x0303, 0x0065, 0x0302, 0x0303, 0x0045, 0x0323, 0x0302, 0x0065, 0x0323, + 0x0302, 0x0049, 0x0309, 0x0069, 0x0309, 0x0049, 0x0323, 0x0069, 0x0323, 0x004f, 0x0323, 0x006f, + 0x0323, 0x004f, 0x0309, 0x006f, 0x0309, 0x004f, 0x0302, 0x0301, 0x006f, 0x0302, 0x0301, 0x004f, + 0x0302, 0x0300, 0x006f, 0x0302, 0x0300, 0x004f, 0x0302, 0x0309, 0x006f, 0x0302, 0x0309, 0x004f, + 0x0302, 0x0303, 0x006f, 0x0302, 0x0303, 0x004f, 0x0323, 0x0302, 0x006f, 0x0323, 0x0302, 0x004f, + 0x031b, 0x0301, 0x006f, 0x031b, 0x0301, 0x004f, 0x031b, 0x0300, 0x006f, 0x031b, 0x0300, 0x004f, + 0x031b, 0x0309, 0x006f, 0x031b, 0x0309, 0x004f, 0x031b, 0x0303, 0x006f, 0x031b, 0x0303, 0x004f, + 0x031b, 0x0323, 0x006f, 0x031b, 0x0323, 0x0055, 0x0323, 0x0075, 0x0323, 0x0055, 0x0309, 0x0075, + 0x0309, 0x0055, 0x031b, 0x0301, 0x0075, 0x031b, 0x0301, 0x0055, 0x031b, 0x0300, 0x0075, 0x031b, + 0x0300, 0x0055, 0x031b, 0x0309, 0x0075, 0x031b, 0x0309, 0x0055, 0x031b, 0x0303, 0x0075, 0x031b, + 0x0303, 0x0055, 0x031b, 0x0323, 0x0075, 0x031b, 0x0323, 0x0059, 0x0300, 0x0079, 0x0300, 0x0059, + 0x0323, 0x0079, 0x0323, 0x0059, 0x0309, 0x0079, 0x0309, 0x0059, 0x0303, 0x0079, 0x0303, 0x03b1, + 0x0313, 0x03b1, 0x0314, 0x03b1, 0x0313, 0x0300, 0x03b1, 0x0314, 0x0300, 0x03b1, 0x0313, 0x0301, + 0x03b1, 0x0314, 0x0301, 0x03b1, 0x0313, 0x0342, 0x03b1, 0x0314, 0x0342, 0x0391, 0x0313, 0x0391, + 0x0314, 0x0391, 0x0313, 0x0300, 0x0391, 0x0314, 0x0300, 0x0391, 0x0313, 0x0301, 0x0391, 0x0314, + 0x0301, 0x0391, 0x0313, 0x0342, 0x0391, 0x0314, 0x0342, 0x03b5, 0x0313, 0x03b5, 0x0314, 0x03b5, + 0x0313, 0x0300, 0x03b5, 0x0314, 0x0300, 0x03b5, 0x0313, 0x0301, 0x03b5, 0x0314, 0x0301, 0x0395, + 0x0313, 0x0395, 0x0314, 0x0395, 0x0313, 0x0300, 0x0395, 0x0314, 0x0300, 0x0395, 0x0313, 0x0301, + 0x0395, 0x0314, 0x0301, 0x03b7, 0x0313, 0x03b7, 0x0314, 0x03b7, 0x0313, 0x0300, 0x03b7, 0x0314, + 0x0300, 0x03b7, 0x0313, 0x0301, 0x03b7, 0x0314, 0x0301, 0x03b7, 0x0313, 0x0342, 0x03b7, 0x0314, + 0x0342, 0x0397, 0x0313, 0x0397, 0x0314, 0x0397, 0x0313, 0x0300, 0x0397, 0x0314, 0x0300, 0x0397, + 0x0313, 0x0301, 0x0397, 0x0314, 0x0301, 0x0397, 0x0313, 0x0342, 0x0397, 0x0314, 0x0342, 0x03b9, + 0x0313, 0x03b9, 0x0314, 0x03b9, 0x0313, 0x0300, 0x03b9, 0x0314, 0x0300, 0x03b9, 0x0313, 0x0301, + 0x03b9, 0x0314, 0x0301, 0x03b9, 0x0313, 0x0342, 0x03b9, 0x0314, 0x0342, 0x0399, 0x0313, 0x0399, + 0x0314, 0x0399, 0x0313, 0x0300, 0x0399, 0x0314, 0x0300, 0x0399, 0x0313, 0x0301, 0x0399, 0x0314, + 0x0301, 0x0399, 0x0313, 0x0342, 0x0399, 0x0314, 0x0342, 0x03bf, 0x0313, 0x03bf, 0x0314, 0x03bf, + 0x0313, 0x0300, 0x03bf, 0x0314, 0x0300, 0x03bf, 0x0313, 0x0301, 0x03bf, 0x0314, 0x0301, 0x039f, + 0x0313, 0x039f, 0x0314, 0x039f, 0x0313, 0x0300, 0x039f, 0x0314, 0x0300, 0x039f, 0x0313, 0x0301, + 0x039f, 0x0314, 0x0301, 0x03c5, 0x0313, 0x03c5, 0x0314, 0x03c5, 0x0313, 0x0300, 0x03c5, 0x0314, + 0x0300, 0x03c5, 0x0313, 0x0301, 0x03c5, 0x0314, 0x0301, 0x03c5, 0x0313, 0x0342, 0x03c5, 0x0314, + 0x0342, 0x03a5, 0x0314, 0x03a5, 0x0314, 0x0300, 0x03a5, 0x0314, 0x0301, 0x03a5, 0x0314, 0x0342, + 0x03c9, 0x0313, 0x03c9, 0x0314, 0x03c9, 0x0313, 0x0300, 0x03c9, 0x0314, 0x0300, 0x03c9, 0x0313, + 0x0301, 0x03c9, 0x0314, 0x0301, 0x03c9, 0x0313, 0x0342, 0x03c9, 0x0314, 0x0342, 0x03a9, 0x0313, + 0x03a9, 0x0314, 0x03a9, 0x0313, 0x0300, 0x03a9, 0x0314, 0x0300, 0x03a9, 0x0313, 0x0301, 0x03a9, + 0x0314, 0x0301, 0x03a9, 0x0313, 0x0342, 0x03a9, 0x0314, 0x0342, 0x03b1, 0x0300, 0x03b1, 0x0301, + 0x03b5, 0x0300, 0x03b5, 0x0301, 0x03b7, 0x0300, 0x03b7, 0x0301, 0x03b9, 0x0300, 0x03b9, 0x0301, + 0x03bf, 0x0300, 0x03bf, 0x0301, 0x03c5, 0x0300, 0x03c5, 0x0301, 0x03c9, 0x0300, 0x03c9, 0x0301, + 0x03b1, 0x0345, 0x0313, 0x03b1, 0x0345, 0x0314, 0x03b1, 0x0345, 0x0313, 0x0300, 0x03b1, 0x0345, + 0x0314, 0x0300, 0x03b1, 0x0345, 0x0313, 0x0301, 0x03b1, 0x0345, 0x0314, 0x0301, 0x03b1, 0x0345, + 0x0313, 0x0342, 0x03b1, 0x0345, 0x0314, 0x0342, 0x0391, 0x0345, 0x0313, 0x0391, 0x0345, 0x0314, + 0x0391, 0x0345, 0x0313, 0x0300, 0x0391, 0x0345, 0x0314, 0x0300, 0x0391, 0x0345, 0x0313, 0x0301, + 0x0391, 0x0345, 0x0314, 0x0301, 0x0391, 0x0345, 0x0313, 0x0342, 0x0391, 0x0345, 0x0314, 0x0342, + 0x03b7, 0x0345, 0x0313, 0x03b7, 0x0345, 0x0314, 0x03b7, 0x0345, 0x0313, 0x0300, 0x03b7, 0x0345, + 0x0314, 0x0300, 0x03b7, 0x0345, 0x0313, 0x0301, 0x03b7, 0x0345, 0x0314, 0x0301, 0x03b7, 0x0345, + 0x0313, 0x0342, 0x03b7, 0x0345, 0x0314, 0x0342, 0x0397, 0x0345, 0x0313, 0x0397, 0x0345, 0x0314, + 0x0397, 0x0345, 0x0313, 0x0300, 0x0397, 0x0345, 0x0314, 0x0300, 0x0397, 0x0345, 0x0313, 0x0301, + 0x0397, 0x0345, 0x0314, 0x0301, 0x0397, 0x0345, 0x0313, 0x0342, 0x0397, 0x0345, 0x0314, 0x0342, + 0x03c9, 0x0345, 0x0313, 0x03c9, 0x0345, 0x0314, 0x03c9, 0x0345, 0x0313, 0x0300, 0x03c9, 0x0345, + 0x0314, 0x0300, 0x03c9, 0x0345, 0x0313, 0x0301, 0x03c9, 0x0345, 0x0314, 0x0301, 0x03c9, 0x0345, + 0x0313, 0x0342, 0x03c9, 0x0345, 0x0314, 0x0342, 0x03a9, 0x0345, 0x0313, 0x03a9, 0x0345, 0x0314, + 0x03a9, 0x0345, 0x0313, 0x0300, 0x03a9, 0x0345, 0x0314, 0x0300, 0x03a9, 0x0345, 0x0313, 0x0301, + 0x03a9, 0x0345, 0x0314, 0x0301, 0x03a9, 0x0345, 0x0313, 0x0342, 0x03a9, 0x0345, 0x0314, 0x0342, + 0x03b1, 0x0306, 0x03b1, 0x0304, 0x03b1, 0x0345, 0x0300, 0x03b1, 0x0345, 0x03b1, 0x0345, 0x0301, + 0x03b1, 0x0342, 0x03b1, 0x0345, 0x0342, 0x0391, 0x0306, 0x0391, 0x0304, 0x0391, 0x0300, 0x0391, + 0x0301, 0x0391, 0x0345, 0x03b9, 0x00a8, 0x0342, 0x03b7, 0x0345, 0x0300, 0x03b7, 0x0345, 0x03b7, + 0x0345, 0x0301, 0x03b7, 0x0342, 0x03b7, 0x0345, 0x0342, 0x0395, 0x0300, 0x0395, 0x0301, 0x0397, + 0x0300, 0x0397, 0x0301, 0x0397, 0x0345, 0x1fbf, 0x0300, 0x1fbf, 0x0301, 0x1fbf, 0x0342, 0x03b9, + 0x0306, 0x03b9, 0x0304, 0x03b9, 0x0308, 0x0300, 0x03b9, 0x0308, 0x0301, 0x03b9, 0x0342, 0x03b9, + 0x0308, 0x0342, 0x0399, 0x0306, 0x0399, 0x0304, 0x0399, 0x0300, 0x0399, 0x0301, 0x1ffe, 0x0300, + 0x1ffe, 0x0301, 0x1ffe, 0x0342, 0x03c5, 0x0306, 0x03c5, 0x0304, 0x03c5, 0x0308, 0x0300, 0x03c5, + 0x0308, 0x0301, 0x03c1, 0x0313, 0x03c1, 0x0314, 0x03c5, 0x0342, 0x03c5, 0x0308, 0x0342, 0x03a5, + 0x0306, 0x03a5, 0x0304, 0x03a5, 0x0300, 0x03a5, 0x0301, 0x03a1, 0x0314, 0x00a8, 0x0300, 0x00a8, + 0x0301, 0x0060, 0x03c9, 0x0345, 0x0300, 0x03c9, 0x0345, 0x03bf, 0x0345, 0x0301, 0x03c9, 0x0342, + 0x03c9, 0x0345, 0x0342, 0x039f, 0x0300, 0x039f, 0x0301, 0x03a9, 0x0300, 0x03a9, 0x0301, 0x03a9, + 0x0345, 0x00b4, 0x304b, 0x3099, 0x304d, 0x3099, 0x304f, 0x3099, 0x3051, 0x3099, 0x3053, 0x3099, + 0x3055, 0x3099, 0x3057, 0x3099, 0x3059, 0x3099, 0x305b, 0x3099, 0x305d, 0x3099, 0x305f, 0x3099, + 0x3061, 0x3099, 0x3064, 0x3099, 0x3066, 0x3099, 0x3068, 0x3099, 0x306f, 0x3099, 0x306f, 0x309a, + 0x3072, 0x3099, 0x3072, 0x309a, 0x3075, 0x3099, 0x3075, 0x309a, 0x3078, 0x3099, 0x3078, 0x309a, + 0x307b, 0x3099, 0x307b, 0x309a, 0x3046, 0x3099, 0x309d, 0x3099, 0x30ab, 0x3099, 0x30ad, 0x3099, + 0x30af, 0x3099, 0x30b1, 0x3099, 0x30b3, 0x3099, 0x30b5, 0x3099, 0x30b7, 0x3099, 0x30b9, 0x3099, + 0x30bb, 0x3099, 0x30bd, 0x3099, 0x30bf, 0x3099, 0x30c1, 0x3099, 0x30c4, 0x3099, 0x30c6, 0x3099, + 0x30c8, 0x3099, 0x30cf, 0x3099, 0x30cf, 0x309a, 0x30d2, 0x3099, 0x30d2, 0x309a, 0x30d5, 0x3099, + 0x30d5, 0x309a, 0x30d8, 0x3099, 0x30d8, 0x309a, 0x30db, 0x3099, 0x30db, 0x309a, 0x30a6, 0x3099, + 0x30ef, 0x3099, 0x30f0, 0x3099, 0x30f1, 0x3099, 0x30f2, 0x3099, 0x30fd, 0x3099, 0x05f2, 0x05b7, + 0x05e9, 0x05c1, 0x05e9, 0x05c2, 0x05e9, 0x05bc, 0x05c1, 0x05e9, 0x05bc, 0x05c2, 0x05d0, 0x05b7, + 0x05d0, 0x05b8, 0x05d0, 0x05bc, 0x05d1, 0x05bc, 0x05d2, 0x05bc, 0x05d3, 0x05bc, 0x05d4, 0x05bc, + 0x05d5, 0x05bc, 0x05d6, 0x05bc, 0x05d8, 0x05bc, 0x05d9, 0x05bc, 0x05da, 0x05bc, 0x05db, 0x05bc, + 0x05dc, 0x05bc, 0x05de, 0x05bc, 0x05e0, 0x05bc, 0x05e1, 0x05bc, 0x05e3, 0x05bc, 0x05e4, 0x05bc, + 0x05e6, 0x05bc, 0x05e7, 0x05bc, 0x05e8, 0x05bc, 0x05e9, 0x05bc, 0x05ea, 0x05bc, 0x05d5, 0x05b9, + 0x05d1, 0x05bf, 0x05db, 0x05bf, 0x05e4, 0x05bf, +]; + +/// Case-fold one UTF-16 unit (Apple `gLowerCaseTable`). Returns 0 for an +/// "ignored" code point (the caller drops it); NUL folds to 0xFFFF. +pub fn case_fold(c: u16) -> u16 { + let hi = CASE_FOLD[(c >> 8) as usize]; + if hi != 0 { + CASE_FOLD[(hi + (c & 0xff)) as usize] + } else { + c + } +} + +const HANGUL_S_BASE: u32 = 0xac00; +const HANGUL_L_BASE: u16 = 0x1100; +const HANGUL_V_BASE: u16 = 0x1161; +const HANGUL_T_BASE: u16 = 0x11a7; +const HANGUL_S_COUNT: u32 = 11172; +const HANGUL_T_COUNT: u32 = 28; +const HANGUL_N_COUNT: u32 = 21 * 28; + +/// Algorithmic Hangul syllable decomposition (Unicode/TN1150). Writes 2-3 jamo +/// into `out` and returns the count, or 0 when `uc` is not a Hangul syllable. +fn decompose_hangul(uc: u16, out: &mut [u16; 3]) -> usize { + let index = uc as u32; + if !(HANGUL_S_BASE..HANGUL_S_BASE + HANGUL_S_COUNT).contains(&index) { + return 0; + } + let index = index - HANGUL_S_BASE; + out[0] = HANGUL_L_BASE + (index / HANGUL_N_COUNT) as u16; + out[1] = HANGUL_V_BASE + ((index % HANGUL_N_COUNT) / HANGUL_T_COUNT) as u16; + let t = HANGUL_T_BASE + (index % HANGUL_T_COUNT) as u16; + if t != HANGUL_T_BASE { + out[2] = t; + 3 + } else { + 2 + } +} + +/// Table decomposition for a non-Hangul unit. Returns the static decomposed +/// sequence, or `None` if the unit has no canonical decomposition. +fn decompose_nonhangul(uc: u16) -> Option<&'static [u16]> { + let mut off = DECOMPOSE[((uc >> 12) & 0xf) as usize]; + if off == 0 || off == 0xffff { + return None; + } + off = DECOMPOSE[(off + ((uc >> 8) & 0xf)) as usize]; + if off == 0 { + return None; + } + off = DECOMPOSE[(off + ((uc >> 4) & 0xf)) as usize]; + if off == 0 { + return None; + } + off = DECOMPOSE[(off + (uc & 0xf)) as usize]; + let size = (off & 3) as usize; + if size == 0 { + return None; + } + let start = (off / 4) as usize; + Some(&DECOMPOSE[start..start + size]) +} + +/// Canonically decompose one UTF-16 unit into `buf`, returning the decomposed +/// slice (1-3 units). Units with no decomposition return themselves. +pub fn decompose(uc: u16, buf: &mut [u16; 3]) -> &[u16] { + let n = decompose_hangul(uc, buf); + if n != 0 { + return &buf[..n]; + } + if let Some(seq) = decompose_nonhangul(uc) { + buf[..seq.len()].copy_from_slice(seq); + return &buf[..seq.len()]; + } + buf[0] = uc; + &buf[..1] +} + +/// Encode a name as the canonical HFS+ on-disk form: UTF-16 units, each +/// canonically decomposed per Apple's TN1150 table. This is how classic Mac OS +/// stores HFS+ names; using it (rather than Rust's NFD) keeps the bytes +/// byte-identical to what the OS would write, so lookups by either side match. +pub fn decompose_str(name: &str) -> Vec { + let mut out = Vec::with_capacity(name.len()); + let mut buf = [0u16; 3]; + let mut unit_buf = [0u16; 2]; + for ch in name.chars() { + for &unit in ch.encode_utf16(&mut unit_buf).iter() { + out.extend_from_slice(decompose(unit, &mut buf)); + } + } + out +} diff --git a/src/fs/hfsplus.rs b/src/fs/hfsplus.rs index 47dae992..56bb39e0 100644 --- a/src/fs/hfsplus.rs +++ b/src/fs/hfsplus.rs @@ -2,7 +2,6 @@ use byteorder::{BigEndian, ByteOrder}; use std::cmp::Ordering; use std::collections::{HashMap, HashSet}; use std::io::{Read, Seek, SeekFrom, Write}; -use unicode_normalization::UnicodeNormalization; use super::entry::{EntryType, FileEntry}; use super::filesystem::{ @@ -10,8 +9,7 @@ use super::filesystem::{ }; use super::hfs_common::{ self, bitmap_clear_bit_be, bitmap_collect_clear_runs_be, bitmap_set_bit_be, btree_free_node, - btree_grow_root, btree_insert_into_index, btree_insert_record, btree_remove_record, - btree_split_leaf_with_insert, BTreeHeader, + btree_remove_record, BTreeHeader, BTreeKeyFormat, }; use super::CompactResult; @@ -374,8 +372,7 @@ fn validate_hfsplus_create_name(name: &str) -> Result<(), FilesystemError> { .into(), )); } - let nfd: String = name.nfd().collect(); - let utf16_len = nfd.encode_utf16().count(); + let utf16_len = super::hfs_unicode::decompose_str(name).len(); if utf16_len > 255 { return Err(FilesystemError::InvalidData(format!( "filename is too long ({utf16_len} UTF-16 units); HFS+ allows up to 255 — \ @@ -1666,8 +1663,7 @@ impl HfsPlusFilesystem { /// Build HFS+ catalog key bytes: key_len(2) + parent_id(4) + name_length(2) + name(UTF-16BE NFD). pub(crate) fn build_catalog_key(parent_cnid: u32, name: &str) -> Vec { - let nfd: String = name.nfd().collect(); - let utf16: Vec = nfd.encode_utf16().collect(); + let utf16: Vec = super::hfs_unicode::decompose_str(name); let key_len = 4 + 2 + utf16.len() * 2; let mut key = Vec::with_capacity(2 + key_len); let mut buf = [0u8; 2]; @@ -1807,112 +1803,26 @@ impl HfsPlusFilesystem { } /// Insert a catalog record into the B-tree, handling splits and growth. + /// + /// Delegates to the shared `btree_insert_full`, which — for a non-sequential + /// (per-`put`) insert into a full leaf — first tries a B*-style rotation into + /// an adjacent sibling before splitting, lifting random-insert leaf occupancy + /// from the ~69% a plain split leaves to ~88% (docs/hfsplus_btree_growth_plan.md + /// §4d). `BTreeKeyFormat::HFSPLUS_CATALOG` drives the 2-byte / variable-length + /// index keys. This is the same path classic HFS's `insert_catalog_record` + /// uses; the previous hand-rolled split dance here packed less densely because + /// it never rotated. fn insert_catalog_record(&mut self, key_record: &[u8]) -> Result<(), FilesystemError> { - let header = BTreeHeader::read(&self.catalog_data); - let node_size = header.node_size as usize; let cs = self.case_sensitive(); let cmp = |a: &[u8], b: &[u8]| Self::catalog_compare(a, b, cs); - - // Find the correct leaf node - let (leaf_idx, parent_chain) = - hfs_common::btree_find_insert_leaf(&self.catalog_data, &header, key_record, &cmp); - - // Try to insert into the leaf - let offset = leaf_idx as usize * node_size; - let node = &mut self.catalog_data[offset..offset + node_size]; - match btree_insert_record(node, node_size, key_record, &cmp) { - Ok(_) => { - // Update header leaf_records - let mut h = BTreeHeader::read(&self.catalog_data); - h.leaf_records += 1; - h.write(&mut self.catalog_data); - self.catalog_header.leaf_records = h.leaf_records; - Ok(()) - } - Err(_) => { - // Leaf full — atomic split+insert. Byte-based split point - // tolerates uneven record sizes; avoids the - // split-then-insert failure mode where the target half - // can't accept the new record. - let mut h = BTreeHeader::read(&self.catalog_data); - let splits = btree_split_leaf_with_insert( - &mut self.catalog_data, - node_size, - leaf_idx, - &mut h, - key_record, - &cmp, - )?; - - h.leaf_records += 1; - - // Install the first separator into the existing parent (or - // grow the root). Any additional separators come from N-way - // splits and are routed through `btree_insert_into_index` - // against a freshly-resolved parent — once the tree depth - // grows past 1 the root-grow branch never re-fires. - let first = &splits[0]; - if h.depth == 1 { - btree_grow_root( - &mut self.catalog_data, - node_size, - &mut h, - leaf_idx, - first.0, - &first.1, - )?; - } else if let Some(&(_, parent_idx)) = - parent_chain.iter().find(|&&(nidx, _)| nidx == leaf_idx) - { - btree_insert_into_index( - &mut self.catalog_data, - node_size, - parent_idx, - first.0, - &first.1, - &mut h, - &cmp, - &parent_chain, - )?; - } else { - btree_grow_root( - &mut self.catalog_data, - node_size, - &mut h, - leaf_idx, - first.0, - &first.1, - )?; - } - for split in &splits[1..] { - let header_now = BTreeHeader::read(&self.catalog_data); - let (_, fresh_chain) = hfs_common::btree_find_insert_leaf( - &self.catalog_data, - &header_now, - &split.1, - &cmp, - ); - let parent_for_sep = fresh_chain - .last() - .map(|&(_, p)| p) - .unwrap_or(header_now.root_node); - btree_insert_into_index( - &mut self.catalog_data, - node_size, - parent_for_sep, - split.0, - &split.1, - &mut h, - &cmp, - &fresh_chain, - )?; - } - - h.write(&mut self.catalog_data); - self.catalog_header = BTreeHeaderRecord::parse(&self.catalog_data[14..14 + 106]); - Ok(()) - } - } + hfs_common::btree_insert_full( + &mut self.catalog_data, + key_record, + &BTreeKeyFormat::HFSPLUS_CATALOG, + &cmp, + )?; + self.catalog_header = BTreeHeaderRecord::parse(&self.catalog_data[14..14 + 106]); + Ok(()) } /// Remove a catalog record by (node_idx, rec_idx). @@ -2008,38 +1918,33 @@ impl HfsPlusFilesystem { } impl HfsPlusFilesystem { - /// Insert a record into the extents-overflow B-tree leaf chain. Depth-1 - /// only — splits and root growth are not yet wired up because the - /// existing HFS+ B-tree growth helpers normalize keys assuming the - /// 1-byte HFS-classic format. A full split path lands when modern HFS+ - /// volumes need it (Phase 5+ work). + /// Insert a record into the extents-overflow B-tree, splitting the leaf and + /// growing the root when the target leaf is full — mirroring + /// `insert_catalog_record` / `insert_xattr_record`. + /// + /// The extents-overflow tree uses 2-byte ("big") keys with *fixed*-length + /// index separators (`kBTBigKeysMask` set, `kBTVariableIndexKeysMask` clear), + /// padded to the 10-byte `maxKeyLength`. `BTreeKeyFormat::HFSPLUS_EXTENTS` + /// drives that; before P1's key-format descriptor the shared growth helpers + /// forced the classic 1-byte/0x25 shape, which is why this path was pinned to + /// a single leaf level (docs/hfsplus_btree_growth_plan.md §4a/P3). fn insert_extents_overflow_record(&mut self, key_record: &[u8]) -> Result<(), FilesystemError> { let data = self.extents_overflow_data.as_mut().ok_or_else(|| { FilesystemError::Unsupported( "volume has no extents-overflow B-tree — fragmented files cannot be created".into(), ) })?; - let header = BTreeHeader::read(data); - let node_size = header.node_size as usize; - if node_size == 0 { + if BTreeHeader::read(data).node_size == 0 { return Err(FilesystemError::InvalidData( "extents-overflow B-tree has zero node_size".into(), )); } - let (leaf_idx, _chain) = - hfs_common::btree_find_insert_leaf(data, &header, key_record, &Self::extents_compare); - let offset = leaf_idx as usize * node_size; - let node = &mut data[offset..offset + node_size]; - btree_insert_record(node, node_size, key_record, &Self::extents_compare).map_err(|e| { - FilesystemError::InvalidData(format!( - "extents-overflow leaf {leaf_idx} cannot fit new record: {e} \ - (split-on-overflow not yet implemented for HFS+ extents-overflow)" - )) - })?; - let mut h = BTreeHeader::read(data); - h.leaf_records += 1; - h.write(data); - Ok(()) + hfs_common::btree_insert_full( + data, + key_record, + &BTreeKeyFormat::HFSPLUS_EXTENTS, + &Self::extents_compare, + ) } /// Remove every overflow record belonging to `(file_id, fork_type)`. @@ -2139,8 +2044,7 @@ impl HfsPlusFilesystem { /// Caller is expected to have already NFD-normalized `name` if needed; we /// re-normalize defensively so on-disk keys stay canonical. pub(crate) fn build_inline_attr_record(cnid: u32, name: &str, value: &[u8]) -> Vec { - let nfd: String = name.nfd().collect(); - let utf16: Vec = nfd.encode_utf16().collect(); + let utf16: Vec = super::hfs_unicode::decompose_str(name); let key_body_len = 12 + utf16.len() * 2; let data_section_len = 4 + 8 + 4 + value.len(); let mut rec = vec![0u8; 2 + key_body_len + data_section_len]; @@ -2194,74 +2098,15 @@ impl HfsPlusFilesystem { node_size, ))); } - let (leaf_idx, parent_chain) = - hfs_common::btree_find_insert_leaf(data, &header, key_record, &Self::attr_compare); - let off = leaf_idx as usize * node_size; - let node = &mut data[off..off + node_size]; - match btree_insert_record(node, node_size, key_record, &Self::attr_compare) { - Ok(_) => { - let mut h = BTreeHeader::read(data); - h.leaf_records += 1; - h.write(data); - Ok(()) - } - Err(_) => { - let mut h = BTreeHeader::read(data); - let splits = btree_split_leaf_with_insert( - data, - node_size, - leaf_idx, - &mut h, - key_record, - &Self::attr_compare, - )?; - h.leaf_records += 1; - let first = &splits[0]; - if h.depth == 1 { - btree_grow_root(data, node_size, &mut h, leaf_idx, first.0, &first.1)?; - } else if let Some(&(_, parent_idx)) = - parent_chain.iter().find(|&&(nidx, _)| nidx == leaf_idx) - { - btree_insert_into_index( - data, - node_size, - parent_idx, - first.0, - &first.1, - &mut h, - &Self::attr_compare, - &parent_chain, - )?; - } else { - btree_grow_root(data, node_size, &mut h, leaf_idx, first.0, &first.1)?; - } - for split in &splits[1..] { - let header_now = BTreeHeader::read(data); - let (_, fresh_chain) = hfs_common::btree_find_insert_leaf( - data, - &header_now, - &split.1, - &Self::attr_compare, - ); - let parent_for_sep = fresh_chain - .last() - .map(|&(_, p)| p) - .unwrap_or(header_now.root_node); - btree_insert_into_index( - data, - node_size, - parent_for_sep, - split.0, - &split.1, - &mut h, - &Self::attr_compare, - &fresh_chain, - )?; - } - h.write(data); - Ok(()) - } - } + // The attributes B-tree, like the catalog, uses 2-byte big keys and + // variable-length index separators, so it threads HFSPLUS_ATTRIBUTES + // through the shared insert (which also rotates before splitting). + hfs_common::btree_insert_full( + data, + key_record, + &BTreeKeyFormat::HFSPLUS_ATTRIBUTES, + &Self::attr_compare, + ) } /// Remove every attribute record matching `(cnid, name)`. Returns the @@ -2287,10 +2132,7 @@ impl HfsPlusFilesystem { if node_size == 0 { return Ok(0); } - let target_utf16: Option> = name.map(|n| { - let nfd: String = n.nfd().collect(); - nfd.encode_utf16().collect() - }); + let target_utf16: Option> = name.map(super::hfs_unicode::decompose_str); let mut victims: Vec<(u32, usize)> = Vec::new(); hfs_common::walk_leaf_records::<(), _>( @@ -2704,8 +2546,7 @@ impl HfsPlusFilesystem { // Let me re-read: Thread key: key_len(2) + parent_id(4, =CNID) + name_len(2, =0) // Record data: type(2) + reserved(2) + parentID(4) + name_len(2) + name(UTF-16BE) - let nfd: String = name.nfd().collect(); - let utf16: Vec = nfd.encode_utf16().collect(); + let utf16: Vec = super::hfs_unicode::decompose_str(name); let mut rec = Vec::with_capacity(10 + utf16.len() * 2); let mut buf2 = [0u8; 2]; let mut buf4 = [0u8; 4]; @@ -5159,8 +5000,41 @@ pub fn create_blank_hfsplus( name: &str, case_sensitive: bool, ) -> Vec { - let (front, vh_bytes, image_size) = - build_blank_hfsplus_front(size_bytes, block_size, name, case_sensitive); + create_blank_hfsplus_sized(size_bytes, block_size, name, case_sensitive, 0, 0) +} + +/// Volume-scaled default catalog B-tree size, in bytes, for a fresh HFS+ +/// volume. Mirrors classic HFS's `default_btree_sizes`: ~0.5% of the volume so +/// a blank holds thousands of records before exhausting its node budget (HFS+ +/// has no grow-on-full path — docs/hfsplus_btree_growth_plan.md §4c). The +/// `build_blank_hfsplus_front` caller clamps the result to whole nodes in +/// `[4, header-bitmap capacity]`. +pub fn default_hfsplus_catalog_bytes(volume_bytes: u64) -> u64 { + volume_bytes / 200 +} + +/// Variant of [`create_blank_hfsplus`] that lets the caller request minimum +/// catalog and extents-overflow B-tree sizes (in bytes) — e.g. a clone target +/// that must hold every record from a fragmented source, or a test that needs a +/// deep catalog without a multi-GiB volume. Each minimum is rounded up to whole +/// nodes and raised to the 4-node floor; the catalog also never drops below the +/// volume-scaled [`default_hfsplus_catalog_bytes`]. +pub fn create_blank_hfsplus_sized( + size_bytes: u64, + block_size: u32, + name: &str, + case_sensitive: bool, + min_catalog_bytes: u32, + min_extents_bytes: u32, +) -> Vec { + let (front, vh_bytes, image_size) = build_blank_hfsplus_front( + size_bytes, + block_size, + name, + case_sensitive, + min_catalog_bytes, + min_extents_bytes, + ); let mut img = vec![0u8; image_size]; img[..front.len()].copy_from_slice(&front); let alt = image_size - 1024; @@ -5177,6 +5051,8 @@ fn build_blank_hfsplus_front( block_size: u32, name: &str, case_sensitive: bool, + min_catalog_bytes: u32, + min_extents_bytes: u32, ) -> (Vec, [u8; 512], usize) { assert!( block_size.is_power_of_two() && (512..=4096).contains(&block_size), @@ -5193,16 +5069,42 @@ fn build_blank_hfsplus_front( let blocks_per_node: u32 = (block_size / node_size as u32).max(1); let _ = nodes_per_block; - let btree_node_count: u32 = 4; - let btree_blocks: u32 = btree_node_count * blocks_per_node; + // Size the catalog B-tree from the volume (docs/hfsplus_btree_growth_plan.md + // §4c): HFS+ has no grow-on-full path, so a blank that reserved only the + // legacy 4 nodes exhausted after ~24 files. Scale it to ~0.5% of the volume + // (mirroring classic HFS's `default_btree_sizes`) so a fresh volume holds + // thousands of records, honouring an explicit `min_catalog_bytes` floor from + // sized/clone callers. The extents-overflow tree keeps the 4-node default + // unless a caller asks for more (its own scaling is P3 work). + // + // `to_nodes` rounds a byte budget up to whole nodes and clamps to + // [4, header-bitmap capacity]. The blank's node-allocation bitmap lives + // entirely in the header node's record 2 (bytes 270..node_size-8 — see + // `write_blank_btree_header_node`), which addresses `(node_size - 278) * 8` + // nodes without dedicated map nodes; cap there. Volumes whose 0.5% catalog + // would exceed this (~117 MiB at 4096, i.e. hundreds of GiB of volume) are + // far larger than this tool targets, and the defrag/clone path — which does + // build map nodes — covers them. + let max_btree_nodes: u32 = ((node_size - 278) * 8) as u32; + let to_nodes = |bytes: u64| -> u32 { + bytes + .div_ceil(node_size as u64) + .clamp(4, max_btree_nodes as u64) as u32 + }; + let catalog_node_count = + to_nodes(default_hfsplus_catalog_bytes(size_bytes).max(min_catalog_bytes as u64)); + let extents_node_count = to_nodes(min_extents_bytes as u64); + + let catalog_btree_blocks: u32 = catalog_node_count * blocks_per_node; + let extents_btree_blocks: u32 = extents_node_count * blocks_per_node; let bitmap_bytes = total_blocks.div_ceil(8) as u64; let bitmap_blocks = bitmap_bytes.div_ceil(bs) as u32; let bitmap_start: u32 = 1; let extents_start: u32 = bitmap_start + bitmap_blocks; - let catalog_start: u32 = extents_start + btree_blocks; - let reserved_blocks: u32 = catalog_start + btree_blocks; + let catalog_start: u32 = extents_start + extents_btree_blocks; + let reserved_blocks: u32 = catalog_start + catalog_btree_blocks; assert!( reserved_blocks + 1 < total_blocks, "image too small for reserved region: need {} blocks, have {total_blocks}", @@ -5236,8 +5138,8 @@ fn build_blank_hfsplus_front( &mut front[off..off + node_size], node_size, /* leaf_records= */ 0, - /* total_nodes= */ btree_node_count, - /* free_nodes= */ btree_node_count - 2, // header + empty leaf used + /* total_nodes= */ extents_node_count, + /* free_nodes= */ extents_node_count - 2, // header + empty leaf used /* max_key_len= */ 10, /* key_compare_type= */ 0, // unused for extents ); @@ -5246,8 +5148,7 @@ fn build_blank_hfsplus_front( } // --- Catalog B-tree (header + leaf with root + thread) --- - let nfd_name: String = name.nfd().collect(); - let name_utf16: Vec = nfd_name.encode_utf16().collect(); + let name_utf16: Vec = super::hfs_unicode::decompose_str(name); { let off = catalog_start as usize * block_size as usize; let key_compare = if case_sensitive { @@ -5259,8 +5160,8 @@ fn build_blank_hfsplus_front( &mut front[off..off + node_size], node_size, /* leaf_records= */ 2, - btree_node_count, - btree_node_count - 2, + catalog_node_count, + catalog_node_count - 2, /* max_key_len= */ 516, // HFS+ catalog max key key_compare, ); @@ -5309,16 +5210,16 @@ fn build_blank_hfsplus_front( extents: extent_array(bitmap_start, bitmap_blocks), }, extents_file: ForkData { - logical_size: btree_blocks as u64 * bs, + logical_size: extents_btree_blocks as u64 * bs, clump_size: 0, - total_blocks: btree_blocks, - extents: extent_array(extents_start, btree_blocks), + total_blocks: extents_btree_blocks, + extents: extent_array(extents_start, extents_btree_blocks), }, catalog_file: ForkData { - logical_size: btree_blocks as u64 * bs, + logical_size: catalog_btree_blocks as u64 * bs, clump_size: 0, - total_blocks: btree_blocks, - extents: extent_array(catalog_start, btree_blocks), + total_blocks: catalog_btree_blocks, + extents: extent_array(catalog_start, catalog_btree_blocks), }, attributes_file: ForkData::empty(), startup_file: ForkData::empty(), @@ -5348,7 +5249,7 @@ pub fn write_blank_hfsplus_into( case_sensitive: bool, ) -> std::io::Result<()> { let (front, vh_bytes, _image_size) = - build_blank_hfsplus_front(size_bytes, block_size, name, case_sensitive); + build_blank_hfsplus_front(size_bytes, block_size, name, case_sensitive, 0, 0); target.seek(SeekFrom::Start(0))?; target.write_all(&front)?; target.seek(SeekFrom::Start(size_bytes - 1024))?; @@ -6448,10 +6349,12 @@ mod tests { let entries = fs.list_directory(&root).unwrap(); assert!(entries.is_empty(), "freshly built blank must list empty"); - // 32 MiB / 4096 = 8192 blocks; reserved = 1 (VH) + 1 (bitmap) + 4 - // (extents) + 4 (catalog) = 10. Free = 8182. + // 32 MiB / 4096 = 8192 blocks. The catalog is now volume-scaled + // (~0.5%, docs/hfsplus_btree_growth_plan.md §4c): 32 MiB / 200 = + // 167,772 bytes -> 41 nodes @ 4096. Reserved = 1 (boot/VH) + 1 (bitmap) + // + 4 (extents, default) + 41 (catalog) = 47. Free = 8145. assert_eq!(fs.vh.total_blocks, 8192); - assert_eq!(fs.vh.free_blocks, 8192 - 10); + assert_eq!(fs.vh.free_blocks, 8192 - 47); // Edit-mode prep should succeed (unmounted bit set, no journal). fs.prepare_for_edit().expect("blank must accept edit prep"); } @@ -7990,4 +7893,523 @@ mod tests { assert!(fe.type_code.is_some()); assert_eq!(fe.type_code, Some(*b"TEXT")); } + + #[test] + fn test_hfsplus_catalog_variable_index_keys_grow_multilevel() { + // P1 (docs/hfsplus_btree_growth_plan.md §4a): the HFS+ catalog B-tree + // must grow past a single index level without corrupting. The shared + // split/grow helpers used to force the classic 1-byte-length, fixed-0x25 + // index-key shape onto HFS+ records, so the first leaf split produced a + // malformed separator (it read the high byte of the 2-byte key length as + // the whole length) and descent misrouted — the "leaves must be strictly + // ascending" / mis-routed-descent corruption. With + // `BTreeKeyFormat::HFSPLUS_CATALOG` the separator is the child's + // variable-length 2-byte key, verbatim. + // + // Driven at the catalog-buffer level through the exact shared + // `btree_insert_full` path `insert_catalog_record` uses. node_size is + // 512 (vs the production 4096) so a few thousand multi-parent records + // reach depth >= 3 in a < 1 MiB buffer; the machinery is node-size + // agnostic. The volume-level fsck gate lands in P2 once blank catalogs + // are sized to hold the records (today a blank catalog is only 4 nodes). + use super::hfs_common::{ + btree_find_record, btree_insert_full, BTreeHeader, BTreeKeyFormat, + }; + use byteorder::{BigEndian, ByteOrder}; + type Fs = HfsPlusFilesystem>>; + + let node_size = 512usize; + // A single-node bitmap (512-byte node) covers ~1872 nodes, so stay under + // that — no map nodes needed, matching a real blank-built tree. + let total_nodes = 1800u32; + let mut buf = vec![0u8; total_nodes as usize * node_size]; + write_blank_btree_header_node( + &mut buf[0..node_size], + node_size, + /* leaf_records= */ 0, + total_nodes, + /* free_nodes= */ total_nodes - 2, + /* max_key_len= */ 516, + KEY_COMPARE_CASE_FOLDING, + ); + write_empty_leaf_node(&mut buf[node_size..2 * node_size]); + + let kf = BTreeKeyFormat::HFSPLUS_CATALOG; + let cmp = |a: &[u8], b: &[u8]| Fs::catalog_compare(a, b, false); + + // ~2500 files in shuffled key order (multiplicative hash is a bijection + // mod 2^32 — unique names that land mid-leaf, not appended) sprayed + // across 50 parent directories, so inserts exercise every leaf and + // index split, not just tail appends. + const DIRS: u32 = 50; + const N: u32 = 2500; + let mut inserted: Vec<(u32, String)> = Vec::with_capacity(N as usize); + for i in 0..N { + let hash = i.wrapping_mul(2_654_435_761); + let parent = 16 + (hash % DIRS); // arbitrary distinct parent CNIDs + let name = format!("f{hash:08x}"); + let mut rec = Fs::build_catalog_key(parent, &name); + // Append filler so records have a realistic size (the compare only + // reads the leading key portion; the body is irrelevant here). + rec.extend_from_slice(&[0u8; 40]); + btree_insert_full(&mut buf, &rec, &kf, &cmp) + .unwrap_or_else(|e| panic!("insert #{i} (parent {parent}, {name}): {e}")); + inserted.push((parent, name)); + } + + // The tree must have grown past a single index level. + let header = BTreeHeader::read(&buf); + assert!( + header.depth >= 3, + "catalog did not reach depth >= 3 (got {}); test needs a deeper tree to \ + exercise index-node splits", + header.depth + ); + assert_eq!( + header.leaf_records, N, + "leaf_records header count drifted from inserts" + ); + + // (a) Every leaf key, walked in sibling-chain order, is strictly + // ascending — the direct symptom of a malformed separator was + // out-of-order leaves. + let mut keys: Vec> = Vec::with_capacity(N as usize); + super::hfs_common::walk_leaf_records::<(), _>( + &buf, + header.first_leaf_node, + node_size, + |_n, _r, _o, rec| { + let kl = BigEndian::read_u16(&rec[0..2]) as usize; + keys.push(rec[..(2 + kl).min(rec.len())].to_vec()); + None + }, + ); + assert_eq!( + keys.len(), + N as usize, + "leaf walk recovered {} of {N} records — chain is broken or has cycles", + keys.len() + ); + for w in keys.windows(2) { + assert_eq!( + cmp(&w[0], &w[1]), + std::cmp::Ordering::Less, + "catalog leaves are not strictly ascending — separator/descent corruption" + ); + } + + // (b) Every inserted record is findable by a root-to-leaf descent, i.e. + // every index separator routes correctly at every level. (A nested `fn` + // rather than a closure so it satisfies the `for<'a> Fn(&'a [u8]) -> + // &'a [u8]` bound `btree_find_record` requires.) + fn key_extract(rec: &[u8]) -> &[u8] { + let kl = BigEndian::read_u16(&rec[0..2]) as usize; + &rec[..(2 + kl).min(rec.len())] + } + for (parent, name) in &inserted { + let search = Fs::build_catalog_key(*parent, name); + let (found, _chain) = btree_find_record(&buf, &header, &search, &key_extract, &cmp); + assert!( + found.is_some(), + "record (parent {parent}, {name}) not findable by descent — misrouted index" + ); + } + } + + #[test] + fn test_hfsplus_blank_sized_catalog_20k_inserts_fsck_clean() { + // P2 (docs/hfsplus_btree_growth_plan.md §4c): a fresh HFS+ volume sized + // to hold the records accepts >= 20k incremental catalog inserts across + // many parent dirs and stays fsck-clean — the volume-level gate P1 + // deferred. A blank catalog used to be a fixed 4 nodes (exhausting after + // ~24 files); it is now volume-scaled, and `create_blank_hfsplus_sized` + // lets this test pin a large catalog into a modest 64 MiB image instead + // of needing a multi-GiB volume to scale into. + use super::hfs_common::BTreeHeader; + type Fs = HfsPlusFilesystem>>; + + // 64 MiB volume; catalog pinned to 16 MiB (~4096 nodes @ 4096) — ample + // headroom for 20k file records at the byte-split's ~69% leaf occupancy. + let img = + create_blank_hfsplus_sized(64 * 1024 * 1024, 4096, "PutShuffle", false, 16 << 20, 0); + let mut fs = Fs::open(std::io::Cursor::new(img), 0).unwrap(); + fs.prepare_for_edit().unwrap(); + + // Real directories so threads / valence / folder_count stay fsck-correct. + const DIRS: u32 = 50; + let root = fs.root().unwrap(); + let mut dir_ids = Vec::with_capacity(DIRS as usize); + for d in 0..DIRS { + let de = fs + .create_directory(&root, &format!("dir{d:03}"), &Default::default()) + .unwrap(); + dir_ids.push(de.location as u32); + } + let base_cnid = fs.vh.next_catalog_id; + + // 20k files in shuffled key order (multiplicative hash is a bijection + // mod 2^32 — unique names landing mid-leaf, not appended), sprayed across + // the directories. Insert the catalog records directly: `create_file` + // snapshots the whole catalog per call for rollback, which would make a + // 20k-record build O(N * catalog_size). fsck doesn't require per-file + // thread records (only flags orphaned threads), so — like the classic + // HFS scaling test — we insert just the file records and fix up counts. + let n: u32 = 20_000; + let mut per_dir = vec![0i32; DIRS as usize]; + let empty_fork = ForkData::empty(); + for i in 0..n { + let hash = i.wrapping_mul(2_654_435_761); + let d = (hash % DIRS) as usize; + let name = format!("f{hash:08x}"); + let mut kr = Fs::build_catalog_key(dir_ids[d], &name); + if !kr.len().is_multiple_of(2) { + kr.push(0); + } + kr.extend_from_slice(&Fs::build_file_record( + base_cnid + i, + &empty_fork, + &empty_fork, + &[0u8; 4], + &[0u8; 4], + )); + fs.insert_catalog_record(&kr) + .unwrap_or_else(|e| panic!("insert #{i} into dir{d}: {e}")); + per_dir[d] += 1; + } + // Mirror the bookkeeping the create path maintains so fsck's cross-checks + // (file_count, next_catalog_id, per-dir valence) pass. + fs.vh.file_count += n; + fs.vh.next_catalog_id = base_cnid + n; + for (d, &cnt) in per_dir.iter().enumerate() { + fs.update_parent_valence(dir_ids[d], cnt).unwrap(); + } + + // The catalog must have grown several index levels... + let h = BTreeHeader::read(fs.catalog_data()); + assert!( + h.depth >= 3, + "catalog depth {} < 3 after 20k inserts", + h.depth + ); + // ...without a premature DiskFull (every insert above succeeded). + + // No IndexSiblingLinkBroken / order / count damage at scale. + let result = fs.fsck().unwrap().unwrap(); + assert!( + result.errors.is_empty(), + "fsck found {} errors after 20k inserts: {:?}", + result.errors.len(), + result + .errors + .iter() + .take(8) + .map(|e| &e.message) + .collect::>() + ); + } + + #[test] + fn test_hfsplus_extents_overflow_grows_multilevel() { + // P3 (docs/hfsplus_btree_growth_plan.md): the extents-overflow B-tree — + // 2-byte ("big") keys with *fixed* 10-byte index separators — must split + // past a single leaf level. It was pinned depth-1 because the shared + // helpers forced the classic key shape; it now threads + // `BTreeKeyFormat::HFSPLUS_EXTENTS`. Driven through the same + // `btree_insert_full` machinery `insert_extents_overflow_record` uses. + use super::hfs_common::{ + btree_find_record, btree_insert_full, BTreeHeader, BTreeKeyFormat, + }; + type Fs = HfsPlusFilesystem>>; + + let node_size = 512usize; + let total_nodes = 1800u32; + let mut buf = vec![0u8; total_nodes as usize * node_size]; + write_blank_btree_header_node( + &mut buf[0..node_size], + node_size, + 0, + total_nodes, + total_nodes - 2, + /* max_key_len= */ 10, + /* key_compare_type= */ 0, + ); + write_empty_leaf_node(&mut buf[node_size..2 * node_size]); + + let kf = BTreeKeyFormat::HFSPLUS_EXTENTS; + // 600 overflow records keyed by distinct shuffled fileIDs (the multiplier + // is odd, so it's a bijection mod 2^32 — distinct i give distinct keys + // that land mid-leaf, not appended). + const N: u32 = 600; + let mut ids: Vec = Vec::with_capacity(N as usize); + for i in 0..N { + let file_id = i.wrapping_mul(2_654_435_761); + let rec = Fs::build_extents_overflow_record(file_id, 0, 8, &[]); + btree_insert_full(&mut buf, &rec, &kf, &Fs::extents_compare) + .unwrap_or_else(|e| panic!("extents insert #{i} (file {file_id}): {e}")); + ids.push(file_id); + } + + let header = BTreeHeader::read(&buf); + assert!( + header.depth >= 2, + "extents-overflow tree stayed depth {} — split past a leaf level failed", + header.depth + ); + assert_eq!(header.leaf_records, N); + + let mut recs: Vec> = Vec::with_capacity(N as usize); + super::hfs_common::walk_leaf_records::<(), _>( + &buf, + header.first_leaf_node, + node_size, + |_n, _r, _o, r| { + recs.push(r.to_vec()); + None + }, + ); + assert_eq!(recs.len(), N as usize, "leaf walk lost records"); + for w in recs.windows(2) { + assert_eq!( + Fs::extents_compare(&w[0], &w[1]), + std::cmp::Ordering::Less, + "extents-overflow leaves out of order — bad separator" + ); + } + + // The extents key portion is always the fixed 12 bytes (2-byte len + 10). + fn ext_key(rec: &[u8]) -> &[u8] { + &rec[..12.min(rec.len())] + } + let ext_cmp = |a: &[u8], b: &[u8]| Fs::extents_compare(a, b); + for &file_id in &ids { + let search = Fs::build_extents_overflow_record(file_id, 0, 8, &[]); + let (found, _) = btree_find_record(&buf, &header, &search, &ext_key, &ext_cmp); + assert!( + found.is_some(), + "extents record for file {file_id} not findable" + ); + } + } + + #[test] + fn test_hfsplus_attributes_grows_multilevel() { + // P3: the attributes B-tree — 2-byte keys with *variable*-length index + // separators — splits past one leaf level via + // `BTreeKeyFormat::HFSPLUS_ATTRIBUTES`, the same path `insert_xattr_record` + // uses. (P1 wired the descriptor in; this proves the attribute key layout, + // distinct from the catalog's, routes correctly at every level.) + use super::hfs_common::{ + btree_find_record, btree_insert_full, BTreeHeader, BTreeKeyFormat, + }; + use byteorder::{BigEndian, ByteOrder}; + type Fs = HfsPlusFilesystem>>; + + let node_size = 512usize; + let total_nodes = 1800u32; + let mut buf = vec![0u8; total_nodes as usize * node_size]; + write_blank_btree_header_node( + &mut buf[0..node_size], + node_size, + 0, + total_nodes, + total_nodes - 2, + /* max_key_len= */ 264, + /* key_compare_type= */ 0, + ); + write_empty_leaf_node(&mut buf[node_size..2 * node_size]); + + let kf = BTreeKeyFormat::HFSPLUS_ATTRIBUTES; + const N: u32 = 600; + let value = [0xABu8; 8]; + let mut cnids: Vec = Vec::with_capacity(N as usize); + for i in 0..N { + let cnid = i.wrapping_mul(2_654_435_761); + let rec = Fs::build_inline_attr_record(cnid, "xattr", &value); + btree_insert_full(&mut buf, &rec, &kf, &Fs::attr_compare) + .unwrap_or_else(|e| panic!("attr insert #{i} (cnid {cnid}): {e}")); + cnids.push(cnid); + } + + let header = BTreeHeader::read(&buf); + assert!( + header.depth >= 2, + "attributes tree stayed depth {} — split past a leaf level failed", + header.depth + ); + assert_eq!(header.leaf_records, N); + + let mut recs: Vec> = Vec::with_capacity(N as usize); + super::hfs_common::walk_leaf_records::<(), _>( + &buf, + header.first_leaf_node, + node_size, + |_n, _r, _o, r| { + recs.push(r.to_vec()); + None + }, + ); + assert_eq!(recs.len(), N as usize, "leaf walk lost records"); + for w in recs.windows(2) { + assert_eq!( + Fs::attr_compare(&w[0], &w[1]), + std::cmp::Ordering::Less, + "attributes leaves out of order — bad separator" + ); + } + + fn attr_key(rec: &[u8]) -> &[u8] { + let kl = BigEndian::read_u16(&rec[0..2]) as usize; + &rec[..(2 + kl).min(rec.len())] + } + let attr_cmp = |a: &[u8], b: &[u8]| Fs::attr_compare(a, b); + for &cnid in &cnids { + let search = Fs::build_inline_attr_record(cnid, "xattr", &value); + let (found, _) = btree_find_record(&buf, &header, &search, &attr_key, &attr_cmp); + assert!(found.is_some(), "attr record for cnid {cnid} not findable"); + } + } + + #[test] + fn test_hfsplus_fragmented_file_splits_extents_overflow_btree_real_path() { + // P3 real-path integration: a heavily fragmented file produces enough + // extents-overflow records to split that B-tree past a single leaf level + // through the real `insert_extents_overflow_record`, and the file reads + // back byte-for-byte through the resulting multi-level tree. (Read-back, + // not fsck, because the all-used-then-free-singletons bitmap trick used to + // force fragmentation leaves the bitmap deliberately over-allocated; the + // point here is the overflow B-tree's split + read path, which P3 enabled.) + type Fs = HfsPlusFilesystem>>; + // Extents-overflow tree pinned to 256 KiB (64 nodes @ 4096) so it has room + // to split — the 4-node default could not. + let img = create_blank_hfsplus_sized(32 << 20, 4096, "Frag", false, 0, 256 << 10); + let mut fs = Fs::open(std::io::Cursor::new(img), 0).unwrap(); + fs.prepare_for_edit().unwrap(); + + // Force maximal fragmentation: mark every block used, then free isolated + // single blocks (every other block) in the user region. A file of N blocks + // must then use N one-block extents: 8 inline + (N-8) overflow extents = + // ceil((N-8)/8) overflow records. 520 blocks -> 64 overflow records, well + // past one 4096-byte extents leaf (~52 records), so the tree must split. + fs.ensure_bitmap().unwrap(); + let block_size = fs.vh.block_size as usize; + let n_blocks: u32 = 520; + let first_free = fs.vh.next_allocation; + { + let bitmap = fs.bitmap.as_mut().unwrap(); + for b in bitmap.iter_mut() { + *b = 0xFF; + } + let mut blk = first_free + 1; + for _ in 0..n_blocks { + bitmap_clear_bit_be(bitmap, blk); + blk += 2; // a used block between each free one -> singletons + } + } + fs.vh.free_blocks = n_blocks; + + let payload: Vec = (0..(n_blocks as usize * block_size)) + .map(|i| (i % 251) as u8) + .collect(); + let mut reader = std::io::Cursor::new(payload.clone()); + let root = fs.root().unwrap(); + let entry = fs + .create_file( + &root, + "frag.bin", + &mut reader, + payload.len() as u64, + &CreateFileOptions::default(), + ) + .expect("fragmented file create should succeed via a splitting overflow B-tree"); + + // The extents-overflow tree must have grown past a single leaf level. + let depth = + super::hfs_common::BTreeHeader::read(fs.extents_overflow_data.as_ref().unwrap()).depth; + assert!( + depth >= 2, + "extents-overflow tree depth {depth} < 2 — 64 overflow records did not split it" + ); + + // The data fork is the expected single-block-extent shape... + let cnid = entry.location as u32; + let (data_fork, _rsrc) = fs.find_file_by_id(cnid).unwrap(); + assert_eq!(data_fork.total_blocks, n_blocks); + + // ...and reads back byte-for-byte through the multi-level overflow tree. + let got = fs.read_file(&entry, payload.len() + 1).unwrap(); + assert_eq!(got.len(), payload.len()); + assert_eq!( + got, payload, + "fragmented read-back mismatch through split overflow tree" + ); + } + + #[test] + fn test_hfsplus_catalog_shuffled_inserts_pack_densely() { + // P5 (docs/hfsplus_btree_growth_plan.md §4d): with the B*-style rotation + // now on the HFS+ catalog insert path (insert_catalog_record -> + // btree_insert_full), shuffled multi-dir inserts must pack leaves to + // ~80%+ occupancy, matching the classic-HFS result — not the ~69% a plain + // split leaves. Measured through the same btree_insert_full the live path + // uses, with the HFSPLUS_CATALOG (variable-key) format. + use super::hfs_common::{btree_insert_full, BTreeHeader, BTreeKeyFormat}; + use byteorder::{BigEndian, ByteOrder}; + type Fs = HfsPlusFilesystem>>; + + let node_size = 512usize; + let total_nodes = 1800u32; + let mut buf = vec![0u8; total_nodes as usize * node_size]; + write_blank_btree_header_node( + &mut buf[0..node_size], + node_size, + 0, + total_nodes, + total_nodes - 2, + 516, + KEY_COMPARE_CASE_FOLDING, + ); + write_empty_leaf_node(&mut buf[node_size..2 * node_size]); + + let kf = BTreeKeyFormat::HFSPLUS_CATALOG; + let cmp = |a: &[u8], b: &[u8]| Fs::catalog_compare(a, b, false); + const DIRS: u32 = 40; + const N: u32 = 3000; + for i in 0..N { + let hash = i.wrapping_mul(2_654_435_761); + let parent = 16 + (hash % DIRS); + let name = format!("f{hash:08x}"); + let mut rec = Fs::build_catalog_key(parent, &name); + rec.extend_from_slice(&[0u8; 40]); + btree_insert_full(&mut buf, &rec, &kf, &cmp) + .unwrap_or_else(|e| panic!("insert #{i}: {e}")); + } + + // Average leaf occupancy across the whole leaf chain. + let header = BTreeHeader::read(&buf); + assert!( + header.depth >= 3, + "need a multi-level tree to judge packing" + ); + let mut used_total = 0usize; + let mut leaves = 0usize; + let mut node = header.first_leaf_node; + let mut seen = std::collections::HashSet::new(); + while node != 0 && seen.insert(node) { + let off = node as usize * node_size; + let num = BigEndian::read_u16(&buf[off + 10..off + 12]) as usize; + // Records occupy 14..free_off; the offset table holds num+1 u16s. + let free_off_pos = off + node_size - 2 * (num + 1); + let free_off = BigEndian::read_u16(&buf[free_off_pos..free_off_pos + 2]) as usize; + used_total += free_off + 2 * (num + 1); + leaves += 1; + node = BigEndian::read_u32(&buf[off..off + 4]); // forward link + } + // ~0.84 with the rotation (was ~0.69 with plain split), matching classic. + let occupancy = used_total as f64 / (leaves * node_size) as f64; + assert!( + occupancy >= 0.80, + "shuffled multi-dir leaf occupancy {occupancy:.3} (<0.80) over {leaves} leaves — \ + the rotation is not packing densely" + ); + } } diff --git a/src/fs/hfsplus_defrag.rs b/src/fs/hfsplus_defrag.rs index 265ec988..c332c1b0 100644 --- a/src/fs/hfsplus_defrag.rs +++ b/src/fs/hfsplus_defrag.rs @@ -417,7 +417,6 @@ fn allocate_fork( use byteorder::{BigEndian, ByteOrder}; use std::cmp::Ordering; -use unicode_normalization::UnicodeNormalization; use super::filesystem::FilesystemError; use super::hfs_common; @@ -651,7 +650,12 @@ pub fn build_target_metadata( key_record.push(0); } key_record.extend_from_slice(&record); - hfs_common::btree_insert_full(&mut catalog, &key_record, &cmp)?; + hfs_common::btree_insert_full( + &mut catalog, + &key_record, + &hfs_common::BTreeKeyFormat::HFSPLUS_CATALOG, + &cmp, + )?; // Root thread record. let thread = build_thread_record(CATALOG_FOLDER_THREAD, /* parent=1 */ 1, label); @@ -660,7 +664,12 @@ pub fn build_target_metadata( tk.push(0); } tk.extend_from_slice(&thread); - hfs_common::btree_insert_full(&mut catalog, &tk, &cmp)?; + hfs_common::btree_insert_full( + &mut catalog, + &tk, + &hfs_common::BTreeKeyFormat::HFSPLUS_CATALOG, + &cmp, + )?; let _ = (folder_meta, record); } @@ -719,7 +728,12 @@ pub fn build_target_metadata( key_record.push(0); } key_record.extend_from_slice(&body); - hfs_common::btree_insert_full(&mut catalog, &key_record, &cmp)?; + hfs_common::btree_insert_full( + &mut catalog, + &key_record, + &hfs_common::BTreeKeyFormat::HFSPLUS_CATALOG, + &cmp, + )?; let thread = build_thread_record(CATALOG_FOLDER_THREAD, target_parent, &d.name); let mut tk = build_catalog_key(target_cnid, ""); @@ -727,7 +741,12 @@ pub fn build_target_metadata( tk.push(0); } tk.extend_from_slice(&thread); - hfs_common::btree_insert_full(&mut catalog, &tk, &cmp)?; + hfs_common::btree_insert_full( + &mut catalog, + &tk, + &hfs_common::BTreeKeyFormat::HFSPLUS_CATALOG, + &cmp, + )?; *child_count.entry(target_parent).or_default() += 1; queue.push_back(d.cnid); @@ -769,7 +788,12 @@ pub fn build_target_metadata( key_record.push(0); } key_record.extend_from_slice(&body); - hfs_common::btree_insert_full(&mut catalog, &key_record, &cmp)?; + hfs_common::btree_insert_full( + &mut catalog, + &key_record, + &hfs_common::BTreeKeyFormat::HFSPLUS_CATALOG, + &cmp, + )?; let thread = build_thread_record(CATALOG_FILE_THREAD, target_parent, &f.name); let mut tk = build_catalog_key(target_cnid, ""); @@ -777,7 +801,12 @@ pub fn build_target_metadata( tk.push(0); } tk.extend_from_slice(&thread); - hfs_common::btree_insert_full(&mut catalog, &tk, &cmp)?; + hfs_common::btree_insert_full( + &mut catalog, + &tk, + &hfs_common::BTreeKeyFormat::HFSPLUS_CATALOG, + &cmp, + )?; *child_count.entry(target_parent).or_default() += 1; @@ -823,7 +852,12 @@ pub fn build_target_metadata( key_record.push(0); } key_record.extend_from_slice(&body); - hfs_common::btree_insert_full(&mut catalog, &key_record, &cmp)?; + hfs_common::btree_insert_full( + &mut catalog, + &key_record, + &hfs_common::BTreeKeyFormat::HFSPLUS_CATALOG, + &cmp, + )?; let thread = build_thread_record(CATALOG_FILE_THREAD, target_parent, &h.name); let mut tk = build_catalog_key(target_cnid, ""); @@ -831,7 +865,12 @@ pub fn build_target_metadata( tk.push(0); } tk.extend_from_slice(&thread); - hfs_common::btree_insert_full(&mut catalog, &tk, &cmp)?; + hfs_common::btree_insert_full( + &mut catalog, + &tk, + &hfs_common::BTreeKeyFormat::HFSPLUS_CATALOG, + &cmp, + )?; *child_count.entry(target_parent).or_default() += 1; *file_link_counts.entry(h.inode_num).or_default() += 1; @@ -856,7 +895,12 @@ pub fn build_target_metadata( key_record.push(0); } key_record.extend_from_slice(&body); - hfs_common::btree_insert_full(&mut catalog, &key_record, &cmp)?; + hfs_common::btree_insert_full( + &mut catalog, + &key_record, + &hfs_common::BTreeKeyFormat::HFSPLUS_CATALOG, + &cmp, + )?; let thread = build_thread_record(CATALOG_FILE_THREAD, target_parent, &h.name); let mut tk = build_catalog_key(target_cnid, ""); @@ -864,7 +908,12 @@ pub fn build_target_metadata( tk.push(0); } tk.extend_from_slice(&thread); - hfs_common::btree_insert_full(&mut catalog, &tk, &cmp)?; + hfs_common::btree_insert_full( + &mut catalog, + &tk, + &hfs_common::BTreeKeyFormat::HFSPLUS_CATALOG, + &cmp, + )?; *child_count.entry(target_parent).or_default() += 1; *dir_link_counts.entry(h.inode_num).or_default() += 1; @@ -1561,8 +1610,7 @@ fn build_file_body( /// Build a thread record (`type(2)+reserved(2)+parentID(4)+name`). fn build_thread_record(record_type: i16, parent_cnid: u32, name: &str) -> Vec { - let nfd: String = name.nfd().collect(); - let utf16: Vec = nfd.encode_utf16().collect(); + let utf16: Vec = crate::fs::hfs_unicode::decompose_str(name); let mut rec = Vec::with_capacity(10 + utf16.len() * 2); let mut b2 = [0u8; 2]; let mut b4 = [0u8; 4]; @@ -1636,6 +1684,7 @@ fn emit_xattrs_for_cnid( hfs_common::btree_insert_full( attr_buf, &rec, + &hfs_common::BTreeKeyFormat::HFSPLUS_ATTRIBUTES, &HfsPlusFilesystem::>>::attr_compare, )?; } @@ -2732,4 +2781,114 @@ mod tests { .collect::>() ); } + + /// P4 (docs/hfsplus_btree_growth_plan.md): the streamed defrag builder builds + /// its target catalog with `hfs_common::btree_insert_full` + + /// `BTreeKeyFormat::HFSPLUS_CATALOG` (wired in P1). Confirm that on a source + /// large enough for a *multi-level* catalog the clone re-builds an equally + /// multi-level catalog that is fsck-clean and round-trips — i.e. the + /// variable-length index keys behave correctly through the clone path, not + /// just the live-edit path. + #[test] + fn stream_clone_of_multilevel_catalog_is_fsck_clean() { + use super::super::hfs_common::BTreeHeader; + + // 64 MiB source (auto-sized catalog) with 300 files across 10 dirs — + // ~620 catalog records, comfortably past a single leaf node. + const DIRS: usize = 10; + const PER_DIR: usize = 30; + let mut src_img = create_blank_hfsplus(64 * 1024 * 1024, 4096, "Big", false); + { + let cur = Cursor::new(&mut src_img); + let mut fs = HfsPlusFilesystem::open(cur, 0).unwrap(); + fs.prepare_for_edit().unwrap(); + let root = fs.root().unwrap(); + for d in 0..DIRS { + let dir = fs + .create_directory( + &root, + &format!("d{d:02}"), + &CreateDirectoryOptions::default(), + ) + .unwrap(); + for f in 0..PER_DIR { + let body = format!("contents of d{d:02}/f{f:03}\n"); + let mut data = Cursor::new(body.into_bytes()); + let len = data.get_ref().len() as u64; + fs.create_file( + &dir, + &format!("f{f:03}.txt"), + &mut data, + len, + &CreateFileOptions::default(), + ) + .unwrap(); + } + } + fs.sync_metadata().unwrap(); + // Sanity: the source catalog really is multi-level. + let depth = BTreeHeader::read(fs.catalog_data()).depth; + assert!( + depth >= 2, + "source catalog depth {depth} < 2 — test needs a deeper tree" + ); + } + + // Clone via the streamed defrag path. + let cur = Cursor::new(&mut src_img); + let mut src_fs = HfsPlusFilesystem::open(cur, 0).unwrap(); + let target_size: u64 = 32 * 1024 * 1024; + let mut out: Vec = Vec::with_capacity(target_size as usize); + let report = stream_defragmented_hfsplus(&mut src_fs, target_size, &mut out, None) + .expect("multi-level clone should succeed"); + assert_eq!( + report.files_copied as usize, + DIRS * PER_DIR, + "all files copied" + ); + assert_eq!(report.dirs_copied as usize, DIRS, "all dirs copied"); + + // The defrag-built target catalog must itself be multi-level... + let cur = Cursor::new(out); + let mut tgt = HfsPlusFilesystem::open(cur, 0).expect("target opens"); + let tgt_depth = BTreeHeader::read(tgt.catalog_data()).depth; + assert!( + tgt_depth >= 2, + "cloned catalog collapsed to depth {tgt_depth} — multi-level rebuild failed" + ); + + // ...fsck-clean... + let fsck = tgt.fsck().expect("HFS+ fsck").expect("fsck runs"); + assert!( + fsck.is_clean(), + "cloned multi-level catalog not fsck-clean: {:?}", + fsck.errors + .iter() + .take(8) + .map(|e| (&e.code, &e.message)) + .collect::>() + ); + + // ...and round-trips: every dir/file present, contents intact (spot-check). + let root = tgt.root().unwrap(); + let dirs = tgt.list_directory(&root).unwrap(); + assert_eq!( + dirs.iter().filter(|e| e.name.starts_with('d')).count(), + DIRS, + "directory count drifted after clone" + ); + let d05 = dirs.iter().find(|e| e.name == "d05").unwrap().clone(); + let d05_kids = tgt.list_directory(&d05).unwrap(); + assert_eq!(d05_kids.len(), PER_DIR, "d05 child count drifted"); + let f017 = d05_kids + .iter() + .find(|e| e.name == "f017.txt") + .unwrap() + .clone(); + let bytes = tgt.read_file(&f017, usize::MAX).unwrap(); + assert_eq!( + bytes, b"contents of d05/f017\n", + "cloned file contents differ" + ); + } } diff --git a/src/fs/mod.rs b/src/fs/mod.rs index 3bcebaff..d6fbc45c 100644 --- a/src/fs/mod.rs +++ b/src/fs/mod.rs @@ -30,6 +30,7 @@ pub mod hfs_boot; pub mod hfs_clone; pub mod hfs_common; pub mod hfs_fsck; +pub mod hfs_unicode; pub mod hfsplus; pub mod hfsplus_clone; pub mod hfsplus_defrag; diff --git a/src/fs/tar_import.rs b/src/fs/tar_import.rs index b648fead..d3f9d5b2 100644 --- a/src/fs/tar_import.rs +++ b/src/fs/tar_import.rs @@ -13,7 +13,7 @@ //! `EditableFilesystem` mutation, callers MUST call `sync_metadata()` (and, //! for a container, `commit`) after import returns. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::fs::File; use std::io::{Read, Seek, SeekFrom}; use std::path::{Component, Path}; @@ -158,18 +158,43 @@ pub fn import_tar_from_path( } /// Import an (already-decompressed, or plain) tar stream into `dest`. +/// +/// Brackets the work in [`EditableFilesystem::begin_bulk`] / +/// [`end_bulk`](EditableFilesystem::end_bulk). The importer aborts on the first +/// hard error (every create is `?`-propagated) and callers discard the volume +/// on `Err`, so per-operation rollback is redundant here; bulk mode lets HFS +/// skip cloning its whole catalog on every entry. `end_bulk` runs even on +/// error so the filesystem is never left stuck in bulk mode. pub fn import_tar( efs: &mut dyn EditableFilesystem, dest: &FileEntry, archive: R, opts: &TarImportOptions, progress: &dyn Fn(&TarImportStats), +) -> Result { + efs.begin_bulk(); + let result = import_tar_inner(efs, dest, archive, opts, progress); + efs.end_bulk(); + result +} + +fn import_tar_inner( + efs: &mut dyn EditableFilesystem, + dest: &FileEntry, + archive: R, + opts: &TarImportOptions, + progress: &dyn Fn(&TarImportStats), ) -> Result { let mut stats = TarImportStats::default(); let mut ar = tar::Archive::new(archive); // archive-relative dir path -> the image FileEntry for that dir. let mut dir_cache: HashMap = HashMap::new(); dir_cache.insert(String::new(), dest.clone()); + // image dir path -> set of child names known to exist, so the per-entry + // conflict check is O(1) instead of listing the (growing) directory every + // time — otherwise a large single-directory import is O(n^2). Seeded lazily + // from the on-disk listing the first time we touch a directory. + let mut dir_children: HashMap> = HashMap::new(); for entry in ar.entries().context("reading tar entries")? { let mut entry = entry.context("reading tar entry")?; @@ -208,8 +233,20 @@ pub fn import_tar( let name = &leaf[0]; let parent = ensure_dir(efs, &mut dir_cache, parent_comps, &mut stats)?; - // Conflict handling. - if let Some(existing) = find_child(efs, &parent, name)? { + // Conflict handling. Seed this directory's existing-names set once + // (from the on-disk listing — empty for a directory we just created), + // then consult/update it per entry so the check is O(1). + let parent_key = parent.path.clone(); + if !dir_children.contains_key(&parent_key) { + let existing: HashSet = efs + .list_directory(&parent) + .map_err(|e| anyhow!("list_directory {}: {e}", parent.path))? + .into_iter() + .map(|c| c.name) + .collect(); + dir_children.insert(parent_key.clone(), existing); + } + if dir_children[&parent_key].contains(name) { match opts.conflict { ImportConflict::Error => bail!( "{} already exists in the image (pass --force or --skip-existing)", @@ -221,8 +258,14 @@ pub fn import_tar( continue; } ImportConflict::Overwrite => { - efs.delete_entry(&parent, &existing) - .map_err(|e| anyhow!("overwrite delete {}: {e}", raw_path.display()))?; + if let Some(existing) = find_child(efs, &parent, name)? { + efs.delete_entry(&parent, &existing) + .map_err(|e| anyhow!("overwrite delete {}: {e}", raw_path.display()))?; + } + dir_children + .get_mut(&parent_key) + .expect("seeded above") + .remove(name); stats.overwritten += 1; } } @@ -236,7 +279,13 @@ pub fn import_tar( .map(|p| p.to_string_lossy().into_owned()) .unwrap_or_default(); match efs.create_symlink(&parent, name, &target, &Default::default()) { - Ok(_) => stats.symlinks += 1, + Ok(_) => { + stats.symlinks += 1; + dir_children + .get_mut(&parent_key) + .expect("seeded above") + .insert(name.clone()); + } Err(ref e) if is_unsupported(e) => stats.symlinks_skipped += 1, Err(e) => return Err(anyhow!("create_symlink {}: {e}", raw_path.display())), } @@ -249,6 +298,10 @@ pub fn import_tar( let new_entry = efs .create_file(&parent, name, &mut entry, size, &Default::default()) .map_err(|e| anyhow!("create_file {}: {e}", raw_path.display()))?; + dir_children + .get_mut(&parent_key) + .expect("seeded above") + .insert(name.clone()); stats.files += 1; stats.total_bytes += size; if opts.apply_permissions { diff --git a/src/gui/optical_tab.rs b/src/gui/optical_tab.rs index 4f74c0af..a09851e0 100644 --- a/src/gui/optical_tab.rs +++ b/src/gui/optical_tab.rs @@ -2,16 +2,17 @@ use std::path::PathBuf; use std::sync::{Arc, Mutex}; use opticaldiscs::detect::DiscImageInfo; -use opticaldiscs::drives::OpticalDrive; use opticaldiscs::formats::DiscFormat; use rusty_backup::backup::LogLevel as BackupLogLevel; +use rusty_backup::model::optical_devices::{list_rip_devices, RipDevice}; use rusty_backup::optical::browse_view::OpticalDiscBrowseView; use rusty_backup::optical::convert::ConvertProgress; -use rusty_backup::optical::rip::{RipConfig, RipFormat, RipProgress}; +use rusty_backup::optical::rip::{OpticalTarget, RipConfig, RipFormat, RipProgress}; use rusty_backup::rbformats::chd_options::{ codec_label, parse_codec_string, ChdOptions, ChdProfile, }; +use rusty_backup::remote::RemoteConnection; use rusty_backup::update::UpdateConfig; use super::chd_options_ui::{ChdOptionsControl, ChdOptionsMode}; @@ -37,11 +38,30 @@ enum OutputFormat { Chd, } +/// Background state for the "Add remote daemon" connect — runs off the UI +/// thread so an unreachable host can't freeze egui. +#[derive(Default)] +struct ConnectStatus { + done: bool, + result: Option>, String>>, +} + /// State for the Optical disc tab. pub struct OpticalTab { source_mode: SourceMode, - drives: Vec, + /// Unified drive list: local drives + every connected daemon's drives. + rip_devices: Vec, selected_drive_idx: Option, + /// Connected remote daemons, re-queried when the drive list refreshes. + remote_daemons: Vec>>, + /// "Add remote daemon" dialog state. + add_remote_open: bool, + add_remote_addr: String, + add_remote_error: Option, + add_remote_status: Option>>, + /// MRU of daemon addresses (newest first), mirrored to `config.json` — the + /// "Add remote daemon" dialog's quick-pick list. + recent_daemons: Vec, image_file_path: Option, disc_info: Option, disc_info_error: Option, @@ -66,8 +86,14 @@ impl Default for OpticalTab { fn default() -> Self { Self { source_mode: SourceMode::ImageFile, - drives: Vec::new(), + rip_devices: Vec::new(), selected_drive_idx: None, + remote_daemons: Vec::new(), + add_remote_open: false, + add_remote_addr: String::new(), + add_remote_error: None, + add_remote_status: None, + recent_daemons: UpdateConfig::load().recent_daemon_addrs, image_file_path: None, disc_info: None, disc_info_error: None, @@ -125,11 +151,147 @@ impl OpticalTab { } pub fn refresh_drives(&mut self) { - self.drives = opticaldiscs::drives::list_drives(); + self.rip_devices = list_rip_devices(&self.remote_daemons); } pub fn drive_count(&self) -> usize { - self.drives.len() + self.rip_devices.len() + } + + /// Poll the background "Add remote daemon" connect; on success add the + /// daemon and refresh the unified drive list. + fn poll_add_remote(&mut self, log: &mut LogPanel) { + let Some(status) = &self.add_remote_status else { + return; + }; + let finished = status.lock().map(|s| s.done).unwrap_or(false); + if !finished { + return; + } + let result = self + .add_remote_status + .take() + .and_then(|s| s.lock().ok().and_then(|mut s| s.result.take())); + match result { + Some(Ok(conn)) => { + let label = conn + .lock() + .map(|c| c.addr().to_string()) + .unwrap_or_default(); + self.remember_daemon_addr(&label); + self.remote_daemons.push(conn); + self.refresh_drives(); + self.add_remote_open = false; + self.add_remote_addr.clear(); + self.add_remote_error = None; + log.info(format!("Connected to remote daemon {label}")); + } + Some(Err(e)) => { + self.add_remote_error = Some(e); + } + None => {} + } + } + + /// Record a successfully-connected daemon address in the MRU list (in-memory + /// + persisted to `config.json`), newest first. + fn remember_daemon_addr(&mut self, addr: &str) { + let addr = addr.trim(); + if addr.is_empty() { + return; + } + self.recent_daemons.retain(|a| a != addr); + self.recent_daemons.insert(0, addr.to_string()); + self.recent_daemons.truncate(8); + let mut cfg = UpdateConfig::load(); + cfg.remember_daemon(addr); + let _ = cfg.save(); + } + + /// Spawn the connect for the address in the dialog on a worker thread. + fn start_add_remote(&mut self) { + let addr = self.add_remote_addr.trim().to_string(); + if addr.is_empty() { + self.add_remote_error = Some("Enter a host:port".to_string()); + return; + } + self.add_remote_error = None; + let status = Arc::new(Mutex::new(ConnectStatus::default())); + self.add_remote_status = Some(Arc::clone(&status)); + std::thread::spawn(move || { + let result = RemoteConnection::connect_shared(&addr).map_err(|e| format!("{e:#}")); + if let Ok(mut s) = status.lock() { + s.result = Some(result); + s.done = true; + } + }); + } + + /// The "Add remote daemon" modal: a host:port field that connects on a + /// worker thread (see [`Self::start_add_remote`] / [`Self::poll_add_remote`]). + fn show_add_remote_dialog(&mut self, ctx: &egui::Context) { + if !self.add_remote_open { + return; + } + let connecting = self.add_remote_status.is_some(); + let mut open = self.add_remote_open; + egui::Window::new("Add remote daemon") + .collapsible(false) + .resizable(false) + .open(&mut open) + .show(ctx, |ui| { + ui.label("Daemon address (host:port):"); + let resp = ui.add_enabled( + !connecting, + egui::TextEdit::singleline(&mut self.add_remote_addr) + .hint_text("mister.local:7341"), + ); + let submit = resp.lost_focus() && ui.input(|i| i.key_pressed(egui::Key::Enter)); + ui.horizontal(|ui| { + if ui + .add_enabled(!connecting, egui::Button::new("Connect")) + .clicked() + || (submit && !connecting) + { + self.start_add_remote(); + } + if ui.button("Cancel").clicked() { + self.add_remote_open = false; + self.add_remote_status = None; + self.add_remote_error = None; + } + }); + if connecting { + ui.horizontal(|ui| { + ui.spinner(); + ui.label("Connecting..."); + }); + } + if let Some(err) = &self.add_remote_error { + ui.colored_label(egui::Color32::from_rgb(255, 100, 100), err); + } + + if !self.recent_daemons.is_empty() { + ui.separator(); + ui.label("Recent:"); + // Clone to avoid borrowing `self` while the click handler + // calls `&mut self` methods. + let recents = self.recent_daemons.clone(); + for addr in recents { + if ui + .add_enabled(!connecting, egui::Button::new(&addr).small()) + .clicked() + { + self.add_remote_addr = addr.clone(); + self.start_add_remote(); + } + } + } + }); + // Honor the window's [x] close button. + if !open { + self.add_remote_open = false; + } } pub fn is_running(&self) -> bool { @@ -138,6 +300,8 @@ impl OpticalTab { pub fn show(&mut self, ui: &mut egui::Ui, log: &mut LogPanel, progress: &mut ProgressState) { self.poll_progress(log, progress); + self.poll_add_remote(log); + self.show_add_remote_dialog(ui.ctx()); ui.heading("Optical Disc"); ui.add_space(8.0); @@ -153,7 +317,11 @@ impl OpticalTab { // Keep the option visible (so users know it exists) but gray it // out until they elevate via the top-bar "Show Physical Devices" // button. Image-file convert stays available unelevated. - let phys_enabled = super::physical_devices_available(); + // A local drive needs elevation (SCSI pass-through); a remote + // drive is opened by the daemon, so adding one also unlocks the + // mode. + let phys_enabled = + super::physical_devices_available() || !self.remote_daemons.is_empty(); ui.add_enabled_ui(phys_enabled, |ui| { ui.radio_value( &mut self.source_mode, @@ -161,11 +329,15 @@ impl OpticalTab { "Physical drive", ) .on_disabled_hover_text( - "Click \"Show Physical Devices\" (top bar) to enable — \ - ripping a physical disc requires administrator rights.", + "Click \"Show Physical Devices\" (top bar) for a local drive, \ + or \"Add remote daemon...\" for a networked one.", ); }); ui.radio_value(&mut self.source_mode, SourceMode::ImageFile, "Image file"); + if ui.button("Add remote daemon...").clicked() { + self.add_remote_open = true; + self.add_remote_error = None; + } }); match self.source_mode { @@ -175,24 +347,19 @@ impl OpticalTab { let current_label = self .selected_drive_idx - .and_then(|idx| self.drives.get(idx)) - .map(|d| d.display_name.clone()) + .and_then(|idx| self.rip_devices.get(idx)) + .map(|d| d.picker_label()) .unwrap_or_else(|| "Select a drive...".into()); egui::ComboBox::from_id_salt("optical_drive") .selected_text(¤t_label) - .width(300.0) + .width(360.0) .show_ui(ui, |ui| { - for (i, drive) in self.drives.iter().enumerate() { - let label = format!( - "{} ({})", - drive.display_name, - drive.device_path.display() - ); + for (i, drive) in self.rip_devices.iter().enumerate() { ui.selectable_value( &mut self.selected_drive_idx, Some(i), - &label, + drive.picker_label(), ); } }); @@ -201,7 +368,7 @@ impl OpticalTab { self.refresh_drives(); log.info(format!( "Refreshed: {} optical drive(s) found", - self.drives.len() + self.rip_devices.len() )); } }); @@ -439,7 +606,11 @@ impl OpticalTab { if self.action == Action::Rip { ui.horizontal(|ui| { ui.add_space(60.0); - ui.checkbox(&mut self.eject_after, "Eject after ripping"); + ui.checkbox(&mut self.eject_after, "Eject after ripping") + .on_hover_text( + "Ejects the source drive on its own machine — the \ + remote daemon's drive for a remote source.", + ); }); } }); @@ -488,18 +659,25 @@ impl OpticalTab { fn has_valid_source(&self) -> bool { match self.source_mode { - SourceMode::PhysicalDrive => self.selected_drive_idx.is_some(), + SourceMode::PhysicalDrive => self + .selected_drive_idx + .and_then(|idx| self.rip_devices.get(idx)) + .is_some(), SourceMode::ImageFile => self.image_file_path.is_some(), } } + /// The path the browse view / disc-info detector opens. Only local sources + /// are browsable — a remote drive is rip-only here (browsing a remote disc + /// is the remote Inspect tab's job), so it yields `None`. fn get_browsable_path(&self) -> Option { match self.source_mode { SourceMode::ImageFile => self.image_file_path.clone(), SourceMode::PhysicalDrive => self .selected_drive_idx - .and_then(|idx| self.drives.get(idx)) - .map(|d| d.device_path.clone()), + .and_then(|idx| self.rip_devices.get(idx)) + .filter(|d| !d.is_remote()) + .map(|d| PathBuf::from(&d.device_path)), } } @@ -563,8 +741,13 @@ impl OpticalTab { } fn start_rip(&mut self, log: &mut LogPanel) { - let drive = match self.selected_drive_idx.and_then(|idx| self.drives.get(idx)) { - Some(d) => d, + // Resolve the target up front (owned) so the &mut self calls below don't + // conflict with a borrow of `rip_devices`. + let target = match self + .selected_drive_idx + .and_then(|idx| self.rip_devices.get(idx)) + { + Some(d) => d.to_target(), None => { log.error("No drive selected"); return; @@ -581,7 +764,7 @@ impl OpticalTab { // CHD rip: rip to temp BIN/CUE, then convert to CHD if self.output_format == OutputFormat::Chd { - self.start_rip_to_chd(drive.device_path.clone(), output_path, log); + self.start_rip_to_chd(target, output_path, log); return; } @@ -592,7 +775,7 @@ impl OpticalTab { }; let config = RipConfig { - device_path: drive.device_path.clone(), + device: target, output_path, format, eject_after: self.eject_after, @@ -600,7 +783,7 @@ impl OpticalTab { log.info(format!( "Starting rip: {} -> {}", - config.device_path.display(), + config.device.display(), config.output_path.display() )); @@ -620,7 +803,7 @@ impl OpticalTab { } /// Rip to CHD: rip to temporary BIN/CUE, convert to CHD, clean up temps. - fn start_rip_to_chd(&mut self, device_path: PathBuf, chd_path: PathBuf, log: &mut LogPanel) { + fn start_rip_to_chd(&mut self, target: OpticalTarget, chd_path: PathBuf, log: &mut LogPanel) { let eject_after = self.eject_after; if self.chd_cd_control.mode == ChdOptionsMode::Custom { self.persist_chd_options(&self.chd_cd_control.custom); @@ -629,7 +812,7 @@ impl OpticalTab { log.info(format!( "Starting rip to CHD: {} -> {}", - device_path.display(), + target.display(), chd_path.display() )); @@ -639,13 +822,8 @@ impl OpticalTab { std::thread::spawn(move || { let _wake = rusty_backup::os::wakelock::acquire("Rusty Backup: optical rip to CHD"); - let result = rip_to_chd_worker( - &device_path, - &chd_path, - eject_after, - chd_options, - &progress_arc, - ); + let result = + rip_to_chd_worker(target, &chd_path, eject_after, chd_options, &progress_arc); if let Err(e) = result { if let Ok(mut p) = progress_arc.lock() { if !p.finished { @@ -846,9 +1024,10 @@ fn run_conversion( } } -/// Background worker: rip disc to temp BIN/CUE, convert to CHD, clean up. +/// Background worker: rip disc (local or remote) to temp BIN/CUE, convert to +/// CHD locally, clean up. The encode always runs here on the desktop. fn rip_to_chd_worker( - device_path: &std::path::Path, + target: OpticalTarget, chd_path: &std::path::Path, eject_after: bool, chd_options: ChdOptions, @@ -865,7 +1044,7 @@ fn rip_to_chd_worker( let temp_bin = parent.join(".rusty-backup-rip-temp.bin"); let rip_config = RipConfig { - device_path: device_path.to_path_buf(), + device: target, output_path: temp_cue.clone(), format: RipFormat::BinCue, eject_after, diff --git a/src/gui/progress.rs b/src/gui/progress.rs index 6750fbc9..91ec31d4 100644 --- a/src/gui/progress.rs +++ b/src/gui/progress.rs @@ -1,5 +1,9 @@ use egui::{Color32, RichText}; +// The rate/ETA estimator lives in the (non-GUI) model layer so the CLI shares +// it; re-exported here for the tabs that refer to `progress::RateTracker`. +pub use rusty_backup::model::rate_tracker::RateTracker; + #[derive(Debug, Clone, Copy, PartialEq)] pub enum LogLevel { Info, @@ -114,115 +118,6 @@ impl LogPanel { } } -/// Rolling-window transfer-rate + ETA estimator. Shared by progress -/// bars in different tabs (backup, restore, VHD export, physical-disk -/// export) so the visible rate/ETA semantics stay consistent across -/// the app. -/// -/// Drop one in next to your view's progress state, call -/// [`RateTracker::record`] each frame with the current byte counter and -/// stage label (any string that changes when the work shifts to a new -/// phase), then ask for [`RateTracker::rate_bytes_per_sec`] / -/// [`RateTracker::eta_secs`] for display. -#[derive(Default)] -pub struct RateTracker { - /// (timestamp, bytes_so_far) samples. Front = oldest. Window is - /// trimmed to ~10s so the ETA stays responsive to stalls without - /// flapping each frame. - samples: std::collections::VecDeque<(std::time::Instant, u64)>, - /// Stage label observed on the previous frame, used to detect - /// stage transitions. When the label changes the rate window is - /// cleared so a new stage starts with a fresh ETA estimate - /// instead of dragging the previous stage's numbers along. - last_stage: String, -} - -impl RateTracker { - /// Record one sample. `stage` should be the operation label or any - /// other string that changes when the work moves to a new phase - /// (e.g. "Backing up partition 1" → "Checksum verify"). When the - /// stage changes the rolling window is cleared. - pub fn record(&mut self, current_bytes: u64, stage: &str) { - const WINDOW: std::time::Duration = std::time::Duration::from_secs(10); - if stage != self.last_stage { - self.last_stage = stage.to_string(); - self.samples.clear(); - } - // Bytes regressing (e.g. a checksum phase restarts the counter) - // also resets the window — we'd otherwise compute a negative - // rate. - if let Some(&(_, last_bytes)) = self.samples.back() { - if current_bytes < last_bytes { - self.samples.clear(); - } - } - let now = std::time::Instant::now(); - self.samples.push_back((now, current_bytes)); - while let Some(&(t, _)) = self.samples.front() { - if now.duration_since(t) > WINDOW { - self.samples.pop_front(); - } else { - break; - } - } - } - - /// Compute bytes/sec over the rolling window. Returns `None` when - /// there isn't enough data yet (≤ 1 sample or < 250ms elapsed) so - /// the view can hide the rate text until it's meaningful. - pub fn rate_bytes_per_sec(&self) -> Option { - if self.samples.len() < 2 { - return None; - } - let (t_first, b_first) = *self.samples.front()?; - let (t_last, b_last) = *self.samples.back()?; - let dt = t_last.duration_since(t_first).as_secs_f64(); - if dt < 0.25 { - return None; - } - let db = b_last.saturating_sub(b_first) as f64; - if db == 0.0 { - return Some(0.0); - } - Some(db / dt) - } - - /// Estimated seconds remaining at the current rate. Returns `None` - /// when rate is unknown, zero, or `total_bytes` is missing. - pub fn eta_secs(&self, current_bytes: u64, total_bytes: u64) -> Option { - let rate = self.rate_bytes_per_sec()?; - if rate <= 0.0 || total_bytes == 0 { - return None; - } - let remaining = total_bytes.saturating_sub(current_bytes) as f64; - Some((remaining / rate) as u64) - } - - /// Convenience: format the trailing " - /s, ETA " - /// suffix that progress bars append to their `bytes / total` - /// label. Returns an empty string when neither rate nor ETA is - /// available, so callers can splice this in unconditionally. - pub fn suffix(&self, current_bytes: u64, total_bytes: u64) -> String { - match ( - self.rate_bytes_per_sec(), - self.eta_secs(current_bytes, total_bytes), - ) { - (Some(r), Some(eta)) if r > 0.0 => { - format!(" - {}/s, ETA {}", format_rate(r), format_eta(eta)) - } - (Some(r), _) if r > 0.0 => format!(" - {}/s", format_rate(r)), - _ => String::new(), - } - } - - /// Reset all sampling state. Useful when a worker is torn down and - /// a fresh one starts in the same view slot. - pub fn reset(&mut self) { - self.samples.clear(); - self.last_stage.clear(); - } -} - /// Progress state for long-running operations. #[derive(Default)] pub struct ProgressState { @@ -281,96 +176,7 @@ impl ProgressState { } } -/// Format a bytes-per-second rate as KiB/s / MiB/s / GiB/s. Mirrors -/// `partition::format_size` style for visual consistency in the -/// progress bar. -fn format_rate(bytes_per_sec: f64) -> String { - const KIB: f64 = 1024.0; - const MIB: f64 = KIB * 1024.0; - const GIB: f64 = MIB * 1024.0; - if bytes_per_sec >= GIB { - format!("{:.2} GiB", bytes_per_sec / GIB) - } else if bytes_per_sec >= MIB { - format!("{:.1} MiB", bytes_per_sec / MIB) - } else if bytes_per_sec >= KIB { - format!("{:.0} KiB", bytes_per_sec / KIB) - } else { - format!("{:.0} B", bytes_per_sec) - } -} - -/// Format an ETA in seconds as `Hh Mm` / `Mm Ss` / `Ss`. Stays -/// compact so it fits inline in the progress-bar text. -fn format_eta(total_secs: u64) -> String { - let h = total_secs / 3600; - let m = (total_secs % 3600) / 60; - let s = total_secs % 60; - if h > 0 { - format!("{h}h {m:02}m") - } else if m > 0 { - format!("{m}m {s:02}s") - } else { - format!("{s}s") - } -} - /// Get current timestamp in local time. fn chrono_now() -> String { chrono::Local::now().format("%H:%M:%S").to_string() } - -#[cfg(test)] -mod tests { - use super::*; - use std::thread::sleep; - use std::time::Duration; - - #[test] - fn rate_tracker_returns_none_before_two_samples() { - let mut t = RateTracker::default(); - t.record(0, "stage"); - assert!(t.rate_bytes_per_sec().is_none()); - assert!(t.eta_secs(0, 1024).is_none()); - assert_eq!(t.suffix(0, 1024), ""); - } - - #[test] - fn rate_tracker_computes_rate_after_window_opens() { - let mut t = RateTracker::default(); - t.record(0, "stage"); - sleep(Duration::from_millis(300)); - t.record(1_000_000, "stage"); - let rate = t.rate_bytes_per_sec().expect("rate available"); - // We slept ~300ms and added 1 MB so rate should be in the - // ballpark of 3 MB/s. Allow a generous range for CI noise. - assert!(rate > 1_000_000.0 && rate < 20_000_000.0, "rate = {rate}"); - let suffix = t.suffix(1_000_000, 10_000_000); - assert!(suffix.starts_with(" - ")); - assert!(suffix.contains("/s")); - assert!(suffix.contains("ETA")); - } - - #[test] - fn rate_tracker_resets_on_stage_change() { - let mut t = RateTracker::default(); - t.record(0, "stage-A"); - sleep(Duration::from_millis(300)); - t.record(1_000_000, "stage-A"); - assert!(t.rate_bytes_per_sec().is_some()); - t.record(0, "stage-B"); - // Window cleared; only one sample in the new stage. - assert!(t.rate_bytes_per_sec().is_none()); - } - - #[test] - fn rate_tracker_resets_on_byte_regression() { - let mut t = RateTracker::default(); - t.record(500, "stage"); - sleep(Duration::from_millis(300)); - t.record(1_000_000, "stage"); - assert!(t.rate_bytes_per_sec().is_some()); - // A regression (counter restart) clears the window. - t.record(0, "stage"); - assert!(t.rate_bytes_per_sec().is_none()); - } -} diff --git a/src/model/mod.rs b/src/model/mod.rs index a689e5c5..f7a665d7 100644 --- a/src/model/mod.rs +++ b/src/model/mod.rs @@ -23,8 +23,11 @@ pub mod file_types; pub mod fsck_runner; pub mod hfs_expand_runner; pub mod min_size_runner; +#[cfg(feature = "optical")] +pub mod optical_devices; pub mod partition_editor; pub mod physical_write_runner; +pub mod rate_tracker; // The remote file-browser core depends on `crate::remote`, which is itself // behind the `remote` feature. #[cfg(feature = "remote")] diff --git a/src/model/optical_devices.rs b/src/model/optical_devices.rs new file mode 100644 index 00000000..bbe71079 --- /dev/null +++ b/src/model/optical_devices.rs @@ -0,0 +1,177 @@ +//! The unified rip-device picker model. +//! +//! A rip can read from a locally-attached drive or a drive on a remote +//! `rb-cli serve` daemon. This module merges both into one [`RipDevice`] list so +//! the GUI shows a single pulldown and the CLI a single listing — each entry +//! tagged with where it lives and able to mint the right +//! [`crate::optical::rip::OpticalTarget`]. See `docs/remote_ripping.md`. + +use crate::optical::rip::OpticalTarget; + +#[cfg(feature = "remote")] +use crate::remote::connection::RemoteConnection; +#[cfg(feature = "remote")] +use std::sync::{Arc, Mutex}; + +/// Where a [`RipDevice`] physically lives. +pub enum DeviceLocation { + /// A drive attached to this machine. + Local, + /// A drive on a remote daemon, reached over an open connection. `label` is + /// the daemon's `host:port`. + #[cfg(feature = "remote")] + Remote { + conn: Arc>, + label: String, + }, +} + +/// One selectable optical drive in the unified picker. +pub struct RipDevice { + /// Human-readable drive name (e.g. `TSSTcorp CD/DVDW SH-224`). + pub display_name: String, + /// Device path on its own machine (e.g. `/dev/sr0`). + pub device_path: String, + pub location: DeviceLocation, +} + +impl RipDevice { + /// Label for a GUI pulldown: local shows `name (path)`; remote prefixes the + /// daemon as `[host:port] name (path)`. + pub fn picker_label(&self) -> String { + match &self.location { + DeviceLocation::Local => format!("{} ({})", self.display_name, self.device_path), + #[cfg(feature = "remote")] + DeviceLocation::Remote { label, .. } => { + format!("[{}] {} ({})", label, self.display_name, self.device_path) + } + } + } + + /// The `--device` argument that would re-open this drive from the CLI: a + /// local path, or `rb://host:port/dev/sr0` for a remote drive. + pub fn cli_device_arg(&self) -> String { + match &self.location { + DeviceLocation::Local => self.device_path.clone(), + #[cfg(feature = "remote")] + DeviceLocation::Remote { label, .. } => format!("rb://{}{}", label, self.device_path), + } + } + + /// The [`OpticalTarget`] a rip reads from (borrowing — clones the cheap + /// device path / `Arc` connection handle, leaving the device in its list). + pub fn to_target(&self) -> OpticalTarget { + match &self.location { + DeviceLocation::Local => OpticalTarget::Local(self.device_path.clone()), + #[cfg(feature = "remote")] + DeviceLocation::Remote { conn, .. } => OpticalTarget::Remote { + conn: conn.clone(), + device_path: self.device_path.clone(), + }, + } + } + + /// Consume into the [`OpticalTarget`] a rip reads from. + pub fn into_target(self) -> OpticalTarget { + match self.location { + DeviceLocation::Local => OpticalTarget::Local(self.device_path), + #[cfg(feature = "remote")] + DeviceLocation::Remote { conn, .. } => OpticalTarget::Remote { + conn, + device_path: self.device_path, + }, + } + } + + /// Is this drive on a remote daemon? (Local-only features like disc-info + /// auto-detection and Browse Contents are skipped for remote drives.) + pub fn is_remote(&self) -> bool { + #[cfg(feature = "remote")] + { + matches!(self.location, DeviceLocation::Remote { .. }) + } + #[cfg(not(feature = "remote"))] + { + false + } + } +} + +/// All locally-attached optical drives (the same enumeration the GUI's drive +/// combo and `rb-cli optical drives` use). +pub fn list_local_rip_devices() -> Vec { + opticaldiscs::drives::list_drives() + .into_iter() + .map(|d| RipDevice { + display_name: d.display_name, + device_path: d.device_path.to_string_lossy().into_owned(), + location: DeviceLocation::Local, + }) + .collect() +} + +/// Append a connected daemon's optical drives to `out`, returning the count +/// added. Errors are swallowed — an offline daemon, or one built without the +/// `optical` feature (its `list_optical_drives` replies with an error), simply +/// contributes no drives. This is also how the picker stays capability-gated +/// without inspecting the handshake bits. +#[cfg(feature = "remote")] +pub fn append_remote_rip_devices( + out: &mut Vec, + conn: &Arc>, +) -> usize { + let (label, drives) = { + let mut guard = match conn.lock() { + Ok(g) => g, + Err(_) => return 0, + }; + let label = guard.addr().to_string(); + match guard.list_optical_drives() { + Ok(d) => (label, d), + Err(_) => return 0, + } + }; + let added = drives.len(); + for d in drives { + out.push(RipDevice { + display_name: d.display_name, + device_path: d.device_path, + location: DeviceLocation::Remote { + conn: conn.clone(), + label: label.clone(), + }, + }); + } + added +} + +/// Local + every connected daemon's optical drives, merged into one list. +#[cfg(feature = "remote")] +pub fn list_rip_devices(remotes: &[Arc>]) -> Vec { + let mut out = list_local_rip_devices(); + for conn in remotes { + append_remote_rip_devices(&mut out, conn); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn local_label_and_cli_arg() { + let d = RipDevice { + display_name: "TSSTcorp CD/DVDW".to_string(), + device_path: "/dev/sr0".to_string(), + location: DeviceLocation::Local, + }; + assert_eq!(d.picker_label(), "TSSTcorp CD/DVDW (/dev/sr0)"); + assert_eq!(d.cli_device_arg(), "/dev/sr0"); + match d.into_target() { + OpticalTarget::Local(p) => assert_eq!(p, "/dev/sr0"), + #[cfg(feature = "remote")] + _ => panic!("expected a local target"), + } + } +} diff --git a/src/model/rate_tracker.rs b/src/model/rate_tracker.rs new file mode 100644 index 00000000..8a86e908 --- /dev/null +++ b/src/model/rate_tracker.rs @@ -0,0 +1,187 @@ +//! A small rolling-window throughput + ETA estimator, shared by the GUI +//! progress bar and the CLI's progress lines. +//! +//! Drop one next to a progress counter, call [`RateTracker::record`] each tick +//! with the running byte total and a stage label (any string that changes when +//! the work moves to a new phase), then read [`RateTracker::rate_bytes_per_sec`] +//! / [`RateTracker::eta_secs`] / [`RateTracker::suffix`]. + +use std::collections::VecDeque; +use std::time::{Duration, Instant}; + +/// Rolling-window rate + ETA tracker. The window is ~10s so the ETA stays +/// responsive to stalls without flapping each tick; a stage change or a +/// regressing byte counter clears it so a new phase starts with a fresh +/// estimate. +#[derive(Default)] +pub struct RateTracker { + /// (timestamp, bytes_so_far) samples. Front = oldest. + samples: VecDeque<(Instant, u64)>, + /// Stage label seen on the previous tick, to detect phase transitions. + last_stage: String, +} + +impl RateTracker { + /// Record one sample. `stage` should be the operation label or any other + /// string that changes when the work moves to a new phase. When the stage + /// changes (or the counter regresses) the rolling window is cleared. + pub fn record(&mut self, current_bytes: u64, stage: &str) { + const WINDOW: Duration = Duration::from_secs(10); + if stage != self.last_stage { + self.last_stage = stage.to_string(); + self.samples.clear(); + } + // Bytes regressing (e.g. a checksum phase restarts the counter) also + // resets the window — we'd otherwise compute a negative rate. + if let Some(&(_, last_bytes)) = self.samples.back() { + if current_bytes < last_bytes { + self.samples.clear(); + } + } + let now = Instant::now(); + self.samples.push_back((now, current_bytes)); + while let Some(&(t, _)) = self.samples.front() { + if now.duration_since(t) > WINDOW { + self.samples.pop_front(); + } else { + break; + } + } + } + + /// Compute bytes/sec over the rolling window. Returns `None` when there + /// isn't enough data yet (≤ 1 sample or < 250ms elapsed) so callers can hide + /// the rate text until it's meaningful. + pub fn rate_bytes_per_sec(&self) -> Option { + if self.samples.len() < 2 { + return None; + } + let (t_first, b_first) = *self.samples.front()?; + let (t_last, b_last) = *self.samples.back()?; + let dt = t_last.duration_since(t_first).as_secs_f64(); + if dt < 0.25 { + return None; + } + let db = b_last.saturating_sub(b_first) as f64; + if db == 0.0 { + return Some(0.0); + } + Some(db / dt) + } + + /// Estimated seconds remaining at the current rate. Returns `None` when the + /// rate is unknown, zero, or `total_bytes` is missing. + pub fn eta_secs(&self, current_bytes: u64, total_bytes: u64) -> Option { + let rate = self.rate_bytes_per_sec()?; + if rate <= 0.0 || total_bytes == 0 { + return None; + } + let remaining = total_bytes.saturating_sub(current_bytes) as f64; + Some((remaining / rate) as u64) + } + + /// Format the trailing ` - /s, ETA ` suffix that progress + /// displays append to their `bytes / total` label. Empty when neither rate + /// nor ETA is available, so callers can splice it in unconditionally. + pub fn suffix(&self, current_bytes: u64, total_bytes: u64) -> String { + match ( + self.rate_bytes_per_sec(), + self.eta_secs(current_bytes, total_bytes), + ) { + (Some(r), Some(eta)) if r > 0.0 => { + format!(" - {}/s, ETA {}", format_rate(r), format_eta(eta)) + } + (Some(r), _) if r > 0.0 => format!(" - {}/s", format_rate(r)), + _ => String::new(), + } + } + + /// Reset all sampling state. Useful when a worker is torn down and a fresh + /// one starts in the same slot. + pub fn reset(&mut self) { + self.samples.clear(); + self.last_stage.clear(); + } +} + +/// Format a bytes-per-second rate as KiB/s / MiB/s / GiB/s. +fn format_rate(bytes_per_sec: f64) -> String { + const KIB: f64 = 1024.0; + const MIB: f64 = KIB * 1024.0; + const GIB: f64 = MIB * 1024.0; + if bytes_per_sec >= GIB { + format!("{:.2} GiB", bytes_per_sec / GIB) + } else if bytes_per_sec >= MIB { + format!("{:.1} MiB", bytes_per_sec / MIB) + } else if bytes_per_sec >= KIB { + format!("{:.0} KiB", bytes_per_sec / KIB) + } else { + format!("{:.0} B", bytes_per_sec) + } +} + +/// Format an ETA in seconds as `Hh Mm` / `Mm Ss` / `Ss`. +fn format_eta(total_secs: u64) -> String { + let h = total_secs / 3600; + let m = (total_secs % 3600) / 60; + let s = total_secs % 60; + if h > 0 { + format!("{h}h {m:02}m") + } else if m > 0 { + format!("{m}m {s:02}s") + } else { + format!("{s}s") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::thread::sleep; + + #[test] + fn rate_tracker_returns_none_before_two_samples() { + let mut t = RateTracker::default(); + t.record(0, "stage"); + assert!(t.rate_bytes_per_sec().is_none()); + assert!(t.eta_secs(0, 1024).is_none()); + assert_eq!(t.suffix(0, 1024), ""); + } + + #[test] + fn rate_tracker_computes_rate_after_window_opens() { + let mut t = RateTracker::default(); + t.record(0, "stage"); + sleep(Duration::from_millis(300)); + t.record(1_000_000, "stage"); + let rate = t.rate_bytes_per_sec().expect("rate available"); + // ~300ms + 1 MB => ~3 MB/s; allow a generous range for CI noise. + assert!(rate > 1_000_000.0 && rate < 20_000_000.0, "rate = {rate}"); + let suffix = t.suffix(1_000_000, 10_000_000); + assert!(suffix.starts_with(" - ")); + assert!(suffix.contains("/s")); + assert!(suffix.contains("ETA")); + } + + #[test] + fn rate_tracker_resets_on_stage_change() { + let mut t = RateTracker::default(); + t.record(0, "stage-A"); + sleep(Duration::from_millis(300)); + t.record(1_000_000, "stage-A"); + assert!(t.rate_bytes_per_sec().is_some()); + t.record(0, "stage-B"); + assert!(t.rate_bytes_per_sec().is_none()); + } + + #[test] + fn rate_tracker_resets_on_byte_regression() { + let mut t = RateTracker::default(); + t.record(500, "stage"); + sleep(Duration::from_millis(300)); + t.record(1_000_000, "stage"); + assert!(t.rate_bytes_per_sec().is_some()); + t.record(0, "stage"); + assert!(t.rate_bytes_per_sec().is_none()); + } +} diff --git a/src/optical/mod.rs b/src/optical/mod.rs index 13f9a5ee..c8f50491 100644 --- a/src/optical/mod.rs +++ b/src/optical/mod.rs @@ -4,6 +4,10 @@ pub mod browse_view; pub mod convert; pub mod rip; +pub mod source; pub use convert::ConvertProgress; -pub use rip::{run_rip, RipConfig, RipFormat, RipProgress}; +pub use rip::{run_rip, OpticalTarget, RipConfig, RipFormat, RipProgress}; +#[cfg(feature = "remote")] +pub use source::RemoteCdReader; +pub use source::{LocalCdReader, OpticalSource}; diff --git a/src/optical/rip.rs b/src/optical/rip.rs index 7474e258..dddbd0ab 100644 --- a/src/optical/rip.rs +++ b/src/optical/rip.rs @@ -1,13 +1,14 @@ use std::collections::VecDeque; use std::fs::File; use std::io::Write; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::sync::{Arc, Mutex}; use anyhow::{bail, Context, Result}; -use cd_da_reader::{CdReader, RetryConfig, SectorReadMode}; +use cd_da_reader::SectorReadMode; use crate::backup::{LogLevel, LogMessage}; +use crate::optical::source::{LocalCdReader, OpticalSource}; /// Output format for disc ripping. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -18,11 +19,71 @@ pub enum RipFormat { BinCue, } +/// Where a rip reads its sectors from. +/// +/// `Local` is a physically-attached drive; `Remote` is a drive on another +/// machine's `rb-cli serve` daemon, reached over an open connection — the daemon +/// issues the SCSI reads and this side does all the encoding. See +/// `docs/remote_ripping.md`. +#[derive(Clone)] +pub enum OpticalTarget { + /// A locally-attached drive by device path (e.g. `/dev/sr0`, `\\.\E:`). + Local(String), + /// A drive on a remote daemon, by its device path on that machine. + #[cfg(feature = "remote")] + Remote { + conn: Arc>, + device_path: String, + }, +} + +impl OpticalTarget { + /// Resolve a `--device` argument string: an `rb://host:port/dev/sr0` URL + /// opens a remote connection; anything else is a local device path. + pub fn resolve(device: &str) -> Result { + #[cfg(feature = "remote")] + if let Some(rref) = crate::remote::protocol::RemoteRef::parse(device) { + let conn = crate::remote::connection::RemoteConnection::connect_shared(&rref.addr()) + .with_context(|| format!("connecting to {}", rref.addr()))?; + return Ok(OpticalTarget::Remote { + conn, + device_path: rref.path, + }); + } + #[cfg(not(feature = "remote"))] + if device.starts_with("rb://") || device.starts_with("rb:/") { + bail!("remote ripping needs the `remote` feature; this binary was built without it"); + } + Ok(OpticalTarget::Local(device.to_string())) + } + + /// Human-readable label for logs / progress. + pub fn display(&self) -> String { + match self { + OpticalTarget::Local(p) => p.clone(), + #[cfg(feature = "remote")] + OpticalTarget::Remote { conn, device_path } => { + let addr = conn + .lock() + .map(|c| c.addr().to_string()) + .unwrap_or_default(); + format!("rb://{addr}{device_path}") + } + } + } +} + +impl std::fmt::Debug for OpticalTarget { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "OpticalTarget({})", self.display()) + } +} + /// Configuration for a disc rip operation. #[derive(Debug, Clone)] pub struct RipConfig { - /// Device path (e.g. "/dev/sr0", "disk6", r"\\.\E:"). - pub device_path: PathBuf, + /// Where to read the disc from (a local drive or a remote daemon's drive). + pub device: OpticalTarget, /// Output path: .iso file for Iso format, .cue file for BinCue (derives .bin). pub output_path: PathBuf, /// Output format. @@ -94,9 +155,21 @@ fn set_progress(progress: &Arc>, current_bytes: u64, current_ /// Main rip orchestrator. Runs on a background thread. pub fn run_rip(config: RipConfig, progress: Arc>) -> Result<()> { + set_operation(&progress, "Opening drive..."); + let source = match open_optical_source(&config) { + Ok(s) => s, + Err(e) => { + if let Ok(mut p) = progress.lock() { + p.error = Some(format!("{e:#}")); + p.finished = true; + } + return Err(e); + } + }; + let result = match config.format { - RipFormat::Iso => rip_iso(&config, &progress), - RipFormat::BinCue => rip_bin_cue(&config, &progress), + RipFormat::Iso => rip_iso(&config, source.as_ref(), &progress), + RipFormat::BinCue => rip_bin_cue(&config, source.as_ref(), &progress), }; if let Err(ref e) = result { @@ -108,7 +181,7 @@ pub fn run_rip(config: RipConfig, progress: Arc>) -> Result<( } if config.eject_after { - if let Err(e) = eject_disc(&config.device_path) { + if let Err(e) = source.eject() { log( &progress, LogLevel::Warning, @@ -124,19 +197,34 @@ pub fn run_rip(config: RipConfig, progress: Arc>) -> Result<( Ok(()) } +/// Build the [`OpticalSource`] for this rip — a local drive or a network proxy +/// to a remote daemon's drive (see `docs/remote_ripping.md`). +fn open_optical_source(config: &RipConfig) -> Result> { + match &config.device { + OpticalTarget::Local(path) => Ok(Box::new(LocalCdReader::open(path)?)), + #[cfg(feature = "remote")] + OpticalTarget::Remote { conn, device_path } => { + Ok(Box::new(crate::optical::source::RemoteCdReader::open( + conn.clone(), + device_path, + cd_da_reader::RetryConfig::default(), + )?)) + } + } +} + /// Sectors to read per chunk for READ CD commands. const SECTORS_PER_CHUNK: u32 = 128; -fn rip_iso(config: &RipConfig, progress: &Arc>) -> Result<()> { - let device_str = config.device_path.to_string_lossy(); - set_operation(progress, "Opening drive..."); - let reader = CdReader::open(&device_str) - .with_context(|| format!("Failed to open drive: {device_str}"))?; - +fn rip_iso( + config: &RipConfig, + source: &dyn OpticalSource, + progress: &Arc>, +) -> Result<()> { set_operation(progress, "Reading TOC..."); - let toc = reader + let toc = source .read_toc() - .with_context(|| "Failed to read table of contents")?; + .context("Failed to read table of contents")?; log( progress, @@ -179,7 +267,6 @@ fn rip_iso(config: &RipConfig, progress: &Arc>) -> Result<()> let mut out = File::create(&config.output_path) .with_context(|| format!("Failed to create {}", config.output_path.display()))?; - let retry_cfg = RetryConfig::default(); let mut bytes_written: u64 = 0; let mut sectors_done: u64 = 0; let mut lba = start_lba; @@ -191,8 +278,8 @@ fn rip_iso(config: &RipConfig, progress: &Arc>) -> Result<()> } let count = remaining.min(SECTORS_PER_CHUNK); - let data = reader - .read_data_sectors(lba, count, SectorReadMode::DataCooked, &retry_cfg) + let data = source + .read_data_sectors(lba, count, SectorReadMode::DataCooked) .with_context(|| format!("Read error at LBA {lba}"))?; out.write_all(&data) @@ -220,16 +307,15 @@ fn rip_iso(config: &RipConfig, progress: &Arc>) -> Result<()> Ok(()) } -fn rip_bin_cue(config: &RipConfig, progress: &Arc>) -> Result<()> { - let device_str = config.device_path.to_string_lossy(); - set_operation(progress, "Opening drive..."); - let reader = CdReader::open(&device_str) - .with_context(|| format!("Failed to open drive: {device_str}"))?; - +fn rip_bin_cue( + config: &RipConfig, + source: &dyn OpticalSource, + progress: &Arc>, +) -> Result<()> { set_operation(progress, "Reading TOC..."); - let toc = reader + let toc = source .read_toc() - .with_context(|| "Failed to read table of contents")?; + .context("Failed to read table of contents")?; log( progress, @@ -263,7 +349,6 @@ fn rip_bin_cue(config: &RipConfig, progress: &Arc>) -> Result let mut bin_file = File::create(&bin_path) .with_context(|| format!("Failed to create {}", bin_path.display()))?; - let retry_cfg = RetryConfig::default(); let mut bytes_written: u64 = 0; let mut sectors_done: u64 = 0; @@ -304,8 +389,8 @@ fn rip_bin_cue(config: &RipConfig, progress: &Arc>) -> Result } let count = track_remaining.min(SECTORS_PER_CHUNK); - let data = reader - .read_data_sectors(lba, count, mode, &retry_cfg) + let data = source + .read_data_sectors(lba, count, mode) .with_context(|| format!("Read error at LBA {lba} (track {})", track.number))?; bin_file @@ -374,51 +459,6 @@ fn lba_to_msf(lba: u32) -> (u8, u8, u8) { (minutes, seconds, frames) } -/// Eject the disc from the drive. -fn eject_disc(path: &Path) -> Result<()> { - #[cfg(target_os = "linux")] - { - let status = std::process::Command::new("eject") - .arg(path) - .status() - .context("Failed to run eject command")?; - if !status.success() { - bail!("eject command failed with status {status}"); - } - } - - #[cfg(target_os = "macos")] - { - let path_str = path.to_string_lossy(); - let status = std::process::Command::new("diskutil") - .args(["eject", &path_str]) - .status() - .context("Failed to run diskutil eject")?; - if !status.success() { - bail!("diskutil eject failed with status {status}"); - } - } - - #[cfg(target_os = "windows")] - { - let path_str = path.to_string_lossy(); - // Use PowerShell to eject. The path should be like "D:" or "E:" - let drive_letter = path_str.trim_start_matches(r"\\.\").trim_end_matches(':'); - let ps_script = format!( - "(New-Object -ComObject Shell.Application).NameSpace(17).ParseName('{drive_letter}:').InvokeVerb('Eject')" - ); - let status = std::process::Command::new("powershell") - .args(["-NoProfile", "-Command", &ps_script]) - .status() - .context("Failed to run PowerShell eject")?; - if !status.success() { - bail!("PowerShell eject failed with status {status}"); - } - } - - Ok(()) -} - #[cfg(test)] mod tests { use super::*; @@ -483,13 +523,14 @@ mod tests { #[test] fn test_rip_config_construction() { let config = RipConfig { - device_path: PathBuf::from("/dev/sr0"), + device: OpticalTarget::Local("/dev/sr0".to_string()), output_path: PathBuf::from("/tmp/disc.iso"), format: RipFormat::Iso, eject_after: false, }; assert_eq!(config.format, RipFormat::Iso); assert!(!config.eject_after); + assert_eq!(config.device.display(), "/dev/sr0"); } #[test] diff --git a/src/optical/source.rs b/src/optical/source.rs new file mode 100644 index 00000000..c3a8414a --- /dev/null +++ b/src/optical/source.rs @@ -0,0 +1,191 @@ +//! The optical-drive read abstraction. +//! +//! Ripping (`rip.rs`) touches the physical drive through only three operations +//! — read the TOC, read raw sectors, eject. Factoring those behind +//! [`OpticalSource`] lets the rip pipeline run unchanged against either a local +//! drive ([`LocalCdReader`], via the `cd-da-reader` crate) or — in a later phase +//! — a remote drive proxied over the rb-daemon. All output encoding (ISO / +//! BIN-CUE assembly, CHD compression) stays caller-side, so swapping the +//! *reader* moves the heavy work onto the desktop while the device only streams +//! raw sectors. See `docs/remote_ripping.md`. + +use std::path::Path; + +use anyhow::{bail, Context, Result}; +use cd_da_reader::{CdReader, RetryConfig, SectorReadMode, Toc}; + +/// A source of optical-disc sectors. The three methods are the entire physical +/// surface the rip pipeline needs, so an implementor can be a locally-attached +/// drive or a network proxy without the rip/encode code knowing the difference. +pub trait OpticalSource { + /// Read the disc's table of contents. + fn read_toc(&self) -> Result; + + /// Read `count` sectors starting at `lba` in the given mode. Retry / backoff + /// is configured when the source is opened (kept next to the drive for the + /// remote case), so it is intentionally not a parameter here. + fn read_data_sectors(&self, lba: u32, count: u32, mode: SectorReadMode) -> Result>; + + /// Eject the disc. + fn eject(&self) -> Result<()>; +} + +/// [`OpticalSource`] backed by a physically-attached drive via `cd-da-reader`. +pub struct LocalCdReader { + inner: CdReader, + device_path: String, + retry: RetryConfig, +} + +impl LocalCdReader { + /// Open `device_path` (e.g. `/dev/sr0`, `disk6`, `\\.\E:`) with the default + /// retry policy. + pub fn open(device_path: &str) -> Result { + Self::with_retry(device_path, RetryConfig::default()) + } + + /// Open `device_path` with an explicit retry policy. + pub fn with_retry(device_path: &str, retry: RetryConfig) -> Result { + let inner = CdReader::open(device_path) + .with_context(|| format!("Failed to open drive: {device_path}"))?; + Ok(Self { + inner, + device_path: device_path.to_string(), + retry, + }) + } +} + +impl OpticalSource for LocalCdReader { + fn read_toc(&self) -> Result { + self.inner.read_toc().map_err(anyhow::Error::from) + } + + fn read_data_sectors(&self, lba: u32, count: u32, mode: SectorReadMode) -> Result> { + self.inner + .read_data_sectors(lba, count, mode, &self.retry) + .map_err(anyhow::Error::from) + } + + fn eject(&self) -> Result<()> { + eject_disc(Path::new(&self.device_path)) + } +} + +/// [`OpticalSource`] that proxies every drive op to a remote daemon over an open +/// [`crate::remote::connection::RemoteConnection`]. The daemon owns the physical +/// drive and runs the retry/backoff loop; this side only requests the TOC + +/// sector ranges and does all the encoding. See `docs/remote_ripping.md`. +#[cfg(feature = "remote")] +mod remote_source { + use std::sync::{Arc, Mutex, MutexGuard}; + + use anyhow::{anyhow, Context, Result}; + use cd_da_reader::{RetryConfig, SectorReadMode, Toc}; + + use super::OpticalSource; + use crate::remote::connection::RemoteConnection; + use crate::remote::protocol::{WireRetryConfig, WireSectorMode}; + + pub struct RemoteCdReader { + conn: Arc>, + handle: u64, + } + + impl RemoteCdReader { + /// Open `device_path` on the daemon behind `conn`. `retry` is sent to the + /// daemon and applied there (next to the drive). + pub fn open( + conn: Arc>, + device_path: &str, + retry: RetryConfig, + ) -> Result { + let handle = conn + .lock() + .map_err(|_| anyhow!("remote connection lock poisoned"))? + .open_optical(device_path, WireRetryConfig::from(&retry)) + .with_context(|| format!("opening remote optical drive {device_path}"))?; + Ok(Self { conn, handle }) + } + + fn lock(&self) -> Result> { + self.conn + .lock() + .map_err(|_| anyhow!("remote connection lock poisoned")) + } + } + + impl OpticalSource for RemoteCdReader { + fn read_toc(&self) -> Result { + let wire = self.lock()?.read_toc(self.handle)?; + Ok(Toc::from(&wire)) + } + + fn read_data_sectors(&self, lba: u32, count: u32, mode: SectorReadMode) -> Result> { + self.lock()? + .read_optical_sectors(self.handle, lba, count, WireSectorMode::from(mode)) + } + + fn eject(&self) -> Result<()> { + self.lock()?.eject_optical(self.handle) + } + } + + impl Drop for RemoteCdReader { + fn drop(&mut self) { + // Best-effort: free the daemon's optical slot. Ignore errors — the + // socket may be gone, and the daemon reaps the session on disconnect. + if let Ok(mut conn) = self.conn.lock() { + let _ = conn.close_optical(self.handle); + } + } + } +} + +#[cfg(feature = "remote")] +pub use remote_source::RemoteCdReader; + +/// Eject the disc from the drive at `path` (OS-specific shell-out). +fn eject_disc(path: &Path) -> Result<()> { + #[cfg(target_os = "linux")] + { + let status = std::process::Command::new("eject") + .arg(path) + .status() + .context("Failed to run eject command")?; + if !status.success() { + bail!("eject command failed with status {status}"); + } + } + + #[cfg(target_os = "macos")] + { + let path_str = path.to_string_lossy(); + let status = std::process::Command::new("diskutil") + .args(["eject", &path_str]) + .status() + .context("Failed to run diskutil eject")?; + if !status.success() { + bail!("diskutil eject failed with status {status}"); + } + } + + #[cfg(target_os = "windows")] + { + let path_str = path.to_string_lossy(); + // Use PowerShell to eject. The path should be like "D:" or "E:" + let drive_letter = path_str.trim_start_matches(r"\\.\").trim_end_matches(':'); + let ps_script = format!( + "(New-Object -ComObject Shell.Application).NameSpace(17).ParseName('{drive_letter}:').InvokeVerb('Eject')" + ); + let status = std::process::Command::new("powershell") + .args(["-NoProfile", "-Command", &ps_script]) + .status() + .context("Failed to run PowerShell eject")?; + if !status.success() { + bail!("PowerShell eject failed with status {status}"); + } + } + + Ok(()) +} diff --git a/src/remote/client.rs b/src/remote/client.rs index 77bf8000..85416386 100644 --- a/src/remote/client.rs +++ b/src/remote/client.rs @@ -11,7 +11,8 @@ use std::path::Path; use crate::remote::protocol::{ read_chunks, read_control, write_control, ChunkWriter, Request, Response, WireEntry, - CAP_FAMILY_F, MIN_PROTOCOL_VERSION, PROTOCOL_VERSION, RB_HELLO_MAGIC, + WireOpticalDrive, WireRetryConfig, WireSectorMode, WireToc, CAP_FAMILY_F, MIN_PROTOCOL_VERSION, + PROTOCOL_VERSION, RB_HELLO_MAGIC, }; /// Result of opening a remote image: a handle + the filesystem's display @@ -462,6 +463,87 @@ impl RemoteSession { self.expect_ok("CloseSession") } + // --- optical tier (Family O): drive a remote CD/DVD; the daemon issues --- + // --- the SCSI reads, this client does all the encoding. --- + + /// List the daemon machine's physical optical drives. + pub fn list_optical_drives(&mut self) -> Result> { + write_control(&mut self.writer, &Request::ListOpticalDrives)?; + match self.read_response()? { + Response::OpticalDrives { drives } => Ok(drives), + Response::Error { message } => bail!("list optical drives: {message}"), + other => bail!("unexpected reply to ListOpticalDrives: {other:?}"), + } + } + + /// Open one of the daemon's optical drives for ripping. Returns the handle. + /// `retry` is applied daemon-side, next to the drive. + pub fn open_optical(&mut self, path: &str, retry: WireRetryConfig) -> Result { + write_control( + &mut self.writer, + &Request::OpenOptical { + path: path.to_string(), + retry, + }, + )?; + match self.read_response()? { + Response::OpticalOpened { handle } => Ok(handle), + Response::Error { message } => bail!("open optical drive {path}: {message}"), + other => bail!("unexpected reply to OpenOptical: {other:?}"), + } + } + + /// Read the open disc's table of contents. + pub fn read_toc(&mut self, handle: u64) -> Result { + write_control(&mut self.writer, &Request::ReadToc { handle })?; + match self.read_response()? { + Response::Toc { toc } => Ok(toc), + Response::Error { message } => bail!("read TOC: {message}"), + other => bail!("unexpected reply to ReadToc: {other:?}"), + } + } + + /// Read `count` sectors from `lba` in `mode`; returns the raw sector bytes. + pub fn read_optical_sectors( + &mut self, + handle: u64, + lba: u32, + count: u32, + mode: WireSectorMode, + ) -> Result> { + write_control( + &mut self.writer, + &Request::ReadOpticalSectors { + handle, + lba, + count, + mode, + }, + )?; + match self.read_response()? { + Response::FileBegin { .. } => { + let mut buf = Vec::new(); + read_chunks(&mut self.reader, &mut buf) + .map_err(|e| anyhow!("reading sectors at LBA {lba}: {e}"))?; + Ok(buf) + } + Response::Error { message } => bail!("read sectors at LBA {lba}: {message}"), + other => bail!("unexpected reply to ReadOpticalSectors: {other:?}"), + } + } + + /// Eject the disc from the open optical drive. + pub fn eject_optical(&mut self, handle: u64) -> Result<()> { + write_control(&mut self.writer, &Request::EjectOptical { handle })?; + self.expect_ok("EjectOptical") + } + + /// Close an open optical handle, freeing the drive for the next session. + pub fn close_optical(&mut self, handle: u64) -> Result<()> { + write_control(&mut self.writer, &Request::CloseOptical { handle })?; + self.expect_ok("CloseOptical") + } + fn expect_ok(&mut self, what: &str) -> Result<()> { match self.read_response()? { Response::Ok => Ok(()), diff --git a/src/remote/connection.rs b/src/remote/connection.rs index b5a6c2d4..6a1e8f3d 100644 --- a/src/remote/connection.rs +++ b/src/remote/connection.rs @@ -23,7 +23,9 @@ use std::sync::{Arc, Mutex}; use anyhow::Result; use crate::remote::client::{OpenedImage, RemoteSession}; -use crate::remote::protocol::WireEntry; +use crate::remote::protocol::{ + WireEntry, WireOpticalDrive, WireRetryConfig, WireSectorMode, WireToc, +}; /// One live daemon connection, brokering many open-image handles on a single /// [`RemoteSession`]. @@ -159,6 +161,44 @@ impl RemoteConnection { self.session.close_block(handle) } + // --- optical tier (Family O): drive a remote CD/DVD for ripping --------- + + /// List the daemon machine's physical optical drives. + pub fn list_optical_drives(&mut self) -> Result> { + self.session.list_optical_drives() + } + + /// Open one of the daemon's optical drives for ripping; returns the handle. + pub fn open_optical(&mut self, path: &str, retry: WireRetryConfig) -> Result { + self.session.open_optical(path, retry) + } + + /// Read the open disc's table of contents. + pub fn read_toc(&mut self, handle: u64) -> Result { + self.session.read_toc(handle) + } + + /// Read `count` sectors from `lba` in `mode`; returns the raw sector bytes. + pub fn read_optical_sectors( + &mut self, + handle: u64, + lba: u32, + count: u32, + mode: WireSectorMode, + ) -> Result> { + self.session.read_optical_sectors(handle, lba, count, mode) + } + + /// Eject the disc from the open optical drive. + pub fn eject_optical(&mut self, handle: u64) -> Result<()> { + self.session.eject_optical(handle) + } + + /// Close an open optical handle, freeing the drive for the next session. + pub fn close_optical(&mut self, handle: u64) -> Result<()> { + self.session.close_optical(handle) + } + /// How many open-image handles this connection currently holds (diagnostics /// / tests). pub fn open_handle_count(&self) -> usize { diff --git a/src/remote/protocol.rs b/src/remote/protocol.rs index f1268974..0562b2dc 100644 --- a/src/remote/protocol.rs +++ b/src/remote/protocol.rs @@ -62,6 +62,11 @@ pub const CAP_FAMILY_F: u16 = 1 << 0; /// 7a — [`read_handshake`] / [`write_binary_hello`]); the chunk/`.cbk` data /// protocol is not, so this bit is advertised only once that lands (7b). pub const CAP_FAMILY_B: u16 = 1 << 1; +/// Family O — optical-drive proxy: rip a remote CD/DVD where the daemon issues +/// the SCSI reads and the desktop does all the encoding (see +/// `docs/remote_ripping.md`). Advertised only when the daemon was built with the +/// `optical` feature. +pub const CAP_FAMILY_O: u16 = 1 << 2; /// Largest control-frame body we'll accept — guards against a hostile or /// corrupt length prefix. Control frames are small JSON; 8 MiB is generous. @@ -204,6 +209,36 @@ pub enum Request { force: bool, }, + // --- optical tier (Family O): proxy a physical CD/DVD drive so the --- + // --- desktop rips + encodes while the daemon only issues SCSI reads. --- + /// List the daemon machine's physical optical drives (reply: + /// `OpticalDrives`). The remote arm of the unified rip-device picker. + ListOpticalDrives, + /// Open one of the daemon's optical drives for ripping (reply: + /// `OpticalOpened{handle}`). `retry` is applied daemon-side, next to the + /// (often flaky) drive. The daemon serves ONE optical session at a time — a + /// second open returns `Error` ("optical drive busy") because `cd-da-reader` + /// holds a process-global drive handle. + OpenOptical { + path: String, + retry: WireRetryConfig, + }, + /// Read the open disc's table of contents (reply: `Toc`). + ReadToc { handle: u64 }, + /// Read `count` sectors from `lba` in `mode` (reply: `FileBegin{size: + /// count * sector_size}`, then a chunk stream of that many bytes). Retry / + /// backoff is applied daemon-side per the `OpenOptical` policy. + ReadOpticalSectors { + handle: u64, + lba: u32, + count: u32, + mode: WireSectorMode, + }, + /// Eject the disc from the open optical drive (reply: `Ok`). + EjectOptical { handle: u64 }, + /// Close an open optical handle, freeing the drive for the next session. + CloseOptical { handle: u64 }, + /// Polite disconnect. Bye, } @@ -241,6 +276,12 @@ pub enum Response { BlockOpened { handle: u64, size: u64 }, /// `ListDevices` result: the daemon machine's physical disks. Devices { devices: Vec }, + /// `OpenOptical` result: the optical session handle. + OpticalOpened { handle: u64 }, + /// `ReadToc` result: the disc's table of contents. + Toc { toc: WireToc }, + /// `ListOpticalDrives` result: the daemon machine's optical drives. + OpticalDrives { drives: Vec }, /// Generic success (e.g. `Close`). Ok, /// Operation failed with a human-readable message. @@ -340,6 +381,174 @@ impl WireDevice { } } +// --------------------------------------------------------------------------- +// Optical tier DTOs (serde mirrors of the `cd-da-reader` types) +// +// Defined unconditionally under the `remote` feature so the protocol enums +// always compile; the `From`/`Into` conversions to the real `cd-da-reader` +// types are gated behind `optical` (only the desktop client and an +// optical-built daemon link that crate). +// --------------------------------------------------------------------------- + +/// Sector read mode — serde mirror of `cd_da_reader::SectorReadMode`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum WireSectorMode { + Audio, + DataCooked, + DataRaw, +} + +impl WireSectorMode { + /// Bytes per sector (2048 cooked, 2352 raw/audio). + pub fn sector_size(self) -> u32 { + match self { + WireSectorMode::DataCooked => 2048, + WireSectorMode::Audio | WireSectorMode::DataRaw => 2352, + } + } +} + +/// One TOC track — serde mirror of `cd_da_reader::Track`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WireTrack { + pub number: u8, + pub start_lba: u32, + pub start_msf: (u8, u8, u8), + pub is_audio: bool, +} + +/// Disc table of contents — serde mirror of `cd_da_reader::Toc`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WireToc { + pub first_track: u8, + pub last_track: u8, + pub tracks: Vec, + pub leadout_lba: u32, +} + +/// Read-retry policy sent at `OpenOptical` and applied daemon-side — serde +/// mirror of `cd_da_reader::RetryConfig`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WireRetryConfig { + pub max_attempts: u8, + pub initial_backoff_ms: u64, + pub max_backoff_ms: u64, + pub reduce_chunk_on_retry: bool, + pub min_sectors_per_read: u32, +} + +/// A physical optical drive on the daemon machine — the remote arm of the +/// unified rip-device picker (`ListOpticalDrives`). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WireOpticalDrive { + pub device_path: String, + pub display_name: String, + pub has_audio_cd: bool, +} + +#[cfg(feature = "optical")] +mod optical_conv { + use super::{WireOpticalDrive, WireRetryConfig, WireSectorMode, WireToc, WireTrack}; + + impl From<&cd_da_reader::Track> for WireTrack { + fn from(t: &cd_da_reader::Track) -> Self { + Self { + number: t.number, + start_lba: t.start_lba, + start_msf: t.start_msf, + is_audio: t.is_audio, + } + } + } + + impl From<&WireTrack> for cd_da_reader::Track { + fn from(t: &WireTrack) -> Self { + cd_da_reader::Track { + number: t.number, + start_lba: t.start_lba, + start_msf: t.start_msf, + is_audio: t.is_audio, + } + } + } + + impl From<&cd_da_reader::Toc> for WireToc { + fn from(toc: &cd_da_reader::Toc) -> Self { + Self { + first_track: toc.first_track, + last_track: toc.last_track, + tracks: toc.tracks.iter().map(WireTrack::from).collect(), + leadout_lba: toc.leadout_lba, + } + } + } + + impl From<&WireToc> for cd_da_reader::Toc { + fn from(toc: &WireToc) -> Self { + cd_da_reader::Toc { + first_track: toc.first_track, + last_track: toc.last_track, + tracks: toc.tracks.iter().map(cd_da_reader::Track::from).collect(), + leadout_lba: toc.leadout_lba, + } + } + } + + impl From for WireSectorMode { + fn from(m: cd_da_reader::SectorReadMode) -> Self { + match m { + cd_da_reader::SectorReadMode::Audio => WireSectorMode::Audio, + cd_da_reader::SectorReadMode::DataCooked => WireSectorMode::DataCooked, + cd_da_reader::SectorReadMode::DataRaw => WireSectorMode::DataRaw, + } + } + } + + impl From for cd_da_reader::SectorReadMode { + fn from(m: WireSectorMode) -> Self { + match m { + WireSectorMode::Audio => cd_da_reader::SectorReadMode::Audio, + WireSectorMode::DataCooked => cd_da_reader::SectorReadMode::DataCooked, + WireSectorMode::DataRaw => cd_da_reader::SectorReadMode::DataRaw, + } + } + } + + impl From<&cd_da_reader::RetryConfig> for WireRetryConfig { + fn from(c: &cd_da_reader::RetryConfig) -> Self { + Self { + max_attempts: c.max_attempts, + initial_backoff_ms: c.initial_backoff_ms, + max_backoff_ms: c.max_backoff_ms, + reduce_chunk_on_retry: c.reduce_chunk_on_retry, + min_sectors_per_read: c.min_sectors_per_read, + } + } + } + + impl From<&WireRetryConfig> for cd_da_reader::RetryConfig { + fn from(c: &WireRetryConfig) -> Self { + cd_da_reader::RetryConfig { + max_attempts: c.max_attempts, + initial_backoff_ms: c.initial_backoff_ms, + max_backoff_ms: c.max_backoff_ms, + reduce_chunk_on_retry: c.reduce_chunk_on_retry, + min_sectors_per_read: c.min_sectors_per_read, + } + } + } + + impl From<&cd_da_reader::DriveInfo> for WireOpticalDrive { + fn from(d: &cd_da_reader::DriveInfo) -> Self { + Self { + device_path: d.path.clone(), + display_name: d.display_name.clone().unwrap_or_else(|| d.path.clone()), + has_audio_cd: d.has_audio_cd, + } + } + } +} + // --------------------------------------------------------------------------- // Framing — control frames // --------------------------------------------------------------------------- @@ -1020,6 +1229,113 @@ mod tests { } } + fn sample_toc() -> WireToc { + WireToc { + first_track: 1, + last_track: 2, + tracks: vec![ + WireTrack { + number: 1, + start_lba: 0, + start_msf: (0, 2, 0), + is_audio: false, + }, + WireTrack { + number: 2, + start_lba: 15000, + start_msf: (3, 22, 0), + is_audio: true, + }, + ], + leadout_lba: 45000, + } + } + + /// Optical-tier requests/responses + DTOs round-trip through serde_json. + /// (`Request`/`Response` don't derive `PartialEq`, so compare Debug strings.) + #[test] + fn optical_protocol_round_trips() { + let reqs = vec![ + Request::ListOpticalDrives, + Request::OpenOptical { + path: "/dev/sr0".into(), + retry: WireRetryConfig { + max_attempts: 4, + initial_backoff_ms: 20, + max_backoff_ms: 300, + reduce_chunk_on_retry: true, + min_sectors_per_read: 1, + }, + }, + Request::ReadToc { handle: 7 }, + Request::ReadOpticalSectors { + handle: 7, + lba: 16, + count: 128, + mode: WireSectorMode::DataRaw, + }, + Request::EjectOptical { handle: 7 }, + Request::CloseOptical { handle: 7 }, + ]; + for r in &reqs { + let s = serde_json::to_string(r).unwrap(); + let back: Request = serde_json::from_str(&s).unwrap(); + assert_eq!(format!("{r:?}"), format!("{back:?}")); + } + + let resps = vec![ + Response::OpticalOpened { handle: 7 }, + Response::Toc { toc: sample_toc() }, + Response::OpticalDrives { + drives: vec![WireOpticalDrive { + device_path: "/dev/sr0".into(), + display_name: "TSSTcorp CD/DVDW".into(), + has_audio_cd: false, + }], + }, + ]; + for r in &resps { + let s = serde_json::to_string(r).unwrap(); + let back: Response = serde_json::from_str(&s).unwrap(); + assert_eq!(format!("{r:?}"), format!("{back:?}")); + } + + assert_eq!(WireSectorMode::DataCooked.sector_size(), 2048); + assert_eq!(WireSectorMode::DataRaw.sector_size(), 2352); + assert_eq!(WireSectorMode::Audio.sector_size(), 2352); + } + + /// The DTOs convert to the real `cd-da-reader` types and back losslessly. + #[cfg(feature = "optical")] + #[test] + fn optical_dtos_convert_to_cd_da_reader() { + let wire = sample_toc(); + let native: cd_da_reader::Toc = (&wire).into(); + let back: WireToc = (&native).into(); + assert_eq!(format!("{wire:?}"), format!("{back:?}")); + + for m in [ + WireSectorMode::Audio, + WireSectorMode::DataCooked, + WireSectorMode::DataRaw, + ] { + let native: cd_da_reader::SectorReadMode = m.into(); + let back: WireSectorMode = native.into(); + assert_eq!(m, back); + } + + let retry = WireRetryConfig { + max_attempts: 3, + initial_backoff_ms: 10, + max_backoff_ms: 200, + reduce_chunk_on_retry: false, + min_sectors_per_read: 2, + }; + let native: cd_da_reader::RetryConfig = (&retry).into(); + let back: WireRetryConfig = (&native).into(); + assert_eq!(format!("{retry:?}"), format!("{back:?}")); + } + /// The resume-map reply's skip flag + entries round-trip (7h). #[test] fn resume_map_skip_flag_round_trips() { diff --git a/src/remote/server.rs b/src/remote/server.rs index 6f12bcdd..8d3b8420 100644 --- a/src/remote/server.rs +++ b/src/remote/server.rs @@ -760,6 +760,11 @@ fn handle_conn(stream: TcpStream, root: &Path, staging_dir: Option<&Path>) -> Re let mut block_handles: HashMap = HashMap::new(); let mut sessions: HashMap = HashMap::new(); let mut next_session: u64 = 1; + // Optical tier: at most one drive session per connection (and, via a + // process-global guard inside `optical_server`, one per daemon — cd-da-reader + // keeps a global drive handle). + #[cfg(feature = "optical")] + let mut optical = optical_server::OpticalState::default(); // The opening frame disambiguates a JSON-free Family-B client (cb-dos — // leads with the binary magic) from a JSON Family-F client (desktop). A @@ -807,11 +812,15 @@ fn handle_conn(stream: TcpStream, root: &Path, staging_dir: Option<&Path>) -> Re )?; return Ok(()); } + #[cfg(feature = "optical")] + let capabilities = CAP_FAMILY_F | crate::remote::protocol::CAP_FAMILY_O; + #[cfg(not(feature = "optical"))] + let capabilities = CAP_FAMILY_F; write_control( &mut writer, &Response::Hello { version: PROTOCOL_VERSION, - capabilities: CAP_FAMILY_F, + capabilities, platform: std::env::consts::OS.to_string(), }, )?; @@ -1183,6 +1192,82 @@ fn handle_conn(stream: TcpStream, root: &Path, staging_dir: Option<&Path>) -> Re None => reply_err(&mut writer, format!("no such session {session}"))?, }, + // --- optical tier (Family O): proxy a physical CD/DVD drive --- + Request::ListOpticalDrives => { + #[cfg(feature = "optical")] + optical.list_drives(&mut writer)?; + #[cfg(not(feature = "optical"))] + reply_err( + &mut writer, + "daemon built without the optical feature".to_string(), + )?; + } + Request::OpenOptical { path, retry } => { + #[cfg(feature = "optical")] + optical.open(&path, &retry, &mut next_handle, &mut writer)?; + #[cfg(not(feature = "optical"))] + { + let _ = (&path, &retry); + reply_err( + &mut writer, + "daemon built without the optical feature".to_string(), + )?; + } + } + Request::ReadToc { handle } => { + #[cfg(feature = "optical")] + optical.read_toc(handle, &mut writer)?; + #[cfg(not(feature = "optical"))] + { + let _ = handle; + reply_err( + &mut writer, + "daemon built without the optical feature".to_string(), + )?; + } + } + Request::ReadOpticalSectors { + handle, + lba, + count, + mode, + } => { + #[cfg(feature = "optical")] + optical.read_sectors(handle, lba, count, mode, &mut writer)?; + #[cfg(not(feature = "optical"))] + { + let _ = (handle, lba, count, mode); + reply_err( + &mut writer, + "daemon built without the optical feature".to_string(), + )?; + } + } + Request::EjectOptical { handle } => { + #[cfg(feature = "optical")] + optical.eject(handle, &mut writer)?; + #[cfg(not(feature = "optical"))] + { + let _ = handle; + reply_err( + &mut writer, + "daemon built without the optical feature".to_string(), + )?; + } + } + Request::CloseOptical { handle } => { + #[cfg(feature = "optical")] + optical.close(handle, &mut writer)?; + #[cfg(not(feature = "optical"))] + { + let _ = handle; + reply_err( + &mut writer, + "daemon built without the optical feature".to_string(), + )?; + } + } + Request::Bye => return Ok(()), } } @@ -1194,6 +1279,163 @@ fn reply_err(writer: &mut W, message: String) -> Result<()> { Ok(()) } +/// Optical-drive proxy (Family O). Wraps a [`crate::optical::source::LocalCdReader`] +/// on the daemon so a desktop client rips a remote drive while the daemon only +/// issues SCSI reads — all encoding stays on the desktop. See +/// `docs/remote_ripping.md`. +#[cfg(feature = "optical")] +mod optical_server { + use std::io::Write; + use std::sync::atomic::{AtomicBool, Ordering}; + + use anyhow::Result; + + use super::reply_err; + use crate::optical::source::{LocalCdReader, OpticalSource}; + use crate::remote::protocol::{ + write_control, ChunkWriter, Response, WireOpticalDrive, WireRetryConfig, WireSectorMode, + WireToc, + }; + + /// At most one optical session exists per process: `cd-da-reader` keeps a + /// global drive handle, so a second open (even on another connection) would + /// clobber the first. The guard is released on drop — including connection + /// teardown, since the per-connection `OpticalState` owns the session. + static BUSY: AtomicBool = AtomicBool::new(false); + + struct OpticalSession { + reader: LocalCdReader, + handle: u64, + } + + impl Drop for OpticalSession { + fn drop(&mut self) { + BUSY.store(false, Ordering::SeqCst); + } + } + + /// Per-connection optical state: at most one open session. + #[derive(Default)] + pub struct OpticalState { + session: Option, + } + + impl OpticalState { + pub fn list_drives(&self, writer: &mut W) -> Result<()> { + match cd_da_reader::CdReader::list_drives() { + Ok(drives) => { + let drives = drives.iter().map(WireOpticalDrive::from).collect(); + write_control(writer, &Response::OpticalDrives { drives })?; + } + Err(e) => reply_err(writer, format!("list optical drives: {e}"))?, + } + Ok(()) + } + + pub fn open( + &mut self, + path: &str, + retry: &WireRetryConfig, + next_handle: &mut u64, + writer: &mut W, + ) -> Result<()> { + // Acquire the process-global slot before touching the drive. + if BUSY + .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) + .is_err() + { + return reply_err( + writer, + "optical drive busy (another rip is in progress)".to_string(), + ); + } + match LocalCdReader::with_retry(path, retry.into()) { + Ok(reader) => { + let handle = *next_handle; + *next_handle += 1; + self.session = Some(OpticalSession { reader, handle }); + write_control(writer, &Response::OpticalOpened { handle })?; + } + Err(e) => { + BUSY.store(false, Ordering::SeqCst); + reply_err(writer, format!("open optical drive {path}: {e:#}"))?; + } + } + Ok(()) + } + + fn session(&self, handle: u64) -> Option<&OpticalSession> { + self.session.as_ref().filter(|s| s.handle == handle) + } + + pub fn read_toc(&self, handle: u64, writer: &mut W) -> Result<()> { + let Some(s) = self.session(handle) else { + return reply_err(writer, format!("no such optical handle {handle}")); + }; + match s.reader.read_toc() { + Ok(toc) => write_control( + writer, + &Response::Toc { + toc: WireToc::from(&toc), + }, + )?, + Err(e) => reply_err(writer, format!("read TOC: {e:#}"))?, + } + Ok(()) + } + + pub fn read_sectors( + &self, + handle: u64, + lba: u32, + count: u32, + mode: WireSectorMode, + writer: &mut W, + ) -> Result<()> { + let Some(s) = self.session(handle) else { + return reply_err(writer, format!("no such optical handle {handle}")); + }; + match s.reader.read_data_sectors(lba, count, mode.into()) { + Ok(data) => { + // Commit to the stream: FileBegin{actual len} then the bytes. + write_control( + writer, + &Response::FileBegin { + size: data.len() as u64, + }, + )?; + let mut cw = ChunkWriter::new(&mut *writer); + cw.write_all(&data)?; + cw.finish()?; + } + Err(e) => reply_err(writer, format!("read sectors at LBA {lba}: {e:#}"))?, + } + Ok(()) + } + + pub fn eject(&self, handle: u64, writer: &mut W) -> Result<()> { + let Some(s) = self.session(handle) else { + return reply_err(writer, format!("no such optical handle {handle}")); + }; + match s.reader.eject() { + Ok(()) => write_control(writer, &Response::Ok)?, + Err(e) => reply_err(writer, format!("eject: {e:#}"))?, + } + Ok(()) + } + + pub fn close(&mut self, handle: u64, writer: &mut W) -> Result<()> { + if self.session.as_ref().map(|s| s.handle) == Some(handle) { + self.session = None; // drops the session -> releases the global slot + write_control(writer, &Response::Ok)?; + } else { + reply_err(writer, format!("no such optical handle {handle}"))?; + } + Ok(()) + } + } +} + /// Read a chunk stream from the client into a staging blob file. fn stage_blob(reader: &mut R, blob: &Path) -> Result<()> { let mut f = std::fs::File::create(blob) diff --git a/src/update.rs b/src/update.rs index 577c9c93..eeab8e7b 100644 --- a/src/update.rs +++ b/src/update.rs @@ -25,6 +25,11 @@ pub struct UpdateConfig { /// so a self-update that adds an extension registers it without a reinstall. #[serde(default)] pub assoc_registered_version: Option, + /// Most-recently-used rb-daemon addresses (`host:port`), newest first. Drives + /// the GUI Optical tab's "Add remote daemon" quick-pick list. Capped to a + /// handful of entries. + #[serde(default)] + pub recent_daemon_addrs: Vec, } #[derive(Debug, Deserialize, Serialize, Clone)] @@ -90,6 +95,7 @@ impl Default for UpdateConfig { last_chd_hunk_size: None, file_associations_enabled: false, assoc_registered_version: None, + recent_daemon_addrs: Vec::new(), } } } @@ -162,6 +168,18 @@ impl UpdateConfig { let config: UpdateConfig = serde_json::from_str(&content)?; Ok(config) } + + /// Record a successfully-used rb-daemon address in the MRU list (dedup, + /// newest first, capped). No-op for a blank address. + pub fn remember_daemon(&mut self, addr: &str) { + let addr = addr.trim(); + if addr.is_empty() { + return; + } + self.recent_daemon_addrs.retain(|a| a != addr); + self.recent_daemon_addrs.insert(0, addr.to_string()); + self.recent_daemon_addrs.truncate(8); + } } #[derive(Debug, Clone)] @@ -364,3 +382,27 @@ pub fn restart_app() -> ! { } std::process::exit(0); } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn remember_daemon_is_mru_deduped_capped() { + let mut cfg = UpdateConfig::default(); + cfg.remember_daemon("a:1"); + cfg.remember_daemon("b:2"); + // Re-using an address moves it to the front (no duplicate). + cfg.remember_daemon("a:1"); + assert_eq!(cfg.recent_daemon_addrs, vec!["a:1", "b:2"]); + // Blank / whitespace is ignored. + cfg.remember_daemon(" "); + assert_eq!(cfg.recent_daemon_addrs.len(), 2); + // Capped at 8, newest first. + for i in 0..10 { + cfg.remember_daemon(&format!("h:{i}")); + } + assert_eq!(cfg.recent_daemon_addrs.len(), 8); + assert_eq!(cfg.recent_daemon_addrs[0], "h:9"); + } +} diff --git a/tests/remote_optical.rs b/tests/remote_optical.rs new file mode 100644 index 00000000..2ca14ac0 --- /dev/null +++ b/tests/remote_optical.rs @@ -0,0 +1,89 @@ +//! Loopback plumbing test for the optical tier (Family O). +//! +//! Drives the client↔daemon optical path over a port-0 `rb-cli serve` listener +//! WITHOUT a physical drive: the daemon (built with `optical`) must handle the +//! optical verbs, opening a bogus device must error cleanly, and the +//! process-global busy guard must be released so a second open isn't wrongly +//! reported "busy". The full rip-a-real-disc validation needs hardware (P1.8 in +//! docs/remote_ripping.md). +#![cfg(all(feature = "remote", feature = "optical"))] + +use std::net::TcpListener; + +use rusty_backup::remote::protocol::WireRetryConfig; +use rusty_backup::remote::{serve_on, RemoteConnection}; + +/// Bind a port-0 listener (so the connect lands in the listen backlog with no +/// sleep/race) and spawn the daemon on it. The returned `TempDir` guard must be +/// kept alive for the daemon's lifetime. +fn spawn_daemon() -> (String, tempfile::TempDir) { + let dir = tempfile::tempdir().unwrap(); + let root = dir.path().canonicalize().unwrap(); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap().to_string(); + std::thread::spawn(move || { + let _ = serve_on(listener, root, None); + }); + (addr, dir) +} + +fn test_retry() -> WireRetryConfig { + WireRetryConfig { + max_attempts: 1, + initial_backoff_ms: 0, + max_backoff_ms: 0, + reduce_chunk_on_retry: false, + min_sectors_per_read: 1, + } +} + +/// The daemon (built with `optical`) handles `ListOpticalDrives` — it round-trips +/// over the wire instead of disclaiming the feature. +#[test] +fn optical_list_drives_round_trips_over_loopback() { + let (addr, _dir) = spawn_daemon(); + let mut conn = RemoteConnection::connect(&addr).unwrap(); + // On a driveless CI box the list may be empty (Ok) or enumeration may error + // (Err) per platform — both prove the wire path works. What must NOT happen + // is the daemon claiming it lacks the optical feature. + if let Err(e) = conn.list_optical_drives() { + let msg = format!("{e:#}"); + assert!( + !msg.contains("built without the optical feature"), + "an optical-built daemon should not disclaim the feature: {msg}" + ); + } +} + +/// The unified picker's remote arm enumerates a daemon's drives over the wire. +/// On a driveless daemon it adds zero — but without erroring or panicking, +/// proving `append_remote_rip_devices` round-trips. +#[test] +fn remote_rip_device_enumeration_over_loopback() { + let (addr, _dir) = spawn_daemon(); + let conn = RemoteConnection::connect_shared(&addr).unwrap(); + let mut out = Vec::new(); + let added = rusty_backup::model::optical_devices::append_remote_rip_devices(&mut out, &conn); + assert_eq!(added, out.len(), "reported count must match pushed entries"); + // (On a box with a real drive this would be > 0, each labeled with the addr.) +} + +/// Opening a device that cannot exist errors cleanly, and the process-global +/// busy guard is released so a *second* open fails at open — not with "busy". +#[test] +fn optical_open_bogus_device_errors_and_releases_busy_guard() { + let (addr, _dir) = spawn_daemon(); + let mut conn = RemoteConnection::connect(&addr).unwrap(); + + let first = conn.open_optical("/dev/rb-test-no-such-optical-xyz", test_retry()); + assert!(first.is_err(), "opening a bogus device should error"); + + let err = conn + .open_optical("/dev/rb-test-no-such-optical-xyz", test_retry()) + .expect_err("second open of a bogus device should still error"); + let msg = format!("{err:#}").to_lowercase(); + assert!( + !msg.contains("busy"), + "busy guard was not released after a failed open: {msg}" + ); +}