|
13 | 13 | //! |
14 | 14 | //! The catalog uses a two-tier consistency model: |
15 | 15 | //! |
16 | | -//! - **Tier 1 (Strong Consistency)**: Low-frequency DDL operations (create/drop) |
17 | | -//! use atomic snapshot + manifest writes for immediate consistency |
| 16 | +//! - **Tier 1 (Strong Consistency)**: API mutations append tier1 DDL events and |
| 17 | +//! then synchronously compact/publish immutable manifest snapshots via |
| 18 | +//! `CatalogWriter` + `SyncCompactor` |
18 | 19 | //! - **Tier 2 (Eventual Consistency)**: High-volume operational facts (lineage events, |
19 | 20 | //! run metadata) are written to an append-only event log and periodically compacted |
20 | 21 | //! |
| 22 | +//! `Tier1Writer` still exists for low-level/bootstrap flows and tests, but it is no |
| 23 | +//! longer the primary API write path. |
| 24 | +//! |
21 | 25 | //! ## Storage Layout |
22 | 26 | //! |
23 | 27 | //! All catalog data is scoped to tenant/workspace isolation boundaries using |
24 | 28 | //! key=value path segments (grep-friendly, self-documenting): |
25 | 29 | //! |
26 | 30 | //! ```text |
27 | 31 | //! tenant={tenant}/workspace={workspace}/ |
28 | | -//! ├── manifests/ # Tier 1: Multi-file manifest structure |
29 | | -//! │ ├── root.manifest.json # Root manifest (points to domain manifests) |
30 | | -//! │ ├── catalog.manifest.json # Catalog state (locked writes) |
31 | | -//! │ ├── lineage.manifest.json # Lineage state (locked writes) |
32 | | -//! │ ├── executions.manifest.json # Execution state (compactor writes) |
33 | | -//! │ └── search.manifest.json # Search state (locked writes) |
| 32 | +//! ├── manifests/ |
| 33 | +//! │ ├── root.manifest.json # Stable entrypoint for readers |
| 34 | +//! │ ├── catalog.manifest.json # Legacy compatibility mirror |
| 35 | +//! │ ├── catalog.pointer.json # Immutable snapshot visibility gate |
| 36 | +//! │ ├── catalog/{manifest_id}.json # Immutable catalog snapshots |
| 37 | +//! │ ├── lineage.manifest.json # Legacy compatibility mirror |
| 38 | +//! │ ├── lineage.pointer.json |
| 39 | +//! │ ├── lineage/{manifest_id}.json |
| 40 | +//! │ ├── search.manifest.json # Legacy compatibility mirror |
| 41 | +//! │ ├── search.pointer.json |
| 42 | +//! │ └── search/{manifest_id}.json |
34 | 43 | //! ├── locks/ |
35 | 44 | //! │ ├── catalog.lock.json # Distributed lock per domain |
36 | 45 | //! │ ├── lineage.lock.json |
|
47 | 56 | //! ## Example |
48 | 57 | //! |
49 | 58 | //! ```rust,ignore |
50 | | -//! use arco_catalog::Tier1Writer; |
| 59 | +//! use std::sync::Arc; |
| 60 | +//! |
| 61 | +//! use arco_catalog::{CatalogWriter, Tier1Compactor}; |
51 | 62 | //! use arco_core::ScopedStorage; |
52 | 63 | //! |
53 | 64 | //! // Create workspace-scoped storage |
54 | 65 | //! let storage = ScopedStorage::new(backend, "acme-corp", "production")?; |
| 66 | +//! let compactor = Arc::new(Tier1Compactor::new(storage.clone())); |
55 | 67 | //! |
56 | | -//! // Initialize catalog manifests (idempotent) |
57 | | -//! let writer = Tier1Writer::new(storage); |
58 | | -//! writer.initialize().await?; |
59 | | -//! |
60 | | -//! // Update catalog with CAS semantics |
61 | | -//! let commit = writer.update(|manifest| { |
62 | | -//! manifest.snapshot_version = 1; |
63 | | -//! Ok(()) |
64 | | -//! }).await?; |
| 68 | +//! // API writes go through the facade and publish immutable snapshots synchronously. |
| 69 | +//! let writer = CatalogWriter::new(storage).with_sync_compactor(compactor); |
| 70 | +//! // writer.create_namespace(...).await?; |
65 | 71 | //! ``` |
66 | 72 |
|
67 | 73 | #![forbid(unsafe_code)] |
|
0 commit comments