From b9aa89bf5a975a58d21a7cb07158a86a56da1734 Mon Sep 17 00:00:00 2001 From: Lorenzo Delgado Date: Tue, 23 Dec 2025 13:29:29 +0100 Subject: [PATCH] feat(amp-data-store): introduce crate with unified store interface Consolidate data storage and caching into single abstraction to simplify architecture and eliminate the cached/uncached store distinction. - Unified `DataStore` combines storage and cache in single type - Integrated parquet metadata cache with memory-aware eviction - Centralized physical table operations and file naming - Single store interface across all services and modules Signed-off-by: Lorenzo Delgado --- Cargo.lock | 965 ++++++++++-------- Cargo.toml | 1 + crates/bin/ampd/Cargo.toml | 1 + crates/bin/ampd/src/controller_cmd.rs | 11 +- crates/bin/ampd/src/server_cmd.rs | 12 +- crates/bin/ampd/src/solo_cmd.rs | 11 +- crates/bin/ampd/src/worker_cmd.rs | 10 +- crates/config/src/lib.rs | 1 - crates/core/common/Cargo.toml | 5 +- crates/core/common/src/catalog/physical.rs | 301 +----- crates/core/common/src/catalog/reader.rs | 16 +- crates/core/common/src/catalog/sql.rs | 5 +- crates/core/common/src/lib.rs | 7 - crates/core/common/src/metadata.rs | 15 +- crates/core/common/src/metadata/parquet.rs | 6 +- crates/core/common/src/metadata/size.rs | 7 +- crates/core/common/src/query_context.rs | 24 +- crates/core/data-store/Cargo.toml | 21 + .../metadata => data-store/src}/file_name.rs | 3 +- .../src/store.rs => data-store/src/lib.rs} | 193 ++-- crates/core/data-store/src/physical_table.rs | 265 +++++ crates/core/datasets-derived/Cargo.toml | 1 + crates/core/datasets-derived/src/catalog.rs | 7 +- crates/core/dump/Cargo.toml | 1 + crates/core/dump/src/check.rs | 9 +- crates/core/dump/src/compaction.rs | 13 +- crates/core/dump/src/compaction/collector.rs | 9 +- crates/core/dump/src/compaction/compactor.rs | 18 +- crates/core/dump/src/compaction/plan.rs | 6 +- crates/core/dump/src/config.rs | 1 - crates/core/dump/src/derived_dataset.rs | 5 +- crates/core/dump/src/lib.rs | 2 +- crates/core/dump/src/parquet_writer.rs | 9 +- crates/core/dump/src/raw_dataset.rs | 5 +- crates/core/dump/src/raw_dataset_writer.rs | 11 +- crates/core/dump/src/streaming_query.rs | 23 +- crates/core/monitoring/src/logging.rs | 1 + crates/services/admin-api/Cargo.toml | 1 + crates/services/admin-api/src/ctx.rs | 4 +- crates/services/controller/Cargo.toml | 1 + crates/services/controller/src/service.rs | 5 +- crates/services/server/Cargo.toml | 1 + crates/services/server/src/config.rs | 3 - crates/services/server/src/flight.rs | 12 +- crates/services/server/src/service.rs | 5 +- crates/services/worker/Cargo.toml | 1 + crates/services/worker/src/service.rs | 10 +- .../services/worker/src/service/job_impl.rs | 6 +- tests/Cargo.toml | 1 + tests/src/main.rs | 15 +- tests/src/testlib/ctx.rs | 6 +- .../src/testlib/fixtures/daemon_controller.rs | 15 +- tests/src/testlib/fixtures/daemon_server.rs | 10 +- tests/src/testlib/fixtures/daemon_worker.rs | 9 +- tests/src/testlib/fixtures/snapshot_ctx.rs | 5 +- tests/src/testlib/helpers.rs | 11 +- tests/src/tests/it_dump.rs | 4 +- tests/src/tests/it_sql_dataset_batch_size.rs | 18 +- 58 files changed, 1114 insertions(+), 1030 deletions(-) create mode 100644 crates/core/data-store/Cargo.toml rename crates/core/{common/src/metadata => data-store/src}/file_name.rs (99%) rename crates/core/{common/src/store.rs => data-store/src/lib.rs} (93%) create mode 100644 crates/core/data-store/src/physical_table.rs diff --git a/Cargo.lock b/Cargo.lock index 35614b03f..902f55149 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -29,9 +29,10 @@ name = "admin-api" version = "0.1.0" dependencies = [ "amp-config", + "amp-data-store", "amp-dataset-store", "async-trait", - "axum 0.8.7", + "axum 0.8.8", "common", "datafusion", "datasets-common", @@ -134,9 +135,9 @@ dependencies = [ [[package]] name = "agave-feature-set" -version = "3.1.4" +version = "3.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "401c88cbf88986ff413b1f582659facfb1d1d6e53b373c9df8ecc4ef8562370f" +checksum = "6d4a155f30015eab7e4a9ba2bcf9e1078c70707230668dc577e41830d97d0313" dependencies = [ "ahash", "solana-epoch-schedule", @@ -148,9 +149,9 @@ dependencies = [ [[package]] name = "agave-reserved-account-keys" -version = "3.1.4" +version = "3.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "997e580e670daab5b0e26331b292e0dd06771d3dd732d58393cea1f52f8187a1" +checksum = "ef75a91b94d212ebf507b65592d1251f85fc3eb719dc94021e6a508144450ab4" dependencies = [ "agave-feature-set", "solana-pubkey 3.0.0", @@ -203,8 +204,8 @@ checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "alloy" -version = "1.0.41" -source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#8ce8a3d52a5b50e9f0cf16fac91888445be6009b" +version = "1.1.3" +source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#c4c5d5b8206d1becb1b83eee563b38cce0ea53a7" dependencies = [ "alloy-consensus", "alloy-contract", @@ -229,9 +230,9 @@ dependencies = [ [[package]] name = "alloy-chains" -version = "0.2.23" +version = "0.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35d744058a9daa51a8cf22a3009607498fcf82d3cf4c5444dd8056cdf651f471" +checksum = "dd208e8a87fbc2ca1a3822dd1ea03b0a7a4a841e6fa70db2c236dd30ae2e7018" dependencies = [ "alloy-primitives", "num_enum", @@ -240,8 +241,8 @@ dependencies = [ [[package]] name = "alloy-consensus" -version = "1.0.41" -source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#8ce8a3d52a5b50e9f0cf16fac91888445be6009b" +version = "1.1.3" +source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#c4c5d5b8206d1becb1b83eee563b38cce0ea53a7" dependencies = [ "alloy-eips", "alloy-primitives", @@ -250,6 +251,7 @@ dependencies = [ "alloy-trie", "alloy-tx-macros", "auto_impl", + "borsh", "c-kzg", "derive_more", "either", @@ -265,8 +267,8 @@ dependencies = [ [[package]] name = "alloy-consensus-any" -version = "1.0.41" -source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#8ce8a3d52a5b50e9f0cf16fac91888445be6009b" +version = "1.1.3" +source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#c4c5d5b8206d1becb1b83eee563b38cce0ea53a7" dependencies = [ "alloy-consensus", "alloy-eips", @@ -278,8 +280,8 @@ dependencies = [ [[package]] name = "alloy-contract" -version = "1.0.41" -source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#8ce8a3d52a5b50e9f0cf16fac91888445be6009b" +version = "1.1.3" +source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#c4c5d5b8206d1becb1b83eee563b38cce0ea53a7" dependencies = [ "alloy-consensus", "alloy-dyn-abi", @@ -300,21 +302,22 @@ dependencies = [ [[package]] name = "alloy-core" -version = "1.4.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca96214615ec8cf3fa2a54b32f486eb49100ca7fe7eb0b8c1137cd316e7250a" +checksum = "9d4087016b0896051dd3d03e0bedda2f4d4d1689af8addc8450288c63a9e5f68" dependencies = [ "alloy-dyn-abi", "alloy-json-abi", "alloy-primitives", + "alloy-rlp", "alloy-sol-types", ] [[package]] name = "alloy-dyn-abi" -version = "1.4.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdff496dd4e98a81f4861e66f7eaf5f2488971848bb42d9c892f871730245c8" +checksum = "369f5707b958927176265e8a58627fc6195e5dfa5c55689396e68b241b3a72e6" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -366,8 +369,8 @@ dependencies = [ [[package]] name = "alloy-eips" -version = "1.0.41" -source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#8ce8a3d52a5b50e9f0cf16fac91888445be6009b" +version = "1.1.3" +source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#c4c5d5b8206d1becb1b83eee563b38cce0ea53a7" dependencies = [ "alloy-eip2124", "alloy-eip2930", @@ -376,6 +379,7 @@ dependencies = [ "alloy-rlp", "alloy-serde", "auto_impl", + "borsh", "c-kzg", "derive_more", "either", @@ -387,8 +391,8 @@ dependencies = [ [[package]] name = "alloy-genesis" -version = "1.0.41" -source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#8ce8a3d52a5b50e9f0cf16fac91888445be6009b" +version = "1.1.3" +source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#c4c5d5b8206d1becb1b83eee563b38cce0ea53a7" dependencies = [ "alloy-eips", "alloy-primitives", @@ -412,9 +416,9 @@ dependencies = [ [[package]] name = "alloy-json-abi" -version = "1.4.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5513d5e6bd1cba6bdcf5373470f559f320c05c8c59493b6e98912fbe6733943f" +checksum = "84e3cf01219c966f95a460c95f1d4c30e12f6c18150c21a30b768af2a2a29142" dependencies = [ "alloy-primitives", "alloy-sol-type-parser", @@ -424,8 +428,8 @@ dependencies = [ [[package]] name = "alloy-json-rpc" -version = "1.0.41" -source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#8ce8a3d52a5b50e9f0cf16fac91888445be6009b" +version = "1.1.3" +source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#c4c5d5b8206d1becb1b83eee563b38cce0ea53a7" dependencies = [ "alloy-primitives", "alloy-sol-types", @@ -438,8 +442,8 @@ dependencies = [ [[package]] name = "alloy-network" -version = "1.0.41" -source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#8ce8a3d52a5b50e9f0cf16fac91888445be6009b" +version = "1.1.3" +source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#c4c5d5b8206d1becb1b83eee563b38cce0ea53a7" dependencies = [ "alloy-consensus", "alloy-consensus-any", @@ -463,8 +467,8 @@ dependencies = [ [[package]] name = "alloy-network-primitives" -version = "1.0.41" -source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#8ce8a3d52a5b50e9f0cf16fac91888445be6009b" +version = "1.1.3" +source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#c4c5d5b8206d1becb1b83eee563b38cce0ea53a7" dependencies = [ "alloy-consensus", "alloy-eips", @@ -475,8 +479,8 @@ dependencies = [ [[package]] name = "alloy-node-bindings" -version = "1.0.41" -source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#8ce8a3d52a5b50e9f0cf16fac91888445be6009b" +version = "1.1.3" +source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#c4c5d5b8206d1becb1b83eee563b38cce0ea53a7" dependencies = [ "alloy-genesis", "alloy-hardforks", @@ -495,9 +499,9 @@ dependencies = [ [[package]] name = "alloy-primitives" -version = "1.4.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "355bf68a433e0fd7f7d33d5a9fc2583fde70bf5c530f63b80845f8da5505cf28" +checksum = "f6a0fb18dd5fb43ec5f0f6a20be1ce0287c79825827de5744afaa6c957737c33" dependencies = [ "alloy-rlp", "bytes", @@ -506,13 +510,14 @@ dependencies = [ "derive_more", "foldhash 0.2.0", "hashbrown 0.16.1", - "indexmap 2.12.1", + "indexmap 2.13.0", "itoa", "k256", "keccak-asm", "paste", "proptest", "rand 0.9.2", + "rapidhash", "ruint", "rustc-hash", "serde", @@ -522,8 +527,8 @@ dependencies = [ [[package]] name = "alloy-provider" -version = "1.0.41" -source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#8ce8a3d52a5b50e9f0cf16fac91888445be6009b" +version = "1.1.3" +source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#c4c5d5b8206d1becb1b83eee563b38cce0ea53a7" dependencies = [ "alloy-chains", "alloy-consensus", @@ -565,8 +570,8 @@ dependencies = [ [[package]] name = "alloy-pubsub" -version = "1.0.41" -source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#8ce8a3d52a5b50e9f0cf16fac91888445be6009b" +version = "1.1.3" +source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#c4c5d5b8206d1becb1b83eee563b38cce0ea53a7" dependencies = [ "alloy-json-rpc", "alloy-primitives", @@ -603,13 +608,13 @@ checksum = "64b728d511962dda67c1bc7ea7c03736ec275ed2cf4c35d9585298ac9ccf3b73" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] name = "alloy-rpc-client" -version = "1.0.41" -source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#8ce8a3d52a5b50e9f0cf16fac91888445be6009b" +version = "1.1.3" +source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#c4c5d5b8206d1becb1b83eee563b38cce0ea53a7" dependencies = [ "alloy-json-rpc", "alloy-primitives", @@ -633,8 +638,8 @@ dependencies = [ [[package]] name = "alloy-rpc-types" -version = "1.0.41" -source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#8ce8a3d52a5b50e9f0cf16fac91888445be6009b" +version = "1.1.3" +source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#c4c5d5b8206d1becb1b83eee563b38cce0ea53a7" dependencies = [ "alloy-primitives", "alloy-rpc-types-anvil", @@ -645,8 +650,8 @@ dependencies = [ [[package]] name = "alloy-rpc-types-anvil" -version = "1.0.41" -source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#8ce8a3d52a5b50e9f0cf16fac91888445be6009b" +version = "1.1.3" +source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#c4c5d5b8206d1becb1b83eee563b38cce0ea53a7" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -656,8 +661,8 @@ dependencies = [ [[package]] name = "alloy-rpc-types-any" -version = "1.0.41" -source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#8ce8a3d52a5b50e9f0cf16fac91888445be6009b" +version = "1.1.3" +source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#c4c5d5b8206d1becb1b83eee563b38cce0ea53a7" dependencies = [ "alloy-consensus-any", "alloy-rpc-types-eth", @@ -666,8 +671,8 @@ dependencies = [ [[package]] name = "alloy-rpc-types-eth" -version = "1.0.41" -source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#8ce8a3d52a5b50e9f0cf16fac91888445be6009b" +version = "1.1.3" +source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#c4c5d5b8206d1becb1b83eee563b38cce0ea53a7" dependencies = [ "alloy-consensus", "alloy-consensus-any", @@ -686,8 +691,8 @@ dependencies = [ [[package]] name = "alloy-serde" -version = "1.0.41" -source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#8ce8a3d52a5b50e9f0cf16fac91888445be6009b" +version = "1.1.3" +source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#c4c5d5b8206d1becb1b83eee563b38cce0ea53a7" dependencies = [ "alloy-primitives", "serde", @@ -696,8 +701,8 @@ dependencies = [ [[package]] name = "alloy-signer" -version = "1.0.41" -source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#8ce8a3d52a5b50e9f0cf16fac91888445be6009b" +version = "1.1.3" +source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#c4c5d5b8206d1becb1b83eee563b38cce0ea53a7" dependencies = [ "alloy-primitives", "async-trait", @@ -710,8 +715,8 @@ dependencies = [ [[package]] name = "alloy-signer-local" -version = "1.0.41" -source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#8ce8a3d52a5b50e9f0cf16fac91888445be6009b" +version = "1.1.3" +source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#c4c5d5b8206d1becb1b83eee563b38cce0ea53a7" dependencies = [ "alloy-consensus", "alloy-network", @@ -725,42 +730,42 @@ dependencies = [ [[package]] name = "alloy-sol-macro" -version = "1.4.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3ce480400051b5217f19d6e9a82d9010cdde20f1ae9c00d53591e4a1afbb312" +checksum = "09eb18ce0df92b4277291bbaa0ed70545d78b02948df756bbd3d6214bf39a218" dependencies = [ "alloy-sol-macro-expander", "alloy-sol-macro-input", "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] name = "alloy-sol-macro-expander" -version = "1.4.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d792e205ed3b72f795a8044c52877d2e6b6e9b1d13f431478121d8d4eaa9028" +checksum = "95d9fa2daf21f59aa546d549943f10b5cce1ae59986774019fbedae834ffe01b" dependencies = [ "alloy-json-abi", "alloy-sol-macro-input", "const-hex", "heck", - "indexmap 2.12.1", + "indexmap 2.13.0", "proc-macro-error2", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", "syn-solidity", "tiny-keccak", ] [[package]] name = "alloy-sol-macro-input" -version = "1.4.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bd1247a8f90b465ef3f1207627547ec16940c35597875cdc09c49d58b19693c" +checksum = "9396007fe69c26ee118a19f4dee1f5d1d6be186ea75b3881adf16d87f8444686" dependencies = [ "alloy-json-abi", "const-hex", @@ -770,15 +775,15 @@ dependencies = [ "proc-macro2", "quote", "serde_json", - "syn 2.0.111", + "syn 2.0.114", "syn-solidity", ] [[package]] name = "alloy-sol-type-parser" -version = "1.4.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "954d1b2533b9b2c7959652df3076954ecb1122a28cc740aa84e7b0a49f6ac0a9" +checksum = "af67a0b0dcebe14244fc92002cd8d96ecbf65db4639d479f5fcd5805755a4c27" dependencies = [ "serde", "winnow", @@ -786,9 +791,9 @@ dependencies = [ [[package]] name = "alloy-sol-types" -version = "1.4.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70319350969a3af119da6fb3e9bddb1bce66c9ea933600cb297c8b1850ad2a3c" +checksum = "09aeea64f09a7483bdcd4193634c7e5cf9fd7775ee767585270cd8ce2d69dc95" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -798,11 +803,10 @@ dependencies = [ [[package]] name = "alloy-transport" -version = "1.0.41" -source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#8ce8a3d52a5b50e9f0cf16fac91888445be6009b" +version = "1.1.3" +source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#c4c5d5b8206d1becb1b83eee563b38cce0ea53a7" dependencies = [ "alloy-json-rpc", - "alloy-primitives", "auto_impl", "base64 0.22.1", "derive_more", @@ -821,8 +825,8 @@ dependencies = [ [[package]] name = "alloy-transport-http" -version = "1.0.41" -source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#8ce8a3d52a5b50e9f0cf16fac91888445be6009b" +version = "1.1.3" +source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#c4c5d5b8206d1becb1b83eee563b38cce0ea53a7" dependencies = [ "alloy-json-rpc", "alloy-transport", @@ -835,8 +839,8 @@ dependencies = [ [[package]] name = "alloy-transport-ipc" -version = "1.0.41" -source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#8ce8a3d52a5b50e9f0cf16fac91888445be6009b" +version = "1.1.3" +source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#c4c5d5b8206d1becb1b83eee563b38cce0ea53a7" dependencies = [ "alloy-json-rpc", "alloy-pubsub", @@ -854,8 +858,8 @@ dependencies = [ [[package]] name = "alloy-transport-ws" -version = "1.0.41" -source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#8ce8a3d52a5b50e9f0cf16fac91888445be6009b" +version = "1.1.3" +source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#c4c5d5b8206d1becb1b83eee563b38cce0ea53a7" dependencies = [ "alloy-pubsub", "alloy-transport", @@ -871,9 +875,9 @@ dependencies = [ [[package]] name = "alloy-trie" -version = "0.9.1" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3412d52bb97c6c6cc27ccc28d4e6e8cf605469101193b50b0bd5813b1f990b5" +checksum = "428aa0f0e0658ff091f8f667c406e034b431cb10abd39de4f507520968acc499" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -887,13 +891,13 @@ dependencies = [ [[package]] name = "alloy-tx-macros" -version = "1.0.41" -source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#8ce8a3d52a5b50e9f0cf16fac91888445be6009b" +version = "1.1.3" +source = "git+https://github.com/sistemd/alloy.git?branch=sistemd%2Fsaturate-unknown-tx-gas-price#c4c5d5b8206d1becb1b83eee563b38cce0ea53a7" dependencies = [ "darling 0.21.3", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -939,6 +943,26 @@ dependencies = [ "tokio", ] +[[package]] +name = "amp-data-store" +version = "0.1.0" +dependencies = [ + "amp-object-store", + "bytes", + "datafusion", + "datasets-common", + "foyer", + "futures", + "metadata-db", + "object_store", + "rand 0.9.2", + "serde", + "serde_json", + "thiserror 2.0.17", + "url", + "uuid", +] + [[package]] name = "amp-dataset-store" version = "0.1.0" @@ -1013,7 +1037,7 @@ dependencies = [ "anyhow", "clap", "common", - "console 0.16.1", + "console 0.16.2", "datasets-common", "dump", "eth-beacon-datasets", @@ -1040,6 +1064,7 @@ name = "ampd" version = "0.1.0" dependencies = [ "amp-config", + "amp-data-store", "amp-dataset-store", "amp-object-store", "clap", @@ -1066,7 +1091,7 @@ dependencies = [ "arrow-array", "arrow-schema", "arrow-to-postgres", - "axum 0.8.7", + "axum 0.8.8", "backon", "bytes", "clap", @@ -1180,9 +1205,12 @@ dependencies = [ [[package]] name = "arc-swap" -version = "1.7.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" +checksum = "51d03449bb8ca2cc2ef70869af31463d1ae5ccc8fa3e334b307203fbf815207e" +dependencies = [ + "rustversion", +] [[package]] name = "ark-ff" @@ -1269,7 +1297,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62945a2f7e6de02a31fe400aa489f0e0f5b2502e69f95f853adb82a96c7a6b60" dependencies = [ "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -1307,7 +1335,7 @@ dependencies = [ "num-traits", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -1555,7 +1583,7 @@ dependencies = [ "arrow-schema", "chrono", "half 2.7.1", - "indexmap 2.12.1", + "indexmap 2.13.0", "lexical-core", "memchr", "num 0.4.3", @@ -1720,9 +1748,9 @@ dependencies = [ [[package]] name = "async-lock" -version = "3.4.1" +version = "3.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd03604047cee9b6ce9de9f70c6cd540a0520c813cbd49bae61f33ab80ed1dc" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" dependencies = [ "event-listener", "event-listener-strategy", @@ -1748,7 +1776,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -1765,7 +1793,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -1808,13 +1836,13 @@ name = "auth-http" version = "0.1.0" dependencies = [ "anyhow", - "axum 0.8.7", + "axum 0.8.8", "axum-extra", "chrono", "headers", "http 1.4.0", "jsonwebtoken", - "lru 0.16.2", + "lru 0.16.3", "privy-rs", "reqwest", "serde", @@ -1833,7 +1861,7 @@ checksum = "ffdcb70bdbc4d478427380519163274ac86e52916e10f0a8889adf0f96d3fee7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -1871,11 +1899,11 @@ dependencies = [ [[package]] name = "axum" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b098575ebe77cb6d14fc7f32749631a6e44edbef6b796f89b020e99ba20d425" +checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8" dependencies = [ - "axum-core 0.5.5", + "axum-core 0.5.6", "axum-macros", "bytes", "form_urlencoded", @@ -1925,9 +1953,9 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.5.5" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59446ce19cd142f8833f856eb31f3eb097812d1479ab224f54d72428ca21ea22" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" dependencies = [ "bytes", "futures-core", @@ -1944,12 +1972,12 @@ dependencies = [ [[package]] name = "axum-extra" -version = "0.12.2" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbfe9f610fe4e99cf0cfcd03ccf8c63c28c616fe714d80475ef731f3b13dd21b" +checksum = "fef252edff26ddba56bbcdf2ee3307b8129acb86f5749b68990c168a6fcc9c76" dependencies = [ - "axum 0.8.7", - "axum-core 0.5.5", + "axum 0.8.8", + "axum-core 0.5.6", "bytes", "futures-core", "futures-util", @@ -1972,7 +2000,7 @@ checksum = "604fde5e028fea851ce1d8570bbdc034bec850d157f7569d10f347d06808c05c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -2027,15 +2055,15 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "base64ct" -version = "1.8.1" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e050f626429857a27ddccb31e0aca21356bfa709c04041aefddac081a8f068a" +checksum = "7d809780667f4410e7c41b07f52439b94d2bdf8528eeedc287fa38d3b7f95d82" [[package]] name = "bigdecimal" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "560f42649de9fa436b73517378a147ec21f6c997a546581df4b4b31677828934" +checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695" dependencies = [ "autocfg", "libm", @@ -2096,7 +2124,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -2215,7 +2243,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -2277,9 +2305,9 @@ checksum = "387e80962b798815a2b5c4bcfdb6bf626fa922ffe9f74e373103b858738e9f31" [[package]] name = "bumpalo" -version = "3.19.0" +version = "3.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" +checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" [[package]] name = "bv" @@ -2314,7 +2342,7 @@ checksum = "f9abbd1bc6865053c427f7198e6af43bfdedc55ab791faed4fbd361d789575ff" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -2434,9 +2462,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.49" +version = "1.2.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90583009037521a116abf44494efecd645ba48b6622457080f080b85544e2215" +checksum = "7a0aeaff4ff1a90589618835a598e545176939b97874f7abc7851caa0618f203" dependencies = [ "find-msvc-tools", "jobserver", @@ -2479,7 +2507,7 @@ checksum = "45565fc9416b9896014f5732ac776f810ee53a66730c17e4020c3ec064a8f88f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -2566,9 +2594,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.53" +version = "4.5.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9e340e012a1bf4935f5282ed1436d1489548e8f72308207ea5df0e23d2d03f8" +checksum = "c6e6ff9dcd79cff5cd969a17a545d79e84ab086e444102a591e288a8aa3ce394" dependencies = [ "clap_builder", "clap_derive", @@ -2576,9 +2604,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.53" +version = "4.5.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d76b5d13eaa18c901fd2f7fca939fefe3a0727a953561fefdf3b2922b8569d00" +checksum = "fa42cf4d2b7a41bc8f663a7cab4031ebafa1bf3875705bfaf8466dc60ab52c00" dependencies = [ "anstream", "anstyle", @@ -2595,7 +2623,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -2606,9 +2634,9 @@ checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d" [[package]] name = "cmake" -version = "0.1.56" +version = "0.1.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b042e5d8a74ae91bb0961acd039822472ec99f8ab0948cbf6d1369588f8be586" +checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d" dependencies = [ "cc", ] @@ -2676,10 +2704,10 @@ name = "common" version = "0.1.0" dependencies = [ "alloy", - "amp-object-store", + "amp-data-store", "async-stream", "async-trait", - "axum 0.8.7", + "axum 0.8.8", "backon", "bytes", "chrono", @@ -2687,9 +2715,6 @@ dependencies = [ "datafusion-datasource", "datafusion-tracing", "datasets-common", - "figment", - "foyer", - "fs-err", "futures", "governor 0.10.4", "indoc", @@ -2700,7 +2725,7 @@ dependencies = [ "object_store", "rand 0.9.2", "regex", - "schemars 1.1.0", + "schemars 1.2.0", "serde", "serde_json", "tempfile", @@ -2750,9 +2775,9 @@ dependencies = [ [[package]] name = "console" -version = "0.16.1" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b430743a6eb14e9764d4260d4c0d8123087d504eeb9c48f2b2a5e810dd369df4" +checksum = "03e45a4a8926227e4197636ba97a9fc9b00477e9f4bd711395687c5f0734bec4" dependencies = [ "encode_unicode", "libc", @@ -2876,9 +2901,10 @@ version = "0.1.0" dependencies = [ "admin-api", "amp-config", + "amp-data-store", "amp-dataset-store", "async-trait", - "axum 0.8.7", + "axum 0.8.8", "common", "datasets-common", "dump", @@ -3066,7 +3092,7 @@ dependencies = [ "document-features", "mio", "parking_lot", - "rustix 1.1.2", + "rustix 1.1.3", "signal-hook", "signal-hook-mio", "winapi", @@ -3182,7 +3208,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -3221,6 +3247,16 @@ dependencies = [ "darling_macro 0.21.3", ] +[[package]] +name = "darling" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +dependencies = [ + "darling_core 0.23.0", + "darling_macro 0.23.0", +] + [[package]] name = "darling_core" version = "0.14.4" @@ -3246,7 +3282,7 @@ dependencies = [ "proc-macro2", "quote", "strsim 0.11.1", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -3261,7 +3297,20 @@ dependencies = [ "quote", "serde", "strsim 0.11.1", - "syn 2.0.111", + "syn 2.0.114", +] + +[[package]] +name = "darling_core" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +dependencies = [ + "ident_case", + "proc-macro2", + "quote", + "strsim 0.11.1", + "syn 2.0.114", ] [[package]] @@ -3283,7 +3332,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core 0.20.11", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -3294,7 +3343,18 @@ checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ "darling_core 0.21.3", "quote", - "syn 2.0.111", + "syn 2.0.114", +] + +[[package]] +name = "darling_macro" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +dependencies = [ + "darling_core 0.23.0", + "quote", + "syn 2.0.114", ] [[package]] @@ -3347,7 +3407,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d162beedaa69905488a8da94f5ac3edb4dd4788b732fadb7bd120b2625c1976" dependencies = [ "data-encoding", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -3468,7 +3528,7 @@ dependencies = [ "chrono", "half 2.7.1", "hashbrown 0.14.5", - "indexmap 2.12.1", + "indexmap 2.13.0", "libc", "log", "object_store", @@ -3652,7 +3712,7 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-functions-window-common", "datafusion-physical-expr-common", - "indexmap 2.12.1", + "indexmap 2.13.0", "paste", "recursive", "serde_json", @@ -3667,7 +3727,7 @@ checksum = "6d155ccbda29591ca71a1344dd6bed26c65a4438072b400df9db59447f590bb6" dependencies = [ "arrow", "datafusion-common", - "indexmap 2.12.1", + "indexmap 2.13.0", "itertools 0.14.0", "paste", ] @@ -3809,7 +3869,7 @@ checksum = "ec6f637bce95efac05cdfb9b6c19579ed4aa5f6b94d951cfa5bb054b7bb4f730" dependencies = [ "datafusion-expr", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -3824,7 +3884,7 @@ dependencies = [ "datafusion-expr", "datafusion-expr-common", "datafusion-physical-expr", - "indexmap 2.12.1", + "indexmap 2.13.0", "itertools 0.14.0", "log", "recursive", @@ -3847,7 +3907,7 @@ dependencies = [ "datafusion-physical-expr-common", "half 2.7.1", "hashbrown 0.14.5", - "indexmap 2.12.1", + "indexmap 2.13.0", "itertools 0.14.0", "log", "parking_lot", @@ -3927,7 +3987,7 @@ dependencies = [ "futures", "half 2.7.1", "hashbrown 0.14.5", - "indexmap 2.12.1", + "indexmap 2.13.0", "itertools 0.14.0", "log", "parking_lot", @@ -3987,7 +4047,7 @@ dependencies = [ "bigdecimal", "datafusion-common", "datafusion-expr", - "indexmap 2.12.1", + "indexmap 2.13.0", "log", "recursive", "regex", @@ -4017,7 +4077,7 @@ dependencies = [ "datafusion", "hex", "metadata-db", - "schemars 1.1.0", + "schemars 1.2.0", "semver 1.0.27", "serde", "serde_json", @@ -4029,12 +4089,13 @@ dependencies = [ name = "datasets-derived" version = "0.1.0" dependencies = [ + "amp-data-store", "common", "datafusion", "datasets-common", "js-runtime", "metadata-db", - "schemars 1.1.0", + "schemars 1.2.0", "serde", "serde_json", "thiserror 2.0.17", @@ -4046,7 +4107,7 @@ name = "datasets-derived-gen" version = "0.1.0" dependencies = [ "datasets-derived", - "schemars 1.1.0", + "schemars 1.2.0", "serde_json", ] @@ -4056,7 +4117,7 @@ version = "0.1.0" dependencies = [ "datasets-raw", "eth-beacon-datasets", - "schemars 1.1.0", + "schemars 1.2.0", "serde_json", "tokio", ] @@ -4067,7 +4128,7 @@ version = "0.1.0" dependencies = [ "datasets-raw", "evm-rpc-datasets", - "schemars 1.1.0", + "schemars 1.2.0", "serde_json", "tokio", ] @@ -4078,7 +4139,7 @@ version = "0.1.0" dependencies = [ "datasets-raw", "firehose-datasets", - "schemars 1.1.0", + "schemars 1.2.0", "serde_json", "tokio", ] @@ -4096,7 +4157,7 @@ name = "datasets-solana-gen" version = "0.1.0" dependencies = [ "datasets-raw", - "schemars 1.1.0", + "schemars 1.2.0", "serde_json", "solana-datasets", "tokio", @@ -4128,7 +4189,7 @@ checksum = "780eb241654bf097afb00fc5f054a09b687dad862e485fdcf8399bb056565370" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -4201,7 +4262,7 @@ dependencies = [ "darling 0.20.11", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -4211,29 +4272,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] name = "derive_more" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10b768e943bed7bf2cab53df09f4bc34bfd217cdb57d971e769874c9a6710618" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" dependencies = [ "derive_more-impl", ] [[package]] name = "derive_more-impl" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d286bfdaf75e988b4a78e013ecd79c581e06399ab53fbacd2d916c2f904f30b" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" dependencies = [ "convert_case", "proc-macro2", "quote", "rustc_version 0.4.1", - "syn 2.0.111", + "syn 2.0.114", "unicode-xid", ] @@ -4306,7 +4367,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -4329,7 +4390,7 @@ checksum = "a6cbae11b3de8fce2a456e8ea3dada226b35fe791f0dc1d360c0941f0bb681f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -4388,6 +4449,7 @@ name = "dump" version = "0.1.0" dependencies = [ "alloy", + "amp-data-store", "amp-dataset-store", "async-stream", "common", @@ -4483,7 +4545,7 @@ dependencies = [ "enum-ordinalize", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -4549,7 +4611,7 @@ checksum = "685adfa4d6f3d765a26bc5dbc936577de9abf756c1feeb3089b01dd395034842" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -4569,7 +4631,7 @@ checksum = "8ca9601fb2d62598ee17836250842873a413586e5d7ed88b356e38ddbb0ec631" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -4581,7 +4643,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -4622,7 +4684,7 @@ dependencies = [ "futures", "governor 0.10.4", "reqwest", - "schemars 1.1.0", + "schemars 1.2.0", "serde", "serde_json", "serde_with", @@ -4661,7 +4723,7 @@ dependencies = [ "datasets-common", "futures", "monitoring", - "schemars 1.1.0", + "schemars 1.2.0", "serde", "serde_json", "serde_with", @@ -4771,9 +4833,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844" +checksum = "645cbb3a84e60b7531617d5ae4e57f7e27308f6445f5abf653209ea76dec8dff" [[package]] name = "firehose-datasets" @@ -4788,7 +4850,7 @@ dependencies = [ "monitoring", "prost 0.13.5", "prost-build", - "schemars 1.1.0", + "schemars 1.2.0", "serde", "serde_json", "thiserror 2.0.17", @@ -4857,9 +4919,9 @@ checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" [[package]] name = "flatbuffers" -version = "25.9.23" +version = "25.12.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09b6620799e7340ebd9968d2e0708eb82cf1971e9a16821e2091b6d6e475eed5" +checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" dependencies = [ "bitflags", "rustc_version 0.4.1", @@ -5038,9 +5100,9 @@ dependencies = [ [[package]] name = "fs-err" -version = "3.2.1" +version = "3.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "824f08d01d0f496b3eca4f001a13cf17690a6ee930043d20817f547455fd98f8" +checksum = "baf68cef89750956493a66a10f512b9e58d9db21f2a573c079c0bdf1207a54a7" dependencies = [ "autocfg", ] @@ -5051,7 +5113,7 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8640e34b88f7652208ce9e88b1a37a2ae95227d84abec377ccd3c5cfeb141ed4" dependencies = [ - "rustix 1.1.2", + "rustix 1.1.3", "windows-sys 0.59.0", ] @@ -5138,7 +5200,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -5312,9 +5374,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c0b69cfcb4e1b9f1bf2f53f95f766e4661169728ec61cd3fe5a0166f2d1386" +checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" dependencies = [ "atomic-waker", "bytes", @@ -5322,7 +5384,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.4.0", - "indexmap 2.12.1", + "indexmap 2.13.0", "slab", "tokio", "tokio-util", @@ -5661,7 +5723,7 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", - "webpki-roots 1.0.4", + "webpki-roots 1.0.5", ] [[package]] @@ -5868,7 +5930,7 @@ checksum = "a0eb5a3343abf848c0984fe4604b2b105da9539376e24fc0a3b0007411ae4fd9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -5884,9 +5946,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.12.1" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" dependencies = [ "equivalent", "hashbrown 0.16.1", @@ -5913,7 +5975,7 @@ version = "0.18.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9375e112e4b463ec1b1c6c011953545c65a30164fbab5b581df32b3abf0dcb88" dependencies = [ - "console 0.16.1", + "console 0.16.2", "portable-atomic", "unicode-width 0.2.0", "unit-prefix", @@ -5946,15 +6008,15 @@ dependencies = [ [[package]] name = "instability" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6778b0196eefee7df739db78758e5cf9b37412268bfa5650bfeed028aed20d9c" +checksum = "357b7205c6cd18dd2c86ed312d1e70add149aea98e7ef72b9fdf0270e555c11d" dependencies = [ - "darling 0.20.11", + "darling 0.23.0", "indoc", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -5997,9 +6059,9 @@ checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" [[package]] name = "iri-string" -version = "0.7.9" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f867b9d1d896b67beb18518eda36fdb77a32ea590de864f1325b294a6d14397" +checksum = "c91338f0783edbd6195decb37bae672fd3b165faffb89bf7b9e6942f8b1a731a" dependencies = [ "memchr", "serde", @@ -6049,9 +6111,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.15" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" [[package]] name = "jni" @@ -6254,9 +6316,9 @@ checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" [[package]] name = "libc" -version = "0.2.178" +version = "0.2.180" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" +checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" [[package]] name = "libloading" @@ -6276,13 +6338,13 @@ checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" [[package]] name = "libredox" -version = "0.1.10" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "416f7e718bdb06000964960ffa43b4335ad4012ae8b99060261aa4a8088d5ccb" +checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616" dependencies = [ "bitflags", "libc", - "redox_syscall", + "redox_syscall 0.7.0", ] [[package]] @@ -6297,9 +6359,9 @@ dependencies = [ [[package]] name = "libz-rs-sys" -version = "0.5.4" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15413ef615ad868d4d65dce091cb233b229419c7c0c4bcaa746c0901c49ff39c" +checksum = "c10501e7805cee23da17c7790e59df2870c0d4043ec6d03f67d31e2b53e77415" dependencies = [ "zlib-rs", ] @@ -6380,9 +6442,9 @@ dependencies = [ [[package]] name = "lru" -version = "0.16.2" +version = "0.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96051b46fc183dc9cd4a223960ef37b9af631b55191852a8274bfef064cda20f" +checksum = "a1dc47f592c06f33f8e3aea9591776ec7c9f9e4124778ff8a3c3b87159f7e593" dependencies = [ "hashbrown 0.16.1", ] @@ -6440,7 +6502,7 @@ checksum = "1b27834086c65ec3f9387b096d66e99f221cf081c2b738042aa252bcd41204e3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -6470,7 +6532,7 @@ dependencies = [ "spin", "tokio", "tokio-util", - "toml 0.9.8", + "toml 0.9.10+spec-1.1.0", "tracing", "tracing-subscriber", ] @@ -6715,7 +6777,7 @@ dependencies = [ "libc", "log", "openssl", - "openssl-probe", + "openssl-probe 0.1.6", "openssl-sys", "schannel", "security-framework 2.11.1", @@ -6865,7 +6927,7 @@ checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -6950,7 +7012,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -6970,9 +7032,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "nybbles" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c4b5ecbd0beec843101bffe848217f770e8b8da81d8355b7d6e226f2199b3dc" +checksum = "7b5676b5c379cf5b03da1df2b3061c4a4e2aa691086a56ac923e08c143f53f59" dependencies = [ "alloy-rlp", "cfg-if", @@ -7084,7 +7146,7 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c8d427828b22ae1fff2833a03d8486c2c881367f1c336349f307f321e7f4d05" dependencies = [ - "indexmap 2.12.1", + "indexmap 2.13.0", "serde", "serde_json", ] @@ -7112,7 +7174,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -7121,6 +7183,12 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" +[[package]] +name = "openssl-probe" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f50d9b3dabb09ecd771ad0aa242ca6894994c130308ca3d7684634df8037391" + [[package]] name = "openssl-sys" version = "0.9.111" @@ -7166,7 +7234,7 @@ version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5fa342b7ded339a38c6df9a3aef833298bca24a7aeec098397b4b3ec741b0e52" dependencies = [ - "axum 0.8.7", + "axum 0.8.8", "futures-util", "http 1.4.0", "http-body", @@ -7312,7 +7380,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -7339,7 +7407,7 @@ checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.18", "smallvec", "windows-link", ] @@ -7416,7 +7484,7 @@ dependencies = [ "proc-macro2", "proc-macro2-diagnostics", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -7454,9 +7522,9 @@ dependencies = [ [[package]] name = "pest" -version = "2.8.4" +version = "2.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbcfd20a6d4eeba40179f05735784ad32bdaef05ce8e8af05f180d45bb3e7e22" +checksum = "2c9eb05c21a464ea704b53158d358a31e6425db2f63a1a7312268b05fe2b75f7" dependencies = [ "memchr", "ucd-trie", @@ -7469,7 +7537,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" dependencies = [ "fixedbitset", - "indexmap 2.12.1", + "indexmap 2.13.0", ] [[package]] @@ -7480,7 +7548,7 @@ checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ "fixedbitset", "hashbrown 0.15.5", - "indexmap 2.12.1", + "indexmap 2.13.0", "serde", ] @@ -7554,7 +7622,7 @@ dependencies = [ "phf_shared 0.11.3", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -7592,7 +7660,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -7641,7 +7709,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "740ebea15c5d1428f910cd1a5f52cebf8d25006245ed8ade92702f4943d91e07" dependencies = [ "base64 0.22.1", - "indexmap 2.12.1", + "indexmap 2.13.0", "quick-xml", "serde", "time", @@ -7672,9 +7740,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.11.1" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" +checksum = "f89776e4d69bb58bc6993e99ffa1d11f228b839984854c7daeb5d37f87cbe950" [[package]] name = "potential_utf" @@ -7717,7 +7785,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -7770,7 +7838,7 @@ dependencies = [ "heck", "hex", "hpke", - "lru 0.16.2", + "lru 0.16.3", "openapiv3", "p256", "prettyplease", @@ -7787,7 +7855,7 @@ dependencies = [ "serde_yaml", "sha2", "spki", - "syn 2.0.111", + "syn 2.0.114", "thiserror 2.0.17", "tokio", "tracing", @@ -7800,7 +7868,7 @@ version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "219cb19e96be00ab2e37d6e299658a0cfa83e52429179969b0f0121b4ac46983" dependencies = [ - "toml_edit 0.23.9", + "toml_edit 0.23.10+spec-1.0.0", ] [[package]] @@ -7822,14 +7890,14 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] name = "proc-macro2" -version = "1.0.103" +version = "1.0.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" +checksum = "535d180e0ecab6268a3e718bb9fd44db66bbbc256257165fc699dadf70d16fe7" dependencies = [ "unicode-ident", ] @@ -7842,7 +7910,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", "version_check", "yansi", ] @@ -7881,7 +7949,7 @@ checksum = "90f6d9109b04e005bbdec84cacec7e81cc15533f2b5dc505f0defc212d270c15" dependencies = [ "heck", "http 1.4.0", - "indexmap 2.12.1", + "indexmap 2.13.0", "openapiv3", "proc-macro2", "quote", @@ -7889,7 +7957,7 @@ dependencies = [ "schemars 0.8.22", "serde", "serde_json", - "syn 2.0.111", + "syn 2.0.114", "thiserror 2.0.17", "typify", "unicode-ident", @@ -7910,7 +7978,7 @@ dependencies = [ "serde_json", "serde_tokenstream", "serde_yaml", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -7968,7 +8036,7 @@ dependencies = [ "prost 0.13.5", "prost-types", "regex", - "syn 2.0.111", + "syn 2.0.114", "tempfile", ] @@ -7982,7 +8050,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -7995,7 +8063,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -8116,9 +8184,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.42" +version = "1.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" +checksum = "dc74d9a594b72ae6656596548f56f667211f8a97b3d4c3d467150794690dc40a" dependencies = [ "proc-macro2", ] @@ -8215,6 +8283,15 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "rapidhash" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2988730ee014541157f48ce4dcc603940e00915edc3c7f9a8d78092256bb2493" +dependencies = [ + "rustversion", +] + [[package]] name = "ratatui" version = "0.29.0" @@ -8283,7 +8360,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" dependencies = [ "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -8301,6 +8378,15 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_syscall" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f3fe0889e69e2ae9e41f4d6c4c0181701d00e4697b356fb1f74173a5e0ee27" +dependencies = [ + "bitflags", +] + [[package]] name = "redox_users" version = "0.5.2" @@ -8329,7 +8415,7 @@ checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -8373,9 +8459,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.26" +version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b4c14b2d9afca6a60277086b0cc6a6ae0b568f6f7916c943a8cdc79f8be240f" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ "base64 0.22.1", "bytes", @@ -8417,7 +8503,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots 1.0.4", + "webpki-roots 1.0.5", ] [[package]] @@ -8471,9 +8557,9 @@ dependencies = [ [[package]] name = "rsa" -version = "0.9.9" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40a0376c50d0358279d9d643e4bf7b7be212f1f4ff1da9070a7b54d22ef75c88" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" dependencies = [ "const-oid", "digest 0.10.7", @@ -8491,9 +8577,9 @@ dependencies = [ [[package]] name = "ruint" -version = "1.17.0" +version = "1.17.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a68df0380e5c9d20ce49534f292a36a7514ae21350726efe1865bdb1fa91d278" +checksum = "c141e807189ad38a07276942c6623032d3753c8859c146104ac2e4d68865945a" dependencies = [ "alloy-rlp", "ark-ff 0.3.0", @@ -8583,9 +8669,9 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" +checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" dependencies = [ "bitflags", "errno", @@ -8596,9 +8682,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.35" +version = "0.23.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f" +checksum = "c665f33d38cea657d9614f766881e4d510e0eda4239891eea56b4cadcf01801b" dependencies = [ "log", "once_cell", @@ -8611,11 +8697,11 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9980d917ebb0c0536119ba501e90834767bffc3d60641457fd84a1f3fd337923" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" dependencies = [ - "openssl-probe", + "openssl-probe 0.2.0", "rustls-pki-types", "schannel", "security-framework 3.5.1", @@ -8632,9 +8718,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.13.1" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "708c0f9d5f54ba0272468c1d306a52c495b31fa155e91bc25371e6df7996908c" +checksum = "21e6f2ab2928ca4291b86736a8bd920a277a399bba1589409d72154ff87c1282" dependencies = [ "web-time", "zeroize", @@ -8698,9 +8784,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.20" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +checksum = "a50f4cf475b65d88e057964e0e9bb1f0aa9bbb2036dc65c64596b42932536984" [[package]] name = "ryu-js" @@ -8754,13 +8840,13 @@ dependencies = [ [[package]] name = "schemars" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9558e172d4e8533736ba97870c4b2cd63f84b382a3d6eb063da41b91cce17289" +checksum = "54e910108742c57a770f492731f99be216a52fadd361b06c8fb59d74ccc267d2" dependencies = [ "dyn-clone", "ref-cast", - "schemars_derive 1.1.0", + "schemars_derive 1.2.0", "serde", "serde_json", ] @@ -8774,19 +8860,19 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] name = "schemars_derive" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301858a4023d78debd2353c7426dc486001bddc91ae31a76fb1f55132f7e2633" +checksum = "4908ad288c5035a8eb12cfdf0d49270def0a268ee162b75eeee0f85d155a7c45" dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -8963,7 +9049,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -8974,20 +9060,20 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] name = "serde_json" -version = "1.0.145" +version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" dependencies = [ "itoa", "memchr", - "ryu", "serde", "serde_core", + "zmij", ] [[package]] @@ -9023,9 +9109,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e24345aa0fe688594e73770a5f6d1b216508b4f93484c0026d521acd30134392" +checksum = "f8bbf91e5a4d6315eee45e704372590b30e260ee83af6639d64557f51b067776" dependencies = [ "serde_core", ] @@ -9039,7 +9125,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -9064,9 +9150,9 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.12.1", + "indexmap 2.13.0", "schemars 0.9.0", - "schemars 1.1.0", + "schemars 1.2.0", "serde_core", "serde_json", "serde_with_macros", @@ -9082,7 +9168,7 @@ dependencies = [ "darling 0.21.3", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -9091,7 +9177,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.12.1", + "indexmap 2.13.0", "itoa", "ryu", "serde", @@ -9112,11 +9198,12 @@ dependencies = [ name = "server" version = "0.1.0" dependencies = [ + "amp-data-store", "amp-dataset-store", "arrow-flight", "async-stream", "async-trait", - "axum 0.8.7", + "axum 0.8.8", "bincode 2.0.1", "bytes", "common", @@ -9221,10 +9308,11 @@ dependencies = [ [[package]] name = "signal-hook-registry" -version = "1.4.7" +version = "1.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7664a098b8e616bdfcc2dc0e9ac44eb231eedf41db4e9fe95d8d32ec728dedad" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" dependencies = [ + "errno", "libc", ] @@ -9329,9 +9417,9 @@ dependencies = [ [[package]] name = "solana-account" -version = "3.2.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "014dcb9293341241dd153b35f89ea906e4170914f4a347a95e7fb07ade47cd6f" +checksum = "60e0ac2a81ae17e1b3570deb50242ab4cfde50b848b898f57288b6271cc7b71f" dependencies = [ "bincode 1.3.3", "serde", @@ -9340,16 +9428,16 @@ dependencies = [ "solana-account-info", "solana-clock", "solana-instruction-error", - "solana-pubkey 3.0.0", + "solana-pubkey 4.0.0", "solana-sdk-ids", "solana-sysvar", ] [[package]] name = "solana-account-decoder" -version = "3.1.4" +version = "3.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "457b212d07785a3d270604848482c74912480f3663e0745119e317ba9194e6d6" +checksum = "a8c957a2afa2378c666660367eceaef341d766a8e8cf35b22178559f30e0678a" dependencies = [ "Inflector", "base64 0.22.1", @@ -9389,9 +9477,9 @@ dependencies = [ [[package]] name = "solana-account-decoder-client-types" -version = "3.1.4" +version = "3.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82c212ff9c46ebc940170a435f55aa79f310684385cfb842876982c9f479b852" +checksum = "b07e3dc3cde61c12ee70c48962b16604b8a5bdf357d93763a7c8dff8f27b4215" dependencies = [ "base64 0.22.1", "bs58", @@ -9448,9 +9536,9 @@ dependencies = [ [[package]] name = "solana-address-lookup-table-interface" -version = "3.0.0" +version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2f56cac5e70517a2f27d05e5100b20de7182473ffd0035b23ea273307905987" +checksum = "5e8df0b083c10ce32490410f3795016b1b5d9b4d094658c0a5e496753645b7cd" dependencies = [ "bincode 1.3.3", "bytemuck", @@ -9459,7 +9547,7 @@ dependencies = [ "solana-clock", "solana-instruction", "solana-instruction-error", - "solana-pubkey 3.0.0", + "solana-pubkey 4.0.0", "solana-sdk-ids", "solana-slot-hashes", ] @@ -9506,16 +9594,16 @@ dependencies = [ [[package]] name = "solana-client" -version = "3.1.4" +version = "3.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90c44487fd856caf81ff89a5306b67f15d0ed742b2d65f4bfa5aa98b51ce3a97" +checksum = "fd6627613e1a0a078d4933f48f405faefe594389ebc3c36fcb821c4257954d19" dependencies = [ "async-trait", "bincode 1.3.3", "dashmap 5.5.3", "futures", "futures-util", - "indexmap 2.12.1", + "indexmap 2.13.0", "indicatif 0.18.3", "log", "quinn", @@ -9624,15 +9712,15 @@ dependencies = [ [[package]] name = "solana-connection-cache" -version = "3.1.4" +version = "3.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e42f2618d16d77cacbede20e1f3d2833bcff19e1633c8d56d48e844a67ffef" +checksum = "b7be124ac54563e492eb6bd052764814e347d3ba5393c7452684000829b9d84a" dependencies = [ "async-trait", "bincode 1.3.3", "crossbeam-channel", "futures-util", - "indexmap 2.12.1", + "indexmap 2.13.0", "log", "rand 0.8.5", "rayon", @@ -9661,9 +9749,9 @@ dependencies = [ [[package]] name = "solana-curve25519" -version = "3.1.4" +version = "3.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2cb3f53e40375fba75e908295c7723d16dad3fb54384140366fade2c7ef37bb" +checksum = "ebca352e7716ff1a0877272f87c772c958489c1d568a92d318dc0c75939d2884" dependencies = [ "bytemuck", "bytemuck_derive", @@ -9690,7 +9778,7 @@ dependencies = [ "monitoring", "prost 0.13.5", "reqwest", - "schemars 1.1.0", + "schemars 1.2.0", "serde", "serde_json", "serde_with", @@ -10004,9 +10092,9 @@ dependencies = [ [[package]] name = "solana-measure" -version = "3.1.4" +version = "3.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdac9c393ca216edb09b2113b3fe394d7bfabd885e0343c16e28a4836ee7ca77" +checksum = "eb1ffbeb8e195dc1953dae2ebae063c92619a2ddf9bc0bedc80bca4f3f0f14eb" [[package]] name = "solana-message" @@ -10030,9 +10118,9 @@ dependencies = [ [[package]] name = "solana-metrics" -version = "3.1.4" +version = "3.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31aada3eda5540c71fbce5898a66aa2c41289bc7bdd9a40063d582540d716c8" +checksum = "3cd3143e9fb2bc093412f148c5a810cfd6f637d7ba829548a43191a3efaacdb3" dependencies = [ "crossbeam-channel", "gethostname", @@ -10061,9 +10149,9 @@ checksum = "ae8dd4c280dca9d046139eb5b7a5ac9ad10403fbd64964c7d7571214950d758f" [[package]] name = "solana-net-utils" -version = "3.1.4" +version = "3.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68e17596032cd2579d4601bc7e57122d3e1a4e7bb0904427952dcbaca189ea00" +checksum = "39a1fb827632b42c29cbe882e0ebab6eb8ea93a48b4592d9676ff0ab942792c5" dependencies = [ "anyhow", "bincode 1.3.3", @@ -10128,9 +10216,9 @@ dependencies = [ [[package]] name = "solana-perf" -version = "3.1.4" +version = "3.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c5da279b55879009057bdcfa0185b8084775b71beef3bba2f3a142a932a4c72" +checksum = "34b10a284e7ec45920a726c7fa6da63833a4a1ed3e83bc0b2b3801c7fdf89f1d" dependencies = [ "ahash", "bincode 1.3.3", @@ -10285,9 +10373,9 @@ dependencies = [ [[package]] name = "solana-pubsub-client" -version = "3.1.4" +version = "3.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f626bb30b648c7a0b3e24ef67a691f7c6799574e76c9d5ebb63e9a7e468c137" +checksum = "da296bc8c240b3d53b1e323e5bfc28fbd7f012992ffa558185429dabd51a97a6" dependencies = [ "crossbeam-channel", "futures-util", @@ -10311,9 +10399,9 @@ dependencies = [ [[package]] name = "solana-quic-client" -version = "3.1.4" +version = "3.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3b0affe113d01ddaa99654c27a626101c12c9134d056bb731a2c5b7c19d0e6c" +checksum = "d0e87a7591e508373c09bdc64fb4f3663106c5d75f198d83348b34abff8a9ecd" dependencies = [ "async-lock", "async-trait", @@ -10350,9 +10438,9 @@ dependencies = [ [[package]] name = "solana-rayon-threadlimit" -version = "3.1.4" +version = "3.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b77c4703113f0bee4fa02b4a96cf0a1ababd379962229f7fb6feb069d7bfdd21" +checksum = "36cfbda7cd798b97c499d87801860d2808e891a117aefee4a56d5832f584be7a" dependencies = [ "log", "num_cpus", @@ -10383,9 +10471,9 @@ dependencies = [ [[package]] name = "solana-rpc-client" -version = "3.1.4" +version = "3.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbcf98658e977b48ebb6a01d6c874e9371a6af314c3f4527c37233352275c1d0" +checksum = "89a7d0ff3ffabd9978c69d615fbbb60977a33d8fa5408b00a57bed3df5ed8c8b" dependencies = [ "async-trait", "base64 0.22.1", @@ -10423,9 +10511,9 @@ dependencies = [ [[package]] name = "solana-rpc-client-api" -version = "3.1.4" +version = "3.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74b5b70f83404be3cfe72795a2d15b9d4e5e8c381f4a678bba83f496dbaa0045" +checksum = "8c034082d8dcf1dee358b7b32406b30aeb6d5ce6fb94c7a811cce7416d447df5" dependencies = [ "anyhow", "jsonrpc-core", @@ -10444,9 +10532,9 @@ dependencies = [ [[package]] name = "solana-rpc-client-nonce-utils" -version = "3.1.4" +version = "3.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93b076ee027fc19f34efaca6b901399ec33f86ee4fbe16a5afb642f6ca8c07c7" +checksum = "d7dcd056bc1b7e121779d6fc113fc6fc89cc3ee00a5fbab12f34f57edd89370d" dependencies = [ "solana-account", "solana-commitment-config", @@ -10461,9 +10549,9 @@ dependencies = [ [[package]] name = "solana-rpc-client-types" -version = "3.1.4" +version = "3.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbbd3ccfcb0cf05ac18e5ae64794b2dbf16cc54f438ca0371785ba78e5485e1e" +checksum = "10f14d2579cf5c5032be11048730bf06a1f3d01815e070eebd27b2d451ed0e77" dependencies = [ "base64 0.22.1", "bs58", @@ -10562,7 +10650,7 @@ dependencies = [ "bs58", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -10762,9 +10850,9 @@ dependencies = [ [[package]] name = "solana-streamer" -version = "3.1.4" +version = "3.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a537fa97d33b15076c4f06e8c07008874c87f1b22984af61b958c8feaa73f3f8" +checksum = "547ede8fb1110415368c6b507d8ffc5f4bc276f7b59447e6fda318d1dc32e138" dependencies = [ "arc-swap", "bytes", @@ -10774,7 +10862,7 @@ dependencies = [ "futures-util", "governor 0.6.3", "histogram", - "indexmap 2.12.1", + "indexmap 2.13.0", "itertools 0.12.1", "libc", "log", @@ -10810,15 +10898,15 @@ dependencies = [ [[package]] name = "solana-svm-feature-set" -version = "3.1.4" +version = "3.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ed3c3ca42d7765231c72600d10038db54329b7970c6fd13d6c1ffb30adda81b" +checksum = "ca638d19ace892ef2bf31af74897fa8828ebf9d6418b46771c67857e66cf5c97" [[package]] name = "solana-svm-type-overrides" -version = "3.1.4" +version = "3.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a3f2dab6bc6fd16f43942922ba53d7bc947666ef3a121d74ccb426f01ae1eaf" +checksum = "b30f2f7e41312046fb07a7f4de7bff814501b983f64a34805756ea76f08f6263" dependencies = [ "rand 0.8.5", ] @@ -10890,9 +10978,9 @@ checksum = "0ced92c60aa76ec4780a9d93f3bd64dfa916e1b998eacc6f1c110f3f444f02c9" [[package]] name = "solana-tls-utils" -version = "3.1.4" +version = "3.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9337db720d7e6b8ff47c6dba314269e139a7f2fc2be4a526f3e871a484c2fc4e" +checksum = "e6a07c8389607975001dbcc934d919e286ea3ae487344fea48a6485cbbb6ccd2" dependencies = [ "rustls", "solana-keypair", @@ -10903,14 +10991,14 @@ dependencies = [ [[package]] name = "solana-tpu-client" -version = "3.1.4" +version = "3.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a873d59073166b18b0957ce399700bc817b61b2454fd9e77f45fe91621724f78" +checksum = "7d0057e3abb53db136b91f68746823b18a47672fa397bdcdce122cbc85712bb6" dependencies = [ "async-trait", "bincode 1.3.3", "futures-util", - "indexmap 2.12.1", + "indexmap 2.13.0", "indicatif 0.18.3", "log", "rayon", @@ -10959,9 +11047,9 @@ dependencies = [ [[package]] name = "solana-transaction-context" -version = "3.1.4" +version = "3.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddceebc27e616cdb37c6aa30a5272145018db0307965a2c9c2ad630f48afd37f" +checksum = "120863f5edd2f70338482e9719c630c88441f9499d796dbfc68f6d0fa92413df" dependencies = [ "bincode 1.3.3", "serde", @@ -10988,9 +11076,9 @@ dependencies = [ [[package]] name = "solana-transaction-metrics-tracker" -version = "3.1.4" +version = "3.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "371bbb54716cc28e298afa60f8df075fb5f391474a98861b14e801b1cf681eaa" +checksum = "5e9aab274d8000016054c0ca20427c53f9c8fa97945c4f0447307b8fd187a1b4" dependencies = [ "base64 0.22.1", "bincode 1.3.3", @@ -11004,9 +11092,9 @@ dependencies = [ [[package]] name = "solana-transaction-status" -version = "3.1.4" +version = "3.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eb034ed7e5a57e25f82622b006d0d05a248d07ff55701ba51152f4178098dcd" +checksum = "3db9eb6c7ce4ae4c7735c3db682891963eae13e6cb22a421fbdf799674e25845" dependencies = [ "Inflector", "agave-reserved-account-keys", @@ -11047,9 +11135,9 @@ dependencies = [ [[package]] name = "solana-transaction-status-client-types" -version = "3.1.4" +version = "3.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b44471ed5cc0bfb2d7e14051b087766a04d90555c4f3b7ef8f7d5e36d7366d27" +checksum = "514e57860b0f435e3d26265779d716f10a3f6b1f19ccc28e37ec136dc6795de9" dependencies = [ "base64 0.22.1", "bincode 1.3.3", @@ -11071,9 +11159,9 @@ dependencies = [ [[package]] name = "solana-udp-client" -version = "3.1.4" +version = "3.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b04dc98ed4ab77a1b9f78d7b9364855531690ecfae000e4e2ffb18c985f7c357" +checksum = "a5eafbdabec704a740b8981e187b006300b434016833cbff5c8b281a2a94f8fd" dependencies = [ "async-trait", "solana-connection-cache", @@ -11087,9 +11175,9 @@ dependencies = [ [[package]] name = "solana-version" -version = "3.1.4" +version = "3.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce24ba7de24ea7caec6d811dfbc9eb234f3aee8aaa748f5dd7fc4b1fb3002aaa" +checksum = "5c226802774fddee968faac3d32b420de505d89f5ad7fe573cacd31a254f476d" dependencies = [ "agave-feature-set", "rand 0.8.5", @@ -11221,7 +11309,7 @@ checksum = "d9e8418ea6269dcfb01c712f0444d2c75542c04448b480e87de59d2865edc750" dependencies = [ "quote", "spl-discriminator-syn", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -11233,7 +11321,7 @@ dependencies = [ "proc-macro2", "quote", "sha2", - "syn 2.0.111", + "syn 2.0.114", "thiserror 1.0.69", ] @@ -11429,7 +11517,7 @@ checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -11464,7 +11552,7 @@ dependencies = [ "futures-util", "hashbrown 0.15.5", "hashlink", - "indexmap 2.12.1", + "indexmap 2.13.0", "log", "memchr", "once_cell", @@ -11490,7 +11578,7 @@ dependencies = [ "quote", "sqlx-core", "sqlx-macros-core", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -11513,7 +11601,7 @@ dependencies = [ "sqlx-mysql", "sqlx-postgres", "sqlx-sqlite", - "syn 2.0.111", + "syn 2.0.114", "tokio", "url", ] @@ -11700,7 +11788,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -11712,7 +11800,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -11734,9 +11822,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.111" +version = "2.0.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "390cc9a294ab71bdb1aa2e99d13be9c753cd2d7bd6560c77118597410c4d2e87" +checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a" dependencies = [ "proc-macro2", "quote", @@ -11745,14 +11833,14 @@ dependencies = [ [[package]] name = "syn-solidity" -version = "1.4.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff790eb176cc81bb8936aed0f7b9f14fc4670069a2d371b3e3b0ecce908b2cb3" +checksum = "5f92d01b5de07eaf324f7fca61cc6bd3d82bbc1de5b6c963e6fe79e86f36580d" dependencies = [ "paste", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -11793,7 +11881,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -11858,14 +11946,14 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tempfile" -version = "3.23.0" +version = "3.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" +checksum = "655da9c7eb6305c55742045d5a8d2037996d61d8de95806335c7c86ce0f82e9c" dependencies = [ "fastrand", "getrandom 0.3.4", "once_cell", - "rustix 1.1.2", + "rustix 1.1.3", "windows-sys 0.61.2", ] @@ -11885,6 +11973,7 @@ dependencies = [ "alloy", "amp-client", "amp-config", + "amp-data-store", "amp-dataset-store", "amp-object-store", "ampctl", @@ -11954,7 +12043,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -11965,7 +12054,7 @@ checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -12066,9 +12155,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.48.0" +version = "1.49.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408" +checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86" dependencies = [ "bytes", "libc", @@ -12090,7 +12179,7 @@ checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -12115,9 +12204,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" dependencies = [ "futures-core", "pin-project-lite", @@ -12159,9 +12248,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.17" +version = "0.7.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2efa149fe76073d6e8fd97ef4f4eca7b67f599660115591483572e406e165594" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" dependencies = [ "bytes", "futures-core", @@ -12177,7 +12266,7 @@ version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" dependencies = [ - "indexmap 2.12.1", + "indexmap 2.13.0", "serde", "serde_spanned 0.6.9", "toml_datetime 0.6.11", @@ -12186,14 +12275,14 @@ dependencies = [ [[package]] name = "toml" -version = "0.9.8" +version = "0.9.10+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0dc8b1fb61449e27716ec0e1bdf0f6b8f3e8f6b05391e8497b8b6d7804ea6d8" +checksum = "0825052159284a1a8b4d6c0c86cbc801f2da5afd2b225fa548c72f2e74002f48" dependencies = [ - "indexmap 2.12.1", + "indexmap 2.13.0", "serde_core", - "serde_spanned 1.0.3", - "toml_datetime 0.7.3", + "serde_spanned 1.0.4", + "toml_datetime 0.7.5+spec-1.1.0", "toml_parser", "toml_writer", "winnow", @@ -12210,9 +12299,9 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.7.3" +version = "0.7.5+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2cdb639ebbc97961c51720f858597f7f24c4fc295327923af55b74c3c724533" +checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" dependencies = [ "serde_core", ] @@ -12223,7 +12312,7 @@ version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ - "indexmap 2.12.1", + "indexmap 2.13.0", "serde", "serde_spanned 0.6.9", "toml_datetime 0.6.11", @@ -12233,21 +12322,21 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.23.9" +version = "0.23.10+spec-1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d7cbc3b4b49633d57a0509303158ca50de80ae32c265093b24c414705807832" +checksum = "84c8b9f757e028cee9fa244aea147aab2a9ec09d5325a9b01e0a49730c2b5269" dependencies = [ - "indexmap 2.12.1", - "toml_datetime 0.7.3", + "indexmap 2.13.0", + "toml_datetime 0.7.5+spec-1.1.0", "toml_parser", "winnow", ] [[package]] name = "toml_parser" -version = "1.0.4" +version = "1.0.6+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0cbe268d35bdb4bb5a56a2de88d0ad0eb70af5384a99d648cd4b3d04039800e" +checksum = "a3198b4b0a8e11f09dd03e133c0280504d0801269e9afa46362ffde1cbeebf44" dependencies = [ "winnow", ] @@ -12260,9 +12349,9 @@ checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" [[package]] name = "toml_writer" -version = "1.0.4" +version = "1.0.6+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df8b2b54733674ad286d16267dcfc7a71ed5c776e4ac7aa3c3e2561f7c637bf2" +checksum = "ab16f14aed21ee8bfd8ec22513f7287cd4a91aa92e44edfe2c17ddd004e92607" [[package]] name = "tonic" @@ -12301,7 +12390,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9" dependencies = [ "async-trait", - "axum 0.8.7", + "axum 0.8.8", "base64 0.22.1", "bytes", "flate2", @@ -12363,7 +12452,7 @@ dependencies = [ "prost-build", "prost-types", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -12405,7 +12494,7 @@ checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" dependencies = [ "futures-core", "futures-util", - "indexmap 2.12.1", + "indexmap 2.13.0", "pin-project-lite", "slab", "sync_wrapper", @@ -12453,9 +12542,9 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" -version = "0.1.43" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d15d90a0b5c19378952d479dc858407149d7bb45a14de0142f6c534b16fc647" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ "log", "pin-project-lite", @@ -12471,14 +12560,14 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] name = "tracing-core" -version = "0.1.35" +version = "0.1.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a04e24fab5c89c6a36eb8558c9656f30d81de51dfa4d3b45f26b21d61fa0a6c" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" dependencies = [ "once_cell", "valuable", @@ -12629,7 +12718,7 @@ dependencies = [ "semver 1.0.27", "serde", "serde_json", - "syn 2.0.111", + "syn 2.0.114", "thiserror 2.0.17", "unicode-ident", ] @@ -12647,7 +12736,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.111", + "syn 2.0.114", "typify-impl", ] @@ -12807,14 +12896,15 @@ dependencies = [ [[package]] name = "url" -version = "2.5.7" +version = "2.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" dependencies = [ "form_urlencoded", "idna", "percent-encoding", "serde", + "serde_derive", ] [[package]] @@ -12847,7 +12937,7 @@ version = "5.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fcc29c80c21c31608227e0912b2d7fddba57ad76b606890627ba8ee7964e993" dependencies = [ - "indexmap 2.12.1", + "indexmap 2.13.0", "serde", "serde_json", "utoipa-gen", @@ -12862,7 +12952,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -13055,7 +13145,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", "wasm-bindgen-shared", ] @@ -13117,9 +13207,9 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee3e3b5f5e80bc89f30ce8d0343bf4e5f12341c51f3e26cbeecbc7c85443e85b" +checksum = "36a29fc0408b113f68cf32637857ab740edfafdf460c326cd2afaa2d84cc05dc" dependencies = [ "rustls-pki-types", ] @@ -13130,14 +13220,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.4", + "webpki-roots 1.0.5", ] [[package]] name = "webpki-roots" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2878ef029c47c6e8cf779119f20fcf52bde7ad42a731b2a304bc221df17571e" +checksum = "12bed680863276c63889429bfd6cab3b99943659923822de1c8a39c49e4d722c" dependencies = [ "rustls-pki-types", ] @@ -13222,7 +13312,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -13233,7 +13323,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -13594,6 +13684,7 @@ name = "worker" version = "0.1.0" dependencies = [ "amp-config", + "amp-data-store", "amp-dataset-store", "backon", "chrono", @@ -13708,7 +13799,7 @@ dependencies = [ "const-hex", "crc", "fnv", - "indexmap 2.12.1", + "indexmap 2.13.0", "multihash", "serde_cbor", "thiserror 2.0.17", @@ -13734,28 +13825,28 @@ checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", "synstructure 0.13.2", ] [[package]] name = "zerocopy" -version = "0.8.31" +version = "0.8.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd74ec98b9250adb3ca554bdde269adf631549f51d8a8f8f0a10b50f1cb298c3" +checksum = "668f5168d10b9ee831de31933dc111a459c97ec93225beb307aed970d1372dfd" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.31" +version = "0.8.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8a8d209fdf45cf5138cbb5a506f6b52522a25afccc534d1475dad8e31105c6a" +checksum = "2c7962b26b0a8685668b671ee4b54d007a67d4eaf05fda79ac0ecf41e32270f1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -13775,7 +13866,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", "synstructure 0.13.2", ] @@ -13790,13 +13881,13 @@ dependencies = [ [[package]] name = "zeroize_derive" -version = "1.4.2" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" +checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] @@ -13829,14 +13920,20 @@ checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.111", + "syn 2.0.114", ] [[package]] name = "zlib-rs" -version = "0.5.4" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40990edd51aae2c2b6907af74ffb635029d5788228222c4bb811e9351c0caad3" + +[[package]] +name = "zmij" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51f936044d677be1a1168fae1d03b583a285a5dd9d8cbf7b24c23aa1fc775235" +checksum = "2fc5a66a20078bf1251bde995aa2fdcc4b800c70b5d92dd2c62abc5c60f679f8" [[package]] name = "zstd" diff --git a/Cargo.toml b/Cargo.toml index d50acf5cd..35a0b0758 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,6 +10,7 @@ members = [ "crates/bin/ampup", "crates/clients/flight", "crates/core/common", + "crates/core/data-store", "crates/core/dataset-store", "crates/core/datasets-common", "crates/core/datasets-derived", diff --git a/crates/bin/ampd/Cargo.toml b/crates/bin/ampd/Cargo.toml index 912b38529..23d9938fe 100644 --- a/crates/bin/ampd/Cargo.toml +++ b/crates/bin/ampd/Cargo.toml @@ -14,6 +14,7 @@ console-subscriber = ["dep:console-subscriber"] snmalloc = ["dep:snmalloc-rs"] [dependencies] +amp-data-store = { path = "../../core/data-store" } amp-object-store = { path = "../../core/object-store" } clap.workspace = true common = { path = "../../core/common" } diff --git a/crates/bin/ampd/src/controller_cmd.rs b/crates/bin/ampd/src/controller_cmd.rs index 3eab12ce4..b2ddc86de 100644 --- a/crates/bin/ampd/src/controller_cmd.rs +++ b/crates/bin/ampd/src/controller_cmd.rs @@ -1,11 +1,12 @@ use std::{net::SocketAddr, sync::Arc}; use amp_config::Config as CommonConfig; +use amp_data_store::DataStore; use amp_dataset_store::{ DatasetStore, manifests::DatasetManifestsStore, providers::ProviderConfigsStore, }; use amp_object_store::ObjectStoreCreationError; -use common::{BoxError, store::Store}; +use common::BoxError; use controller::config::Config; use monitoring::telemetry::metrics::Meter; @@ -16,8 +17,12 @@ pub async fn run(config: CommonConfig, meter: Option, at: SocketAddr) -> .await .map_err(|err| Error::MetadataDbConnection(Box::new(err)))?; - let data_store = Store::new(metadata_db.clone(), config.data_store_url.clone()) - .map_err(Error::DataStoreCreation)?; + let data_store = DataStore::new( + metadata_db.clone(), + config.data_store_url.clone(), + config.parquet.cache_size_mb, + ) + .map_err(Error::DataStoreCreation)?; let dataset_store = { let provider_configs_store = ProviderConfigsStore::new( diff --git a/crates/bin/ampd/src/server_cmd.rs b/crates/bin/ampd/src/server_cmd.rs index 7c1257791..f686c0dfa 100644 --- a/crates/bin/ampd/src/server_cmd.rs +++ b/crates/bin/ampd/src/server_cmd.rs @@ -1,11 +1,12 @@ use std::sync::Arc; use amp_config::{Addrs, Config as CommonConfig}; +use amp_data_store::DataStore; use amp_dataset_store::{ DatasetStore, manifests::DatasetManifestsStore, providers::ProviderConfigsStore, }; use amp_object_store::ObjectStoreCreationError; -use common::{BoxError, store::Store}; +use common::BoxError; use monitoring::telemetry::metrics::Meter; use server::config::Config as ServerConfig; @@ -21,8 +22,12 @@ pub async fn run( .await .map_err(|err| Error::MetadataDbConnection(Box::new(err)))?; - let data_store = Store::new(metadata_db.clone(), config.data_store_url.clone()) - .map_err(Error::DataStoreCreation)?; + let data_store = DataStore::new( + metadata_db.clone(), + config.data_store_url.clone(), + config.parquet.cache_size_mb, + ) + .map_err(Error::DataStoreCreation)?; let dataset_store = { let provider_configs_store = ProviderConfigsStore::new( @@ -146,6 +151,5 @@ pub fn config_from_common(config: &CommonConfig) -> ServerConfig { max_mem_mb: config.max_mem_mb, query_max_mem_mb: config.query_max_mem_mb, spill_location: config.spill_location.clone(), - parquet_cache_size_mb: config.parquet.cache_size_mb, } } diff --git a/crates/bin/ampd/src/solo_cmd.rs b/crates/bin/ampd/src/solo_cmd.rs index e9c3b9fd7..57657002e 100644 --- a/crates/bin/ampd/src/solo_cmd.rs +++ b/crates/bin/ampd/src/solo_cmd.rs @@ -1,11 +1,12 @@ use std::{future::Future, pin::Pin, sync::Arc}; use amp_config::Config as CommonConfig; +use amp_data_store::DataStore; use amp_dataset_store::{ DatasetStore, manifests::DatasetManifestsStore, providers::ProviderConfigsStore, }; use amp_object_store::ObjectStoreCreationError; -use common::{BoxError, store::Store}; +use common::BoxError; use monitoring::telemetry::metrics::Meter; use crate::{controller_cmd, server_cmd, worker_cmd}; @@ -24,8 +25,12 @@ pub async fn run( .await .map_err(|err| Error::MetadataDbConnection(Box::new(err)))?; - let data_store = Store::new(metadata_db.clone(), config.data_store_url.clone()) - .map_err(Error::DataStoreCreation)?; + let data_store = DataStore::new( + metadata_db.clone(), + config.data_store_url.clone(), + config.parquet.cache_size_mb, + ) + .map_err(Error::DataStoreCreation)?; let dataset_store = { let provider_configs_store = ProviderConfigsStore::new( diff --git a/crates/bin/ampd/src/worker_cmd.rs b/crates/bin/ampd/src/worker_cmd.rs index fa0214778..41623d43d 100644 --- a/crates/bin/ampd/src/worker_cmd.rs +++ b/crates/bin/ampd/src/worker_cmd.rs @@ -1,9 +1,9 @@ use amp_config::Config; +use amp_data_store::DataStore; use amp_dataset_store::{ DatasetStore, manifests::DatasetManifestsStore, providers::ProviderConfigsStore, }; use amp_object_store::ObjectStoreCreationError; -use common::store::Store; use monitoring::telemetry::metrics::Meter; use worker::node_id::NodeId; @@ -13,8 +13,12 @@ pub async fn run(config: Config, meter: Option, node_id: NodeId) -> Resul .await .map_err(|err| Error::MetadataDbConnection(Box::new(err)))?; - let data_store = Store::new(metadata_db.clone(), config.data_store_url.clone()) - .map_err(Error::DataStoreCreation)?; + let data_store = DataStore::new( + metadata_db.clone(), + config.data_store_url.clone(), + config.parquet.cache_size_mb, + ) + .map_err(Error::DataStoreCreation)?; let dataset_store = { let provider_configs_store = ProviderConfigsStore::new( diff --git a/crates/config/src/lib.rs b/crates/config/src/lib.rs index f40f89acc..9b29d03a7 100644 --- a/crates/config/src/lib.rs +++ b/crates/config/src/lib.rs @@ -314,7 +314,6 @@ impl Config { self.max_mem_mb, self.query_max_mem_mb, &self.spill_location, - self.parquet.cache_size_mb, ) } diff --git a/crates/core/common/Cargo.toml b/crates/core/common/Cargo.toml index 436568f4d..75a05b842 100644 --- a/crates/core/common/Cargo.toml +++ b/crates/core/common/Cargo.toml @@ -6,7 +6,7 @@ license-file.workspace = true [dependencies] alloy.workspace = true -amp-object-store = { path = "../object-store" } +amp-data-store = { path = "../data-store" } async-stream.workspace = true async-trait.workspace = true axum.workspace = true @@ -17,9 +17,6 @@ datafusion.workspace = true datafusion-datasource.workspace = true datafusion-tracing.workspace = true datasets-common = { path = "../datasets-common" } -figment.workspace = true -foyer = "0.21" -fs-err.workspace = true futures.workspace = true governor.workspace = true indoc.workspace = true diff --git a/crates/core/common/src/catalog/physical.rs b/crates/core/common/src/catalog/physical.rs index 1d94192e2..ad423a195 100644 --- a/crates/core/common/src/catalog/physical.rs +++ b/crates/core/common/src/catalog/physical.rs @@ -1,5 +1,9 @@ use std::{ops::RangeInclusive, sync::Arc}; +use amp_data_store::{ + DataStore, + physical_table::{PhyTablePath, PhyTableRevisionPath, PhyTableUrl}, +}; use datafusion::{ arrow::datatypes::SchemaRef, catalog::{Session, memory::DataSourceExec}, @@ -17,7 +21,7 @@ use datafusion::{ prelude::Expr, }; use datafusion_datasource::compute_all_files_statistics; -use datasets_common::{hash_reference::HashReference, name::Name, table_name::TableName}; +use datasets_common::{hash_reference::HashReference, table_name::TableName}; use futures::{Stream, StreamExt, TryStreamExt, stream, stream::BoxStream}; use metadata_db::LocationId; use object_store::{ObjectMeta, ObjectStore, path::Path}; @@ -33,12 +37,8 @@ use crate::{ segments::{BlockRange, Chain, Segment, canonical_chain, missing_ranges}, }, sql::TableReference, - store::{CachedStore, Store}, }; -/// Path delimiter used in object store paths. -const PATH_DELIMITER: char = '/'; - #[derive(Debug, Clone)] pub struct Catalog { tables: Vec>, @@ -74,9 +74,7 @@ impl Catalog { let mut earliest = None; for table in &self.tables { // Create a snapshot to get synced range - // Use empty cache (0 bytes) since we're only checking metadata - let dummy_cached_store = CachedStore::new(table.store.clone(), 0); - let snapshot = table.snapshot(false, dummy_cached_store).await?; + let snapshot = table.snapshot(false, table.store.clone()).await?; let synced_range = snapshot.synced_range(); match (earliest, &synced_range) { (None, Some(range)) => earliest = Some(range.start()), @@ -97,7 +95,7 @@ impl CatalogSnapshot { pub async fn from_catalog( catalog: Catalog, ignore_canonical_segments: bool, - store: CachedStore, + store: DataStore, ) -> Result { let mut table_snapshots = Vec::new(); for physical_table in &catalog.tables { @@ -148,7 +146,7 @@ impl CatalogSnapshot { /// table. Only active revisions are used for query execution. #[tracing::instrument(skip_all, fields(table = %table), err)] pub async fn register_new_table_revision( - store: Store, + store: DataStore, dataset: HashReference, table: ResolvedTable, ) -> Result { @@ -180,7 +178,7 @@ pub async fn register_new_table_revision( /// This error type is used by `register_new_table_revision()`. #[derive(Debug, thiserror::Error)] #[error("Failed to register and activate new table revision")] -pub struct RegisterNewTableRevisionError(#[source] pub crate::store::RegisterTableRevisionError); +pub struct RegisterNewTableRevisionError(#[source] pub amp_data_store::RegisterTableRevisionError); #[derive(Debug, Clone)] pub struct PhysicalTable { @@ -197,7 +195,7 @@ pub struct PhysicalTable { /// Location ID in the metadata database. location_id: LocationId, /// Data store for accessing metadata database and object storage. - store: Store, + store: DataStore, } // Methods for creating and managing PhysicalTable instances @@ -247,7 +245,7 @@ impl PhysicalTable { /// partway through, the revision will be marked as active but with incomplete file metadata. /// Only the physical table registration (steps 3-4) is transactional. pub async fn restore_latest_revision( - store: Store, + store: DataStore, dataset: &HashReference, table: &ResolvedTable, ) -> Result, RestoreLatestRevisionError> { @@ -286,7 +284,9 @@ impl PhysicalTable { let (file_name, amp_meta, footer) = amp_metadata_from_parquet_file(&store, &object_meta) .await - .map_err(RestoreLatestRevisionError::ReadParquetMetadata)?; + .map_err(|e: BoxError| { + RestoreLatestRevisionError::ReadParquetMetadata(e) + })?; let parquet_meta_json = serde_json::to_value(amp_meta) .map_err(RestoreLatestRevisionError::SerializeMetadata)?; @@ -342,7 +342,10 @@ impl PhysicalTable { /// /// Returns the PhysicalTable if an active revision exists, or None if no active /// revision is found in the metadata database. - pub async fn get_active(store: Store, table: ResolvedTable) -> Result, BoxError> { + pub async fn get_active( + store: DataStore, + table: ResolvedTable, + ) -> Result, BoxError> { let manifest_hash = table.dataset().manifest_hash(); let table_name = table.name(); @@ -468,7 +471,7 @@ impl PhysicalTable { /// Returns a stream of object metadata for each file in the table's directory. pub fn list_files( &self, - ) -> BoxStream<'_, Result> + ) -> BoxStream<'_, Result> { self.store.stream_revision_files_in_object_store(&self.path) } @@ -525,7 +528,7 @@ impl PhysicalTable { pub async fn snapshot( &self, ignore_canonical_segments: bool, - store: CachedStore, + store: DataStore, ) -> Result { let canonical_segments = if ignore_canonical_segments { self.segments().await? @@ -782,14 +785,14 @@ pub enum RestoreLatestRevisionError { /// This occurs when the object store cannot be queried for existing revisions, /// typically due to network issues, permission errors, or storage unavailability. #[error("Failed to find latest revision from object store")] - FindLatestRevision(#[source] crate::store::FindLatestTableRevisionInObjectStoreError), + FindLatestRevision(#[source] amp_data_store::FindLatestTableRevisionInObjectStoreError), /// Failed to register revision in metadata database /// /// This occurs when the metadata database transaction for registering the /// physical table and marking it as active fails. #[error("Failed to register revision in metadata database")] - RegisterRevision(#[source] crate::store::RegisterTableRevisionError), + RegisterRevision(#[source] amp_data_store::RegisterTableRevisionError), /// Failed to begin transaction for marking table active #[error("Failed to begin transaction")] @@ -815,7 +818,7 @@ pub enum RestoreLatestRevisionError { /// Failed to list files in the restored revision #[error("Failed to list files in restored revision")] - ListFiles(#[source] crate::store::ListRevisionFilesInObjectStoreError), + ListFiles(#[source] amp_data_store::ListRevisionFilesInObjectStoreError), /// Failed to read Amp metadata from parquet file /// @@ -834,261 +837,3 @@ pub enum RestoreLatestRevisionError { #[error("Failed to register file in metadata database")] RegisterFile(#[source] metadata_db::Error), } - -/// Physical table URL _new-type_ wrapper -/// -/// Represents a base directory URL in the object store containing all parquet files for a table. -/// Individual file URLs are constructed by appending the filename to this base URL. -/// -/// ## URL Format -/// -/// `////` -/// -/// Where: -/// - `store_base_url`: Object store base URL, may include path prefix after bucket -/// (e.g., `s3://bucket/prefix`, `file:///data/subdir`) -/// - `dataset_name`: Dataset name (without namespace) -/// - `table_name`: Table name -/// - `revision_id`: Unique identifier for this table revision (typically UUIDv7) -/// -/// ## Example -/// -/// ```text -/// s3://my-bucket/prefix/ethereum_mainnet/logs/01234567-89ab-cdef-0123-456789abcdef/ -/// ``` -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct PhyTableUrl(Url); - -impl PhyTableUrl { - /// Constructs a table URL from a base URL and revision path. - /// - /// The URL follows the structure: `//` - /// - /// Where: - /// - `base_url`: Object store root URL (e.g., `s3://bucket/prefix/`, `file:///data/`) - /// - `revision_path`: Complete path to the table revision (dataset_name/table_name/revision_uuid) - pub fn new(base_url: &Url, revision_path: &PhyTableRevisionPath) -> Self { - // SAFETY: Path components (Name, TableName, Uuid) contain only URL-safe characters - let raw_url = base_url - .join(&format!("{}/", revision_path)) - .expect("path is URL-safe"); - PhyTableUrl(raw_url) - } - - /// Get the URL as a string slice - pub fn as_str(&self) -> &str { - self.0.as_str() - } - - /// Get a reference to the inner [`Url`] - pub fn inner(&self) -> &Url { - &self.0 - } -} - -impl std::str::FromStr for PhyTableUrl { - type Err = PhyTableUrlParseError; - - fn from_str(s: &str) -> Result { - let url = s.parse().map_err(|err| PhyTableUrlParseError { - url: s.to_string(), - source: err, - })?; - Ok(PhyTableUrl(url)) - } -} - -impl std::fmt::Display for PhyTableUrl { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.0.as_str()) - } -} - -/// Path to a table directory in object storage (without revision). -/// -/// Represents the parent directory containing all revisions of a table. -/// Format: `/` -/// -/// **NOTE**: The underlying [`object_store::Path`] type automatically strips leading and -/// trailing slashes, so the string representation will not contain a trailing slash. -/// -/// ## Example -/// -/// ```text -/// ethereum_mainnet/logs -/// ``` -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct PhyTablePath(Path); - -impl PhyTablePath { - /// Constructs the path to a table directory (without revision). - pub fn new(dataset_name: impl AsRef, table_name: impl AsRef) -> Self { - Self(format!("{}/{}", dataset_name.as_ref(), table_name.as_ref()).into()) - } - - /// Create a revision path by appending the given revision ID to this table path. - pub fn with_revision(&self, revision_id: impl AsRef) -> PhyTableRevisionPath { - let path = self.0.child(revision_id.as_ref().to_string()); - PhyTableRevisionPath(path) - } - - /// Get a reference to the underlying [`object_store::path::Path`] - pub fn as_object_store_path(&self) -> &Path { - &self.0 - } - - /// Get the path as a string slice - pub fn as_str(&self) -> &str { - self.0.as_ref() - } -} - -impl std::fmt::Display for PhyTablePath { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - self.0.fmt(f) - } -} - -impl std::ops::Deref for PhyTablePath { - type Target = Path; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -/// Path to a table revision directory in object storage. -/// -/// Represents a specific revision of a table, identified by a UUID. -/// Format: `//` -/// -/// **NOTE**: The underlying [`object_store::Path`] type automatically strips leading and -/// trailing slashes, so the string representation will not contain a trailing slash. -/// -/// ## Example -/// -/// ```text -/// ethereum_mainnet/logs/01234567-89ab-cdef-0123-456789abcdef -/// ``` -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct PhyTableRevisionPath(Path); - -impl PhyTableRevisionPath { - /// Constructs the path to a table revision directory. - pub fn new( - dataset_name: impl AsRef, - table_name: impl AsRef, - revision_id: impl AsRef, - ) -> Self { - Self( - format!( - "{}/{}/{}", - dataset_name.as_ref(), - table_name.as_ref(), - revision_id.as_ref() - ) - .into(), - ) - } - - /// Get a reference to the underlying [`Path`] - pub fn as_object_store_path(&self) -> &Path { - &self.0 - } - - /// Get the path as a string slice - pub fn as_str(&self) -> &str { - self.0.as_ref() - } -} - -impl std::str::FromStr for PhyTableRevisionPath { - type Err = PhyTableRevisionPathError; - - fn from_str(s: &str) -> Result { - let mut parts = s.trim_end_matches(PATH_DELIMITER).split(PATH_DELIMITER); - - let revision_uuid: Uuid = parts - .next_back() - .filter(|s| !s.is_empty()) - .ok_or(PhyTableRevisionPathError::NotEnoughComponents(0))? - .parse() - .map_err(PhyTableRevisionPathError::InvalidRevisionUuid)?; - - let table_name: TableName = parts - .next_back() - .filter(|s| !s.is_empty()) - .ok_or(PhyTableRevisionPathError::NotEnoughComponents(1))? - .parse() - .map_err(PhyTableRevisionPathError::InvalidTableName)?; - - let dataset_name: Name = parts - .next_back() - .filter(|s| !s.is_empty()) - .ok_or(PhyTableRevisionPathError::NotEnoughComponents(2))? - .parse() - .map_err(PhyTableRevisionPathError::InvalidDatasetName)?; - - Ok(Self::new(dataset_name, table_name, revision_uuid)) - } -} - -impl std::fmt::Display for PhyTableRevisionPath { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - self.0.fmt(f) - } -} - -impl std::ops::Deref for PhyTableRevisionPath { - type Target = Path; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -/// Error when parsing a path into a [`PhyTableRevisionPath`] -#[derive(Debug, thiserror::Error)] -pub enum PhyTableRevisionPathError { - #[error("path must have at least 3 components, got {0}")] - NotEnoughComponents(usize), - - #[error("invalid dataset name")] - InvalidDatasetName(#[source] datasets_common::name::NameError), - - #[error("invalid table name")] - InvalidTableName(#[source] datasets_common::table_name::TableNameError), - - #[error("invalid revision UUID")] - InvalidRevisionUuid(#[source] uuid::Error), -} - -/// Error type for PhyTableUrl parsing -#[derive(Debug, thiserror::Error)] -#[error("invalid object store URL '{url}'")] -pub struct PhyTableUrlParseError { - url: String, - #[source] - source: url::ParseError, -} - -impl From for metadata_db::physical_table::TablePathOwned { - fn from(value: PhyTableRevisionPath) -> Self { - metadata_db::physical_table::TablePath::from_owned_unchecked(value.as_str().to_owned()) - } -} - -impl<'a> From<&'a PhyTableRevisionPath> for metadata_db::physical_table::TablePath<'a> { - fn from(value: &'a PhyTableRevisionPath) -> Self { - metadata_db::physical_table::TablePath::from_ref_unchecked(value.as_str()) - } -} - -impl From for PhyTableRevisionPath { - fn from(value: metadata_db::physical_table::TablePathOwned) -> Self { - value - .as_str() - .parse() - .expect("database path should be a valid revision path") - } -} diff --git a/crates/core/common/src/catalog/reader.rs b/crates/core/common/src/catalog/reader.rs index b666d984d..346f69165 100644 --- a/crates/core/common/src/catalog/reader.rs +++ b/crates/core/common/src/catalog/reader.rs @@ -1,5 +1,6 @@ use std::{ops::Range, sync::Arc}; +use amp_data_store::{CachedParquetData, DataStore}; use bytes::Bytes; use datafusion::{ arrow::datatypes::SchemaRef, @@ -18,15 +19,12 @@ use datafusion::{ use futures::future::BoxFuture; use metadata_db::{LocationId, files::FileId}; -use crate::{ - BoxError, - store::{CachedParquetData, CachedStore}, -}; +use crate::BoxError; #[derive(Debug, Clone)] pub struct AmpReaderFactory { pub location_id: LocationId, - pub store: CachedStore, + pub store: DataStore, pub schema: SchemaRef, } @@ -35,7 +33,7 @@ impl AmpReaderFactory { self.store .get_cached_parquet_metadata(file, self.schema.clone()) .await - .map_err(|e| e.into()) + .map_err(|err| -> BoxError { err.into() }) } } @@ -81,7 +79,7 @@ pub struct AmpReader { pub file_id: FileId, pub file_metrics: ParquetFileMetrics, pub inner: ParquetObjectReader, - pub store: CachedStore, + pub store: DataStore, pub schema: SchemaRef, } @@ -114,7 +112,9 @@ impl AsyncFileReader for AmpReader { .get_cached_parquet_metadata(file_id, schema) .await .map(|cached| cached.metadata) - .map_err(|err| ParquetError::External(err.into())) + .map_err(|err: amp_data_store::GetCachedMetadataError| { + ParquetError::External(err.into()) + }) }) } } diff --git a/crates/core/common/src/catalog/sql.rs b/crates/core/common/src/catalog/sql.rs index cf362629a..31df84798 100644 --- a/crates/core/common/src/catalog/sql.rs +++ b/crates/core/common/src/catalog/sql.rs @@ -69,6 +69,7 @@ use std::{ sync::Arc, }; +use amp_data_store::DataStore; use datafusion::{logical_expr::ScalarUDF, sql::parser::Statement}; use datasets_common::{ func_name::ETH_CALL_FUNCTION_NAME, hash::Hash, partial_reference::PartialReference, @@ -83,7 +84,7 @@ use super::{ physical::{Catalog, PhysicalTable}, }; use crate::{ - PlanningContext, ResolvedTable, Store, + PlanningContext, ResolvedTable, query_context::QueryEnv, sql::{ FunctionReference, TableReference, resolve_function_references, resolve_table_references, @@ -113,7 +114,7 @@ use crate::{ /// 4. Constructs physical catalog for query execution pub async fn catalog_for_sql( dataset_store: &impl DatasetAccess, - data_store: &Store, + data_store: &DataStore, query: &Statement, env: QueryEnv, ) -> Result { diff --git a/crates/core/common/src/lib.rs b/crates/core/common/src/lib.rs index 8b924a216..f031a851d 100644 --- a/crates/core/common/src/lib.rs +++ b/crates/core/common/src/lib.rs @@ -10,7 +10,6 @@ pub mod planning_context; pub mod query_context; pub mod sql; pub mod sql_str; -pub mod store; pub mod stream_helpers; pub mod utils; @@ -29,14 +28,11 @@ use datafusion::arrow::{ error::ArrowError, }; pub use datafusion::{arrow, parquet}; -pub use foyer::Cache; use futures::{Stream, StreamExt}; use metadata::segments::BlockRange; -use metadata_db::files::FileId; pub use planning_context::{DetachedLogicalPlan, PlanningContext}; pub use query_context::{Error as QueryError, QueryContext}; use serde::{Deserialize, Serialize}; -pub use store::{CachedStore, Store}; pub type BoxError = Box; pub type BoxResult = Result; @@ -59,9 +55,6 @@ pub type EvmAddressArrayType = FixedSizeBinaryArray; /// Payment amount in the EVM. Used for gas or value transfers. pub const EVM_CURRENCY_TYPE: DataType = DataType::Decimal128(DECIMAL128_MAX_PRECISION, 0); -pub use store::CachedParquetData; -pub type ParquetFooterCache = Cache; - #[derive(Clone, Copy, Debug, Default, Deserialize, Serialize)] pub struct Timestamp(pub Duration); diff --git a/crates/core/common/src/metadata.rs b/crates/core/common/src/metadata.rs index 6d5ab0d1b..e43eec80e 100644 --- a/crates/core/common/src/metadata.rs +++ b/crates/core/common/src/metadata.rs @@ -1,5 +1,6 @@ use std::sync::Arc; +use amp_data_store::{DataStore, file_name::FileName}; use datafusion::parquet::{ arrow::{arrow_reader::ArrowReaderOptions, async_reader::AsyncFileReader}, errors::ParquetError, @@ -12,17 +13,15 @@ use metadata_db::{ use object_store::{ObjectMeta, path::Path}; use tracing::instrument; -mod file_name; pub mod parquet; pub mod segments; mod size; use self::parquet::{PARQUET_METADATA_KEY, ParquetMeta}; -pub use self::{ - file_name::FileName, - size::{Generation, Overflow, SegmentSize, get_block_count, le_bytes_to_nonzero_i64_opt}, +pub use self::size::{ + Generation, Overflow, SegmentSize, get_block_count, le_bytes_to_nonzero_i64_opt, }; -use crate::{BoxError, Store}; +use crate::BoxError; #[derive(Debug, Clone)] pub struct FileMetadata { @@ -83,7 +82,7 @@ impl FileMetadata { #[instrument(skip(object_meta, store), err)] pub async fn extract_footer_bytes_from_file( - store: &Store, + store: &DataStore, object_meta: &ObjectMeta, ) -> Result { let parquet_metadata = extract_parquet_metadata_from_file(store, object_meta).await?; @@ -95,7 +94,7 @@ pub async fn extract_footer_bytes_from_file( #[instrument(skip(object_meta, store), err)] pub async fn amp_metadata_from_parquet_file( - store: &Store, + store: &DataStore, object_meta: &ObjectMeta, ) -> Result<(FileName, ParquetMeta, FooterBytes), BoxError> { let parquet_metadata = extract_parquet_metadata_from_file(store, object_meta).await?; @@ -146,7 +145,7 @@ pub async fn amp_metadata_from_parquet_file( } async fn extract_parquet_metadata_from_file( - store: &Store, + store: &DataStore, object_meta: &ObjectMeta, ) -> Result, ParquetError> { let mut reader = store diff --git a/crates/core/common/src/metadata/parquet.rs b/crates/core/common/src/metadata/parquet.rs index 88344c40b..15ecc84fe 100644 --- a/crates/core/common/src/metadata/parquet.rs +++ b/crates/core/common/src/metadata/parquet.rs @@ -18,12 +18,10 @@ //! //! See also: metadata-consistency +use amp_data_store::file_name::FileName; use serde::{Deserialize, Serialize}; -use crate::{ - Timestamp, - metadata::{file_name::FileName, segments::BlockRange}, -}; +use crate::{Timestamp, metadata::segments::BlockRange}; pub const PARQUET_METADATA_KEY: &str = "nozzle_metadata"; pub const PARENT_FILE_ID_METADATA_KEY: &str = "parent_file_ids"; diff --git a/crates/core/common/src/metadata/size.rs b/crates/core/common/src/metadata/size.rs index 5c236fb6e..cec7f6786 100644 --- a/crates/core/common/src/metadata/size.rs +++ b/crates/core/common/src/metadata/size.rs @@ -835,12 +835,11 @@ pub fn le_bytes_to_nonzero_i64_opt(bytes: &[u8]) -> Result, T pub mod test { use std::sync::Arc; + use amp_data_store::file_name::FileName; + use crate::{ Timestamp, - metadata::{ - file_name::FileName, - parquet::{GENERATION_METADATA_KEY, PARQUET_METADATA_KEY, ParquetMeta}, - }, + metadata::parquet::{GENERATION_METADATA_KEY, PARQUET_METADATA_KEY, ParquetMeta}, parquet::{ basic::{Repetition, Type as PhysicalType}, file::{ diff --git a/crates/core/common/src/query_context.rs b/crates/core/common/src/query_context.rs index a2100874e..ed583a9c9 100644 --- a/crates/core/common/src/query_context.rs +++ b/crates/core/common/src/query_context.rs @@ -3,6 +3,7 @@ use std::{ sync::{Arc, LazyLock}, }; +use amp_data_store::DataStore; use arrow::{array::ArrayRef, compute::concat_batches}; use axum::response::IntoResponse; use datafusion::{ @@ -32,10 +33,10 @@ use futures::{TryStreamExt, stream}; use js_runtime::isolate_pool::IsolatePool; use regex::Regex; use thiserror::Error; -use tracing::{debug, field, instrument}; +use tracing::{field, instrument}; use crate::{ - BlockNum, BoxError, Store, arrow, + BlockNum, BoxError, arrow, catalog::physical::{Catalog, CatalogSnapshot, TableSnapshot}, evm::udfs::{ EvmDecodeLog, EvmDecodeParams, EvmDecodeType, EvmEncodeParams, EvmEncodeType, EvmTopic, @@ -47,7 +48,6 @@ use crate::{ forbid_underscore_prefixed_aliases, }, sql::TableReference, - store::CachedStore, utils::error_with_causes, }; @@ -128,7 +128,6 @@ pub struct QueryEnv { // Per-query memory limit configuration pub query_max_mem_mb: usize, - pub parquet_cache_size_mb: u64, } /// Creates a QueryEnv with specified memory and cache configuration @@ -139,7 +138,6 @@ pub fn create_query_env( max_mem_mb: usize, query_max_mem_mb: usize, spill_location: &[PathBuf], - parquet_cache_size_mb: u64, ) -> Result { let spill_allowed = !spill_location.is_empty(); let disk_manager_mode = if spill_allowed { @@ -171,7 +169,6 @@ pub fn create_query_env( object_store_registry: runtime_env.object_store_registry, isolate_pool, query_max_mem_mb, - parquet_cache_size_mb, }) } @@ -182,15 +179,15 @@ pub struct QueryContext { session_config: SessionConfig, catalog: CatalogSnapshot, /// Per-query memory pool (if per-query limits are enabled) - tiered_memory_pool: Arc, - store: CachedStore, + tiered_memory_pool: Arc, + store: DataStore, } impl QueryContext { pub async fn for_catalog( catalog: Catalog, env: QueryEnv, - store: Store, + store: DataStore, ignore_canonical_segments: bool, ) -> Result { // This contains various tuning options for the query engine. @@ -222,9 +219,6 @@ impl QueryContext { opts.execution.parquet.pushdown_filters = true; } - // Create cached store with parquet metadata caching - let store = CachedStore::new(store, env.parquet_cache_size_mb); - // Create a catalog snapshot with canonical chain locked in let catalog_snapshot = CatalogSnapshot::from_catalog(catalog, ignore_canonical_segments, store.clone()) @@ -413,7 +407,7 @@ fn read_only_check(plan: &LogicalPlan) -> Result<(), Error> { fn register_table( ctx: &SessionContext, table: Arc, - store: &Store, + store: &DataStore, ) -> Result<(), DataFusionError> { // The catalog schema needs to be explicitly created or table creation will fail. create_catalog_schema(ctx, table.physical_table().catalog_schema().to_string()); @@ -467,7 +461,7 @@ async fn execute_plan( use datafusion::physical_plan::execute_stream; read_only_check(&plan)?; - debug!("logical plan: {}", plan.to_string().replace('\n', "\\n")); + tracing::debug!("logical plan: {}", plan.to_string().replace('\n', "\\n")); if logical_optimize { plan = ctx.state().optimize(&plan).map_err(Error::PlanningError)?; @@ -483,7 +477,7 @@ async fn execute_plan( forbid_duplicate_field_names(&physical_plan, &plan).map_err(Error::PlanningError)?; - debug!("physical plan: {}", print_physical_plan(&*physical_plan)); + tracing::debug!("physical plan: {}", print_physical_plan(&*physical_plan)); match is_explain { false => execute_stream(physical_plan, ctx.task_ctx()).map_err(Error::PlanningError), diff --git a/crates/core/data-store/Cargo.toml b/crates/core/data-store/Cargo.toml new file mode 100644 index 000000000..ff1ddd84f --- /dev/null +++ b/crates/core/data-store/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "amp-data-store" +edition.workspace = true +version.workspace = true +license-file.workspace = true + +[dependencies] +amp-object-store = { path = "../object-store" } +bytes.workspace = true +datafusion.workspace = true +datasets-common = { path = "../datasets-common" } +foyer = "0.21" +futures.workspace = true +metadata-db = { path = "../metadata-db" } +object_store.workspace = true +rand.workspace = true +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true +url.workspace = true +uuid.workspace = true diff --git a/crates/core/common/src/metadata/file_name.rs b/crates/core/data-store/src/file_name.rs similarity index 99% rename from crates/core/common/src/metadata/file_name.rs rename to crates/core/data-store/src/file_name.rs index 86bae446d..d67103958 100644 --- a/crates/core/common/src/metadata/file_name.rs +++ b/crates/core/data-store/src/file_name.rs @@ -1,4 +1,5 @@ -use crate::BlockNum; +/// Block number type alias. +pub type BlockNum = u64; /// A validated file name for parquet files. /// diff --git a/crates/core/common/src/store.rs b/crates/core/data-store/src/lib.rs similarity index 93% rename from crates/core/common/src/store.rs rename to crates/core/data-store/src/lib.rs index b091ee514..092a01fac 100644 --- a/crates/core/common/src/store.rs +++ b/crates/core/data-store/src/lib.rs @@ -1,6 +1,6 @@ //! Object store abstraction layer. //! -//! This module provides the [`Store`] wrapper. +//! This module provides the [`DataStore`] wrapper. use std::sync::Arc; @@ -19,14 +19,19 @@ use datafusion::{ }; use datasets_common::{hash::Hash, hash_reference::HashReference, table_name::TableName}; use foyer::Cache; +// Re-export foyer::Cache for use by downstream crates +pub use foyer::Cache as FoyerCache; use futures::{Stream, StreamExt as _, TryStreamExt as _, stream::BoxStream}; use metadata_db::{LocationId, MetadataDb, PhysicalTable, files::FileId}; use object_store::{ObjectMeta, ObjectStore, buffered::BufWriter, path::Path}; use url::Url; -use crate::{ - catalog::physical::{PhyTablePath, PhyTableRevisionPath}, - metadata::FileName, +pub mod file_name; +pub mod physical_table; + +use self::{ + file_name::FileName, + physical_table::{PhyTablePath, PhyTableRevisionPath}, }; /// Data store. @@ -39,13 +44,14 @@ use crate::{ /// - Can be extended with helper functions. /// - Provides integrated metadata database operations. #[derive(Debug, Clone)] -pub struct Store { +pub struct DataStore { metadata_db: MetadataDb, object_store: Arc, url: Arc, + parquet_footer_cache: Cache, } -impl Store { +impl DataStore { /// Creates a store for an object store URL (or filesystem directory). /// /// Examples of valid formats for `data_location`: @@ -55,16 +61,27 @@ impl Store { /// - Prefixed: `s3://bucket-name/my_prefix/` /// /// If `data_location` is a relative filesystem path, then `base` will be used as the prefix. + /// + /// The `cache_size_mb` parameter controls the maximum memory footprint of the parquet + /// metadata cache. The cache uses a memory-weighted eviction policy. pub fn new( metadata_db: MetadataDb, url: ObjectStoreUrl, + cache_size_mb: u64, ) -> Result { let object_store: Arc = amp_object_store::new_with_prefix(&url, url.path())?; + + let cache_size_bytes = cache_size_mb * 1024 * 1024; + let parquet_footer_cache = foyer::CacheBuilder::new(cache_size_bytes as usize) + .with_weighter(|_k, v: &CachedParquetData| v.metadata.memory_size()) + .build(); + Ok(Self { metadata_db, object_store, url: Arc::new(url), + parquet_footer_cache, }) } @@ -92,7 +109,7 @@ impl Store { } /// Physical table revision management -impl Store { +impl DataStore { /// Registers a new physical table revision and marks it as active. /// /// This atomically registers the revision location in the metadata database @@ -184,7 +201,7 @@ impl Store { } /// Physical table files management -impl Store { +impl DataStore { /// Registers a file in the metadata database. /// /// Associates the file with a specific location ID and stores its Parquet metadata. @@ -328,10 +345,56 @@ impl Store { .map(|r| r.map_err(DeleteFilesStreamError)), ) } + + /// Gets cached parquet metadata for a file, fetching from database on cache miss. + /// + /// This method encapsulates the cache lookup and database fetch logic. On cache miss, + /// it retrieves the footer bytes from the metadata database, parses the parquet metadata, + /// computes statistics, and stores the result in the cache. + /// + /// The `schema` parameter is required to compute DataFusion statistics from the parquet + /// metadata. + pub async fn get_cached_parquet_metadata( + &self, + file_id: FileId, + schema: SchemaRef, + ) -> Result { + let cache = self.parquet_footer_cache.clone(); + let metadata_db = self.metadata_db.clone(); + let file_id1 = file_id; + + cache + .get_or_fetch(&file_id1, || async move { + // Cache miss, fetch from database + let footer = metadata_db::files::get_footer_bytes(&metadata_db, file_id1) + .await + .map_err(GetCachedMetadataError::FetchFooter)?; + + let metadata = Arc::new( + ParquetMetaDataReader::new() + .with_page_index_policy(PageIndexPolicy::Required) + .parse_and_finish(&Bytes::from_owner(footer)) + .map_err(GetCachedMetadataError::ParseMetadata)?, + ); + + let statistics = Arc::new( + DFParquetMetadata::statistics_from_parquet_metadata(&metadata, &schema) + .map_err(GetCachedMetadataError::ComputeStatistics)?, + ); + + Ok::(CachedParquetData { + metadata, + statistics, + }) + }) + .await + .map(|entry| entry.value().clone()) + .map_err(GetCachedMetadataError::CacheError) + } } /// Object store file readers and writers -impl Store { +impl DataStore { /// Creates a buffered writer for writing a file to a table revision. pub fn create_revision_file_writer( &self, @@ -610,118 +673,6 @@ pub struct DeleteFileInObjectStoreError(#[source] pub object_store::Error); #[error("Failed to delete file during streaming deletion")] pub struct DeleteFilesStreamError(#[source] pub object_store::Error); -/// Cached data store. -/// -/// A wrapper around [`Store`] that adds parquet metadata caching for query execution. -/// -/// This type provides: -/// - All [`Store`] functionality via [`Deref`] -/// - Cached access to parquet file metadata via `get_cached_parquet_metadata()` -/// - Automatic cache management with configurable size -/// -/// Use this type in query contexts where repeated metadata access benefits from caching. -/// For non-query contexts (dump, GC, etc.), use [`Store`] directly. -#[derive(Debug, Clone)] -pub struct CachedStore { - store: Store, - parquet_footer_cache: Cache, -} - -impl CachedStore { - /// Creates a cached store with the specified cache size. - /// - /// The cache uses a memory-weighted eviction policy based on the size of cached - /// parquet metadata. The `cache_size_mb` parameter controls the maximum memory - /// footprint of the cache. - pub fn new(store: Store, cache_size_mb: u64) -> Self { - let cache_size_bytes = cache_size_mb * 1024 * 1024; - let parquet_footer_cache = foyer::CacheBuilder::new(cache_size_bytes as usize) - .with_weighter(|_k, v: &CachedParquetData| v.metadata.memory_size()) - .build(); - - Self { - store, - parquet_footer_cache, - } - } - - /// Creates a cached store from existing store and cache instances. - /// - /// Use this when you already have a cache instance that should be shared - /// across multiple components. - pub fn from_parts( - store: Store, - parquet_footer_cache: Cache, - ) -> Self { - Self { - store, - parquet_footer_cache, - } - } - - /// Returns a clone of the inner `Store`. - /// - /// Use this when you need to pass a `Store` to code that doesn't support `CachedStore`. - /// Since `Store` is cheap to clone (uses `Arc` internally), this operation is efficient. - pub fn as_store(&self) -> Store { - self.store.clone() - } - - /// Gets cached parquet metadata for a file, fetching from database on cache miss. - /// - /// This method encapsulates the cache lookup and database fetch logic. On cache miss, - /// it retrieves the footer bytes from the metadata database, parses the parquet metadata, - /// computes statistics, and stores the result in the cache. - /// - /// The `schema` parameter is required to compute DataFusion statistics from the parquet - /// metadata. - pub async fn get_cached_parquet_metadata( - &self, - file_id: FileId, - schema: SchemaRef, - ) -> Result { - let cache = self.parquet_footer_cache.clone(); - let metadata_db = self.store.metadata_db.clone(); - let file_id1 = file_id; - - cache - .get_or_fetch(&file_id1, || async move { - // Cache miss, fetch from database - let footer = metadata_db::files::get_footer_bytes(&metadata_db, file_id1) - .await - .map_err(GetCachedMetadataError::FetchFooter)?; - - let metadata = Arc::new( - ParquetMetaDataReader::new() - .with_page_index_policy(PageIndexPolicy::Required) - .parse_and_finish(&Bytes::from_owner(footer)) - .map_err(GetCachedMetadataError::ParseMetadata)?, - ); - - let statistics = Arc::new( - DFParquetMetadata::statistics_from_parquet_metadata(&metadata, &schema) - .map_err(GetCachedMetadataError::ComputeStatistics)?, - ); - - Ok::(CachedParquetData { - metadata, - statistics, - }) - }) - .await - .map(|entry| entry.value().clone()) - .map_err(GetCachedMetadataError::CacheError) - } -} - -impl std::ops::Deref for CachedStore { - type Target = Store; - - fn deref(&self) -> &Self::Target { - &self.store - } -} - /// Cached parquet data including metadata and computed statistics. #[derive(Clone)] pub struct CachedParquetData { diff --git a/crates/core/data-store/src/physical_table.rs b/crates/core/data-store/src/physical_table.rs new file mode 100644 index 000000000..9ae95bd41 --- /dev/null +++ b/crates/core/data-store/src/physical_table.rs @@ -0,0 +1,265 @@ +use datasets_common::{name::Name, table_name::TableName}; +use object_store::path::Path; +use url::Url; +use uuid::Uuid; + +/// Path delimiter used in object store paths. +const PATH_DELIMITER: char = '/'; + +/// Physical table URL _new-type_ wrapper +/// +/// Represents a base directory URL in the object store containing all parquet files for a table. +/// Individual file URLs are constructed by appending the filename to this base URL. +/// +/// ## URL Format +/// +/// `////` +/// +/// Where: +/// - `store_base_url`: Object store base URL, may include path prefix after bucket +/// (e.g., `s3://bucket/prefix`, `file:///data/subdir`) +/// - `dataset_name`: Dataset name (without namespace) +/// - `table_name`: Table name +/// - `revision_id`: Unique identifier for this table revision (typically UUIDv7) +/// +/// ## Example +/// +/// ```text +/// s3://my-bucket/prefix/ethereum_mainnet/logs/01234567-89ab-cdef-0123-456789abcdef/ +/// ``` +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct PhyTableUrl(Url); + +impl PhyTableUrl { + /// Constructs a table URL from a base URL and revision path. + /// + /// The URL follows the structure: `//` + /// + /// Where: + /// - `base_url`: Object store root URL (e.g., `s3://bucket/prefix/`, `file:///data/`) + /// - `revision_path`: Complete path to the table revision (dataset_name/table_name/revision_uuid) + pub fn new(base_url: &Url, revision_path: &PhyTableRevisionPath) -> Self { + // SAFETY: Path components (Name, TableName, Uuid) contain only URL-safe characters + let raw_url = base_url + .join(&format!("{}/", revision_path)) + .expect("path is URL-safe"); + PhyTableUrl(raw_url) + } + + /// Get the URL as a string slice + pub fn as_str(&self) -> &str { + self.0.as_str() + } + + /// Get a reference to the inner [`Url`] + pub fn inner(&self) -> &Url { + &self.0 + } +} + +impl std::str::FromStr for PhyTableUrl { + type Err = PhyTableUrlParseError; + + fn from_str(s: &str) -> Result { + let url = s.parse().map_err(|err| PhyTableUrlParseError { + url: s.to_string(), + source: err, + })?; + Ok(PhyTableUrl(url)) + } +} + +impl std::fmt::Display for PhyTableUrl { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0.as_str()) + } +} + +/// Path to a table directory in object storage (without revision). +/// +/// Represents the parent directory containing all revisions of a table. +/// Format: `/` +/// +/// **NOTE**: The underlying [`object_store::Path`] type automatically strips leading and +/// trailing slashes, so the string representation will not contain a trailing slash. +/// +/// ## Example +/// +/// ```text +/// ethereum_mainnet/logs +/// ``` +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PhyTablePath(Path); + +impl PhyTablePath { + /// Constructs the path to a table directory (without revision). + pub fn new(dataset_name: impl AsRef, table_name: impl AsRef) -> Self { + Self(format!("{}/{}", dataset_name.as_ref(), table_name.as_ref()).into()) + } + + /// Create a revision path by appending the given revision ID to this table path. + pub fn with_revision(&self, revision_id: impl AsRef) -> PhyTableRevisionPath { + let path = self.0.child(revision_id.as_ref().to_string()); + PhyTableRevisionPath(path) + } + + /// Get a reference to the underlying [`object_store::path::Path`] + pub fn as_object_store_path(&self) -> &Path { + &self.0 + } + + /// Get the path as a string slice + pub fn as_str(&self) -> &str { + self.0.as_ref() + } +} + +impl std::fmt::Display for PhyTablePath { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} + +impl std::ops::Deref for PhyTablePath { + type Target = Path; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +/// Path to a table revision directory in object storage. +/// +/// Represents a specific revision of a table, identified by a UUID. +/// Format: `//` +/// +/// **NOTE**: The underlying [`object_store::Path`] type automatically strips leading and +/// trailing slashes, so the string representation will not contain a trailing slash. +/// +/// ## Example +/// +/// ```text +/// ethereum_mainnet/logs/01234567-89ab-cdef-0123-456789abcdef +/// ``` +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PhyTableRevisionPath(Path); + +impl PhyTableRevisionPath { + /// Constructs the path to a table revision directory. + pub fn new( + dataset_name: impl AsRef, + table_name: impl AsRef, + revision_id: impl AsRef, + ) -> Self { + Self( + format!( + "{}/{}/{}", + dataset_name.as_ref(), + table_name.as_ref(), + revision_id.as_ref() + ) + .into(), + ) + } + + /// Get a reference to the underlying [`Path`] + pub fn as_object_store_path(&self) -> &Path { + &self.0 + } + + /// Get the path as a string slice + pub fn as_str(&self) -> &str { + self.0.as_ref() + } +} + +impl std::str::FromStr for PhyTableRevisionPath { + type Err = PhyTableRevisionPathError; + + fn from_str(s: &str) -> Result { + let mut parts = s.trim_end_matches(PATH_DELIMITER).split(PATH_DELIMITER); + + let revision_uuid: Uuid = parts + .next_back() + .filter(|s| !s.is_empty()) + .ok_or(PhyTableRevisionPathError::NotEnoughComponents(0))? + .parse() + .map_err(PhyTableRevisionPathError::InvalidRevisionUuid)?; + + let table_name: TableName = parts + .next_back() + .filter(|s| !s.is_empty()) + .ok_or(PhyTableRevisionPathError::NotEnoughComponents(1))? + .parse() + .map_err(PhyTableRevisionPathError::InvalidTableName)?; + + let dataset_name: Name = parts + .next_back() + .filter(|s| !s.is_empty()) + .ok_or(PhyTableRevisionPathError::NotEnoughComponents(2))? + .parse() + .map_err(PhyTableRevisionPathError::InvalidDatasetName)?; + + Ok(Self::new(dataset_name, table_name, revision_uuid)) + } +} + +impl std::fmt::Display for PhyTableRevisionPath { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} + +impl std::ops::Deref for PhyTableRevisionPath { + type Target = Path; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +/// Error when parsing a path into a [`PhyTableRevisionPath`] +#[derive(Debug, thiserror::Error)] +pub enum PhyTableRevisionPathError { + #[error("path must have at least 3 components, got {0}")] + NotEnoughComponents(usize), + + #[error("invalid dataset name")] + InvalidDatasetName(#[source] datasets_common::name::NameError), + + #[error("invalid table name")] + InvalidTableName(#[source] datasets_common::table_name::TableNameError), + + #[error("invalid revision UUID")] + InvalidRevisionUuid(#[source] uuid::Error), +} + +/// Error type for PhyTableUrl parsing +#[derive(Debug, thiserror::Error)] +#[error("invalid object store URL '{url}'")] +pub struct PhyTableUrlParseError { + url: String, + #[source] + source: url::ParseError, +} + +impl From for metadata_db::physical_table::TablePathOwned { + fn from(value: PhyTableRevisionPath) -> Self { + metadata_db::physical_table::TablePath::from_owned_unchecked(value.as_str().to_owned()) + } +} + +impl<'a> From<&'a PhyTableRevisionPath> for metadata_db::physical_table::TablePath<'a> { + fn from(value: &'a PhyTableRevisionPath) -> Self { + metadata_db::physical_table::TablePath::from_ref_unchecked(value.as_str()) + } +} + +impl From for PhyTableRevisionPath { + fn from(value: metadata_db::physical_table::TablePathOwned) -> Self { + value + .as_str() + .parse() + .expect("database path should be a valid revision path") + } +} diff --git a/crates/core/datasets-derived/Cargo.toml b/crates/core/datasets-derived/Cargo.toml index 5c7593fc4..7b74bac6d 100644 --- a/crates/core/datasets-derived/Cargo.toml +++ b/crates/core/datasets-derived/Cargo.toml @@ -8,6 +8,7 @@ license-file.workspace = true schemars = ["dep:schemars", "datasets-common/schemars", "common/schemars", "dep:serde_json"] [dependencies] +amp-data-store = { path = "../data-store" } common = { version = "0.1.0", path = "../common" } datafusion.workspace = true datasets-common = { path = "../datasets-common" } diff --git a/crates/core/datasets-derived/src/catalog.rs b/crates/core/datasets-derived/src/catalog.rs index f62619fb8..d883ccaea 100644 --- a/crates/core/datasets-derived/src/catalog.rs +++ b/crates/core/datasets-derived/src/catalog.rs @@ -15,8 +15,9 @@ use std::{ sync::Arc, }; +use amp_data_store::DataStore; use common::{ - BoxError, PlanningContext, ResolvedTable, Store, + BoxError, PlanningContext, ResolvedTable, catalog::{ dataset_access::DatasetAccess, logical::LogicalCatalog, @@ -46,7 +47,7 @@ use crate::manifest::Function; pub async fn catalog_for_sql_with_deps( store: &impl DatasetAccess, - data_store: &Store, + data_store: &DataStore, query: &Statement, env: &QueryEnv, dependencies: &BTreeMap, @@ -72,7 +73,7 @@ pub async fn catalog_for_sql_with_deps( async fn get_physical_catalog_with_deps( dataset_store: &impl DatasetAccess, - data_store: &Store, + data_store: &DataStore, table_refs: impl IntoIterator>, function_refs: impl IntoIterator>, env: &QueryEnv, diff --git a/crates/core/dump/Cargo.toml b/crates/core/dump/Cargo.toml index b8d98d7ce..98a9a60e7 100644 --- a/crates/core/dump/Cargo.toml +++ b/crates/core/dump/Cargo.toml @@ -5,6 +5,7 @@ version.workspace = true license-file.workspace = true [dependencies] +amp-data-store = { path = "../data-store" } async-stream.workspace = true metadata-db = { path = "../metadata-db" } common = { path = "../common" } diff --git a/crates/core/dump/src/check.rs b/crates/core/dump/src/check.rs index 7a5b9fba5..cee7c35b0 100644 --- a/crates/core/dump/src/check.rs +++ b/crates/core/dump/src/check.rs @@ -1,8 +1,7 @@ use std::collections::{BTreeMap, BTreeSet}; -use common::{ - Store, catalog::physical::PhysicalTable, metadata::FileName, query_context::Error as QueryError, -}; +use amp_data_store::{DataStore, file_name::FileName}; +use common::{catalog::physical::PhysicalTable, query_context::Error as QueryError}; use futures::TryStreamExt as _; use metadata_db::LocationId; use object_store::ObjectMeta; @@ -31,7 +30,7 @@ use object_store::ObjectMeta; /// These deletions are logged at `WARN` level before execution. pub async fn consistency_check( table: &PhysicalTable, - store: &Store, + store: &DataStore, ) -> Result<(), ConsistencyError> { // See also: metadata-consistency @@ -135,7 +134,7 @@ pub enum ConsistencyError { ListObjectStore { location_id: LocationId, #[source] - source: common::store::StreamRevisionFilesInObjectStoreError, + source: amp_data_store::StreamRevisionFilesInObjectStoreError, }, /// Failed to delete orphaned file from object store diff --git a/crates/core/dump/src/compaction.rs b/crates/core/dump/src/compaction.rs index 18b021531..52beb15d3 100644 --- a/crates/core/dump/src/compaction.rs +++ b/crates/core/dump/src/compaction.rs @@ -11,8 +11,9 @@ use std::{ }; pub use algorithm::{CompactionAlgorithm, SegmentSizeLimit}; +use amp_data_store::DataStore; pub use collector::{Collector, CollectorProperties}; -use common::{CachedStore, Timestamp, catalog::physical::PhysicalTable}; +use common::{Timestamp, catalog::physical::PhysicalTable}; pub use compactor::{Compactor, CompactorProperties}; use error::{CollectionResult, CollectorError, CompactionResult, CompactorError}; use futures::FutureExt; @@ -89,7 +90,7 @@ pub struct AmpCollectorInnerTask { impl AmpCollectorInnerTask { pub fn new( metadata_db: MetadataDb, - store: CachedStore, + store: DataStore, props: Arc, table: Arc, metrics: Option>, @@ -103,7 +104,7 @@ impl AmpCollectorInnerTask { ); let collector = Collector::new( metadata_db, - store.as_store(), + store, props.clone(), table.clone(), metrics.clone(), @@ -124,7 +125,7 @@ impl AmpCollectorInnerTask { fn start( metadata_db: MetadataDb, - store: CachedStore, + store: DataStore, props: Arc, table: Arc, metrics: Option>, @@ -262,7 +263,7 @@ impl AmpCompactorTask { pub fn start( metadata_db: MetadataDb, - store: CachedStore, + store: DataStore, props: Arc, table: Arc, metrics: Option>, @@ -289,7 +290,7 @@ impl AmpCompactorTask { impl AmpCompactor { pub fn start( metadata_db: MetadataDb, - store: CachedStore, + store: DataStore, props: Arc, table: Arc, metrics: Option>, diff --git a/crates/core/dump/src/compaction/collector.rs b/crates/core/dump/src/compaction/collector.rs index f924b40d8..468395202 100644 --- a/crates/core/dump/src/compaction/collector.rs +++ b/crates/core/dump/src/compaction/collector.rs @@ -5,8 +5,9 @@ use std::{ time::Duration, }; -use common::{Store, catalog::physical::PhysicalTable, store::DeleteFilesStreamError}; -use futures::{StreamExt, TryStreamExt, stream}; +use amp_data_store::{DataStore, DeleteFilesStreamError}; +use common::catalog::physical::PhysicalTable; +use futures::{StreamExt as _, TryStreamExt as _, stream}; use metadata_db::{MetadataDb, files::FileId, gc::GcManifestRow}; use object_store::{Error as ObjectStoreError, path::Path}; @@ -36,7 +37,7 @@ impl<'a> From<&'a ParquetConfig> for CollectorProperties { #[derive(Clone)] pub struct Collector { metadata_db: MetadataDb, - store: Store, + store: DataStore, table: Arc, props: Arc, metrics: Option>, @@ -55,7 +56,7 @@ impl Debug for Collector { impl Collector { pub fn new( metadata_db: MetadataDb, - store: Store, + store: DataStore, props: Arc, table: Arc, metrics: Option>, diff --git a/crates/core/dump/src/compaction/compactor.rs b/crates/core/dump/src/compaction/compactor.rs index 5cb9e675e..5833be5fa 100644 --- a/crates/core/dump/src/compaction/compactor.rs +++ b/crates/core/dump/src/compaction/compactor.rs @@ -8,10 +8,11 @@ use std::{ time::Duration, }; +use amp_data_store::{DataStore, file_name::FileName}; use common::{ - BlockNum, CachedStore, + BlockNum, catalog::physical::PhysicalTable, - metadata::{FileName, SegmentSize, segments::BlockRange}, + metadata::{SegmentSize, segments::BlockRange}, }; use futures::{StreamExt, TryStreamExt, stream}; use metadata_db::MetadataDb; @@ -52,7 +53,7 @@ impl<'a> From<&'a ParquetConfig> for CompactorProperties { #[derive(Clone)] pub struct Compactor { metadata_db: MetadataDb, - store: CachedStore, + store: DataStore, table: Arc, props: Arc, metrics: Option>, @@ -83,7 +84,7 @@ impl Display for Compactor { impl Compactor { pub fn new( metadata_db: MetadataDb, - store: CachedStore, + store: DataStore, props: Arc, table: Arc, metrics: Option>, @@ -165,7 +166,7 @@ impl Compactor { pub struct CompactionGroup { metadata_db: MetadataDb, - store: CachedStore, + store: DataStore, props: Arc, metrics: Option>, pub(super) size: SegmentSize, @@ -187,7 +188,7 @@ impl Debug for CompactionGroup { impl CompactionGroup { pub fn new_empty( metadata_db: MetadataDb, - store: CachedStore, + store: DataStore, props: Arc, table: Arc, metrics: Option>, @@ -240,10 +241,9 @@ impl CompactionGroup { let filename = FileName::new_with_random_suffix(range.start()); let buf_writer = self .store - .as_store() .create_revision_file_writer(self.table.path(), &filename); let mut writer = ParquetFileWriter::new( - self.store.as_store(), + self.store, buf_writer, filename, Arc::clone(&self.table), @@ -346,7 +346,7 @@ impl ParquetFileWriterOutput { async fn commit_metadata(&self, metadata_db: &MetadataDb) -> Result<(), metadata_db::Error> { let location_id = self.location_id; let file_name = self.object_meta.location.filename().unwrap().to_string(); - let file_name = common::metadata::FileName::new_unchecked(file_name); + let file_name = FileName::new_unchecked(file_name); let object_size = self.object_meta.size; let object_e_tag = self.object_meta.e_tag.clone(); let object_version = self.object_meta.version.clone(); diff --git a/crates/core/dump/src/compaction/plan.rs b/crates/core/dump/src/compaction/plan.rs index 19a303a0a..bdbba844a 100644 --- a/crates/core/dump/src/compaction/plan.rs +++ b/crates/core/dump/src/compaction/plan.rs @@ -6,8 +6,8 @@ use std::{ task::{Context, Poll}, }; +use amp_data_store::DataStore; use common::{ - CachedStore, catalog::{ physical::{PhysicalTable, TableSnapshot}, reader::AmpReaderFactory, @@ -111,7 +111,7 @@ pub struct CompactionPlan<'a> { /// The metadata database for committing compaction results. metadata_db: MetadataDb, /// The data store for object storage operations. - store: CachedStore, + store: DataStore, /// Stream of files to be considered for compaction. files: BoxStream<'a, CompactionResult>, /// Compaction properties configuring the compaction algorithm @@ -138,7 +138,7 @@ impl<'a> CompactionPlan<'a> { #[tracing::instrument(skip_all)] pub fn from_snapshot( metadata_db: MetadataDb, - store: CachedStore, + store: DataStore, opts: Arc, table: &'a TableSnapshot, metrics: &Option>, diff --git a/crates/core/dump/src/config.rs b/crates/core/dump/src/config.rs index 1768bf413..0c95ccf27 100644 --- a/crates/core/dump/src/config.rs +++ b/crates/core/dump/src/config.rs @@ -39,7 +39,6 @@ impl Config { self.max_mem_mb, self.query_max_mem_mb, &self.spill_location, - self.parquet.cache_size_mb, ) } } diff --git a/crates/core/dump/src/derived_dataset.rs b/crates/core/dump/src/derived_dataset.rs index 5e3c66986..e327ea175 100644 --- a/crates/core/dump/src/derived_dataset.rs +++ b/crates/core/dump/src/derived_dataset.rs @@ -96,10 +96,11 @@ use std::{collections::BTreeMap, sync::Arc, time::Instant}; +use amp_data_store::file_name::FileName; use common::{ BlockNum, BoxError, DetachedLogicalPlan, PlanningContext, QueryContext, catalog::physical::{Catalog, PhysicalTable}, - metadata::{FileName, Generation, segments::ResumeWatermark}, + metadata::{Generation, segments::ResumeWatermark}, query_context::QueryEnv, }; use datasets_common::{deps::alias::DepAlias, hash_reference::HashReference}; @@ -502,7 +503,7 @@ async fn dump_sql_query( keep_alive_interval, ) .await? - .as_stream() + .into_stream() }; let mut microbatch_start = start; diff --git a/crates/core/dump/src/lib.rs b/crates/core/dump/src/lib.rs index d505ae078..404c16dc9 100644 --- a/crates/core/dump/src/lib.rs +++ b/crates/core/dump/src/lib.rs @@ -2,11 +2,11 @@ use std::sync::Arc; +use amp_data_store::DataStore; use amp_dataset_store::{DatasetKind, DatasetStore}; use common::{ catalog::physical::PhysicalTable, parquet::file::properties::WriterProperties as ParquetWriterProperties, - store::Store as DataStore, }; use datasets_common::hash_reference::HashReference; use metadata_db::{MetadataDb, NotificationMultiplexerHandle}; diff --git a/crates/core/dump/src/parquet_writer.rs b/crates/core/dump/src/parquet_writer.rs index 7523b0659..be1ce7dcd 100644 --- a/crates/core/dump/src/parquet_writer.rs +++ b/crates/core/dump/src/parquet_writer.rs @@ -1,11 +1,12 @@ use std::sync::Arc; +use amp_data_store::{DataStore, file_name::FileName}; use common::{ - BoxError, Store, Timestamp, + BoxError, Timestamp, arrow::array::RecordBatch, catalog::physical::PhysicalTable, metadata::{ - FileName, Generation, extract_footer_bytes_from_file, + Generation, extract_footer_bytes_from_file, parquet::{ GENERATION_METADATA_KEY, PARENT_FILE_ID_METADATA_KEY, PARQUET_METADATA_KEY, ParquetMeta, }, @@ -60,7 +61,7 @@ pub async fn commit_metadata( } pub struct ParquetFileWriter { - store: Store, + store: DataStore, writer: AsyncArrowWriter, filename: FileName, table: Arc, @@ -69,7 +70,7 @@ pub struct ParquetFileWriter { impl ParquetFileWriter { pub fn new( - store: Store, + store: DataStore, writer: BufWriter, filename: FileName, table: Arc, diff --git a/crates/core/dump/src/raw_dataset.rs b/crates/core/dump/src/raw_dataset.rs index 063d6fbbe..d0dd1a79b 100644 --- a/crates/core/dump/src/raw_dataset.rs +++ b/crates/core/dump/src/raw_dataset.rs @@ -89,8 +89,9 @@ use std::{ time::{Duration, Instant}, }; +use amp_data_store::DataStore; use common::{ - BlockNum, BlockStreamer, BoxError, LogicalCatalog, Store, + BlockNum, BlockStreamer, BoxError, LogicalCatalog, catalog::physical::{Catalog, PhysicalTable}, metadata::segments::merge_ranges, }; @@ -567,7 +568,7 @@ struct DumpPartition { /// The metadata database metadata_db: MetadataDb, /// The data store for object storage operations - data_store: Store, + data_store: DataStore, /// The tables to write to catalog: Catalog, /// The block ranges to scan diff --git a/crates/core/dump/src/raw_dataset_writer.rs b/crates/core/dump/src/raw_dataset_writer.rs index 59c7b9882..fdd45e388 100644 --- a/crates/core/dump/src/raw_dataset_writer.rs +++ b/crates/core/dump/src/raw_dataset_writer.rs @@ -1,10 +1,11 @@ use std::{collections::BTreeMap, ops::RangeInclusive, sync::Arc}; +use amp_data_store::{DataStore, file_name::FileName}; use common::{ - BlockNum, BoxError, RawTableRows, Store, + BlockNum, BoxError, RawTableRows, arrow::array::RecordBatch, catalog::physical::{Catalog, PhysicalTable}, - metadata::{FileName, Generation, segments::BlockRange}, + metadata::{Generation, segments::BlockRange}, }; use datasets_common::table_name::TableName; use metadata_db::MetadataDb; @@ -30,7 +31,7 @@ impl RawDatasetWriter { pub fn new( catalog: Catalog, metadata_db: MetadataDb, - store: Store, + store: DataStore, opts: Arc, missing_ranges_by_table: BTreeMap>>, compactors_by_table: BTreeMap>, @@ -114,7 +115,7 @@ impl RawDatasetWriter { struct RawTableWriter { table: Arc, - store: Store, + store: DataStore, opts: Arc, /// The ranges of block numbers that this writer is responsible for. @@ -132,7 +133,7 @@ struct RawTableWriter { impl RawTableWriter { pub fn new( table: Arc, - store: Store, + store: DataStore, compactor: Arc, opts: Arc, missing_ranges: Vec>, diff --git a/crates/core/dump/src/streaming_query.rs b/crates/core/dump/src/streaming_query.rs index 372502c0a..953e98d91 100644 --- a/crates/core/dump/src/streaming_query.rs +++ b/crates/core/dump/src/streaming_query.rs @@ -7,10 +7,11 @@ use std::{ }; use alloy::{hex::ToHexExt as _, primitives::BlockHash}; +use amp_data_store::DataStore; use amp_dataset_store::DatasetStore; use common::{ BlockNum, BoxError, Dataset, DetachedLogicalPlan, LogicalCatalog, PlanningContext, - QueryContext, SPECIAL_BLOCK_NUM, Store, + QueryContext, SPECIAL_BLOCK_NUM, arrow::{array::RecordBatch, datatypes::SchemaRef}, catalog::physical::{Catalog, PhysicalTable}, incrementalizer::incrementalize_plan, @@ -33,7 +34,7 @@ use tokio::{ }; use tokio_stream::wrappers::ReceiverStream; use tokio_util::task::AbortOnDropHandle; -use tracing::{Instrument, debug, instrument}; +use tracing::{Instrument, instrument}; /// Awaits any update for tables in a query context catalog. struct TableUpdates { @@ -143,7 +144,7 @@ pub struct StreamingQueryHandle { } impl StreamingQueryHandle { - pub fn as_stream(self) -> BoxStream<'static, Result> { + pub fn into_stream(self) -> BoxStream<'static, Result> { let data_stream = MessageStreamWithBlockComplete::new(ReceiverStream::new(self.rx).map(Ok)); let join = self.join_handle; @@ -178,7 +179,7 @@ impl StreamingQueryHandle { /// stream. pub struct StreamingQuery { query_env: QueryEnv, - data_store: Store, + data_store: DataStore, catalog: Catalog, plan: DetachedLogicalPlan, start_block: BlockNum, @@ -208,7 +209,7 @@ impl StreamingQuery { query_env: QueryEnv, catalog: Catalog, dataset_store: &DatasetStore, - data_store: Store, + data_store: DataStore, plan: DetachedLogicalPlan, start_block: BlockNum, end_block: Option, @@ -394,7 +395,7 @@ impl StreamingQuery { // The latest common watermark across the source tables. let Some(common_watermark) = self.latest_src_watermark(&blocks_ctx, chains).await? else { // No common watermark across source tables. - debug!("no common watermark found"); + tracing::debug!("no common watermark found"); return Ok(None); }; @@ -404,7 +405,7 @@ impl StreamingQuery { } let Some(direction) = self.next_microbatch_start(&blocks_ctx).await? else { - debug!("no next microbatch start found"); + tracing::debug!("no next microbatch start found"); return Ok(None); }; let start = direction.segment_start(); @@ -412,7 +413,7 @@ impl StreamingQuery { .next_microbatch_end(&blocks_ctx, start, common_watermark) .await? else { - debug!("no next microbatch end found"); + tracing::debug!("no next microbatch end found"); return Ok(None); }; Ok(Some(MicrobatchRange { @@ -624,7 +625,7 @@ impl StreamingQuery { let plan = ctx.plan_sql(query).await?; let results = ctx.execute_and_concat(plan).await?; if results.num_rows() == 0 { - debug!("blocks table missing block {} {:?}", number, hash); + tracing::debug!("blocks table missing block {} {:?}", number, hash); return Ok(None); } let get_hash_value = |column_name: &str| -> Option { @@ -666,7 +667,7 @@ pub fn keep_alive_stream<'a>( schema: SchemaRef, keep_alive_interval: u64, ) -> BoxStream<'a, Result> { - let period = tokio::time::Duration::from_secs(keep_alive_interval); + let period = Duration::from_secs(keep_alive_interval); let mut keep_alive_interval = tokio::time::interval(period); let missed_tick_behavior = MissedTickBehavior::Delay; @@ -703,7 +704,7 @@ pub fn keep_alive_stream<'a>( #[tracing::instrument(skip(dataset_store, data_store), err)] async fn resolve_blocks_table( dataset_store: &DatasetStore, - data_store: Store, + data_store: DataStore, root_datasets: BTreeMap>, network: &str, ) -> Result { diff --git a/crates/core/monitoring/src/logging.rs b/crates/core/monitoring/src/logging.rs index 370546083..339914986 100644 --- a/crates/core/monitoring/src/logging.rs +++ b/crates/core/monitoring/src/logging.rs @@ -67,6 +67,7 @@ const AMP_CRATES: &[&str] = &[ "admin_client", "amp_client", "amp_config", + "amp_data_store", "amp_dataset_store", "amp_object_store", "ampcc", diff --git a/crates/services/admin-api/Cargo.toml b/crates/services/admin-api/Cargo.toml index dacdc3207..847b954d2 100644 --- a/crates/services/admin-api/Cargo.toml +++ b/crates/services/admin-api/Cargo.toml @@ -5,6 +5,7 @@ license-file.workspace = true edition.workspace = true [dependencies] +amp-data-store = { path = "../../core/data-store" } async-trait.workspace = true axum.workspace = true common = { path = "../../core/common" } diff --git a/crates/services/admin-api/src/ctx.rs b/crates/services/admin-api/src/ctx.rs index a4c6528d5..6d1f876d5 100644 --- a/crates/services/admin-api/src/ctx.rs +++ b/crates/services/admin-api/src/ctx.rs @@ -2,8 +2,8 @@ use std::sync::Arc; use amp_config::BuildInfo; +use amp_data_store::DataStore; use amp_dataset_store::DatasetStore; -use common::store::Store; use metadata_db::MetadataDb; use crate::scheduler::Scheduler; @@ -15,7 +15,7 @@ pub struct Ctx { pub dataset_store: DatasetStore, pub scheduler: Arc, /// Object store for output data (used by dataset restore handler) - pub data_store: Store, + pub data_store: DataStore, /// Build information (version, git SHA, etc.) pub build_info: BuildInfo, } diff --git a/crates/services/controller/Cargo.toml b/crates/services/controller/Cargo.toml index 67b75bba5..17c8b1b62 100644 --- a/crates/services/controller/Cargo.toml +++ b/crates/services/controller/Cargo.toml @@ -6,6 +6,7 @@ license-file.workspace = true [dependencies] admin-api = { path = "../admin-api" } +amp-data-store = { path = "../../core/data-store" } async-trait.workspace = true axum.workspace = true common = { path = "../../core/common" } diff --git a/crates/services/controller/src/service.rs b/crates/services/controller/src/service.rs index ec60b4963..4d20846b1 100644 --- a/crates/services/controller/src/service.rs +++ b/crates/services/controller/src/service.rs @@ -1,6 +1,7 @@ use std::{future::Future, net::SocketAddr, sync::Arc, time::Duration}; use admin_api::ctx::Ctx; +use amp_data_store::DataStore; use amp_dataset_store::DatasetStore; use axum::{ Router, @@ -8,7 +9,7 @@ use axum::{ routing::get, serve::{Listener as _, ListenerExt as _}, }; -use common::{BoxError, store::Store}; +use common::BoxError; use metadata_db::MetadataDb; use monitoring::telemetry::metrics::Meter; use opentelemetry_instrumentation_tower::HTTPMetricsLayerBuilder; @@ -32,7 +33,7 @@ const RECONCILIATION_INTERVAL: Duration = Duration::from_secs(60); pub async fn new( config: Arc, metadata_db: MetadataDb, - data_store: Store, + data_store: DataStore, dataset_store: DatasetStore, meter: Option, at: SocketAddr, diff --git a/crates/services/server/Cargo.toml b/crates/services/server/Cargo.toml index d18eeedb3..02a4a42e2 100644 --- a/crates/services/server/Cargo.toml +++ b/crates/services/server/Cargo.toml @@ -5,6 +5,7 @@ version.workspace = true license-file.workspace = true [dependencies] +amp-data-store = { path = "../../core/data-store" } arrow-flight.workspace = true async-stream.workspace = true async-trait.workspace = true diff --git a/crates/services/server/src/config.rs b/crates/services/server/src/config.rs index ddb29bf94..bd79ccf0c 100644 --- a/crates/services/server/src/config.rs +++ b/crates/services/server/src/config.rs @@ -16,8 +16,6 @@ pub struct Config { pub query_max_mem_mb: usize, /// Paths for DataFusion temporary files for spill-to-disk pub spill_location: Vec, - /// Parquet metadata cache size (in MB) - pub parquet_cache_size_mb: u64, } impl Config { @@ -32,7 +30,6 @@ impl Config { self.max_mem_mb, self.query_max_mem_mb, &self.spill_location, - self.parquet_cache_size_mb, ) } } diff --git a/crates/services/server/src/flight.rs b/crates/services/server/src/flight.rs index 8fe54a660..1ad463857 100644 --- a/crates/services/server/src/flight.rs +++ b/crates/services/server/src/flight.rs @@ -6,6 +6,7 @@ use std::{collections::BTreeMap, pin::Pin, sync::Arc}; +use amp_data_store::DataStore; use amp_dataset_store::{DatasetStore, GetDatasetError}; use arrow_flight::{ ActionType, FlightData, FlightDescriptor, FlightEndpoint, FlightInfo, HandshakeRequest, @@ -35,7 +36,6 @@ use common::{ metadata::segments::{BlockRange, ResumeWatermark}, query_context::{Error as CoreError, QueryEnv}, sql_str::SqlStr, - store::Store, utils::error_with_causes, }; use datafusion::{ @@ -52,7 +52,7 @@ use prost::Message as _; use serde_json::json; use thiserror::Error; use tonic::{Request, Response, Status, service::Routes}; -use tracing::{debug, instrument}; +use tracing::instrument; use crate::{ config::Config, metrics::MetricsRegistry, @@ -65,7 +65,7 @@ type TonicStream = Pin> + Send + 'sta pub struct Service { config: Arc, env: QueryEnv, - data_store: Store, + data_store: DataStore, dataset_store: DatasetStore, notification_multiplexer: Arc, metrics: Option>, @@ -74,7 +74,7 @@ pub struct Service { impl Service { pub async fn create( config: Arc, - data_store: Store, + data_store: DataStore, metadata_db: MetadataDb, dataset_store: DatasetStore, meter: Option, @@ -236,7 +236,7 @@ impl Service { let stream = QueryResultStream::Incremental { stream: query - .as_stream() + .into_stream() .map_err(|err| Error::StreamingExecutionError(err.to_string())) .boxed(), schema, @@ -367,7 +367,7 @@ impl Service { .parse::() .map_err(|err| Error::InvalidQuery(err.to_string()))?; - debug!("SQL query: {}", sql_str); + tracing::debug!("SQL query: {}", sql_str); let stream = self .execute_query( diff --git a/crates/services/server/src/service.rs b/crates/services/server/src/service.rs index 38e601d75..4a4b0f217 100644 --- a/crates/services/server/src/service.rs +++ b/crates/services/server/src/service.rs @@ -5,6 +5,7 @@ use std::{future::Future, net::SocketAddr, sync::Arc}; +use amp_data_store::DataStore; use amp_dataset_store::DatasetStore; use axum::{ Router, @@ -12,7 +13,7 @@ use axum::{ routing::get, serve::{Listener as _, ListenerExt as _}, }; -use common::{BoxError, store::Store}; +use common::BoxError; use datafusion::error::DataFusionError; use futures::FutureExt; use metadata_db::MetadataDb; @@ -30,7 +31,7 @@ use crate::{config::Config, flight, jsonl}; pub async fn new( config: Arc, metadata_db: MetadataDb, - data_store: Store, + data_store: DataStore, dataset_store: DatasetStore, meter: Option, flight_at: impl Into>, diff --git a/crates/services/worker/Cargo.toml b/crates/services/worker/Cargo.toml index 4460b75b6..c106ab1ad 100644 --- a/crates/services/worker/Cargo.toml +++ b/crates/services/worker/Cargo.toml @@ -5,6 +5,7 @@ version.workspace = true license-file.workspace = true [dependencies] +amp-data-store = { path = "../../core/data-store" } backon.workspace = true chrono.workspace = true common = { path = "../../core/common" } diff --git a/crates/services/worker/src/service.rs b/crates/services/worker/src/service.rs index 953fced45..3459970c9 100644 --- a/crates/services/worker/src/service.rs +++ b/crates/services/worker/src/service.rs @@ -1,8 +1,8 @@ use std::{future::Future, sync::Arc, time::Duration}; +use amp_data_store::DataStore; use amp_dataset_store::DatasetStore; use backon::{ExponentialBuilder, Retryable}; -use common::{CachedParquetData, ParquetFooterCache, store::Store as DataStore}; use futures::TryStreamExt as _; use metadata_db::{ Error as MetadataDbError, MetadataDb, NotificationMultiplexerHandle, @@ -77,12 +77,6 @@ pub async fn new( // Create notification multiplexer let notification_multiplexer = Arc::new(notification_multiplexer::spawn(metadata_db.clone())); - // Create shared parquet footer cache - let parquet_opts = dump::parquet_opts(&config.parquet); - let parquet_footer_cache = ParquetFooterCache::builder(parquet_opts.cache_size_mb) - .with_weighter(|_k, v: &CachedParquetData| v.metadata.memory_size()) - .build(); - // Worker bootstrap: If the worker is restarted, it needs to be able to resume its state // from the last known state. // @@ -105,7 +99,6 @@ pub async fn new( data_store, notification_multiplexer, meter, - parquet_footer_cache, }, ); @@ -222,7 +215,6 @@ pub(crate) struct WorkerJobCtx { pub data_store: DataStore, pub notification_multiplexer: Arc, pub meter: Option, - pub parquet_footer_cache: ParquetFooterCache, } pub struct Worker { diff --git a/crates/services/worker/src/service/job_impl.rs b/crates/services/worker/src/service/job_impl.rs index f7a1669aa..0449b78be 100644 --- a/crates/services/worker/src/service/job_impl.rs +++ b/crates/services/worker/src/service/job_impl.rs @@ -87,13 +87,9 @@ pub(super) async fn new( } .into(); - let cached_store = common::CachedStore::from_parts( - ctx.data_store.clone(), - job_ctx.parquet_footer_cache.clone(), - ); let compactor = AmpCompactor::start( ctx.metadata_db.clone(), - cached_store, + ctx.data_store.clone(), opts.clone(), physical_table.clone(), metrics.clone(), diff --git a/tests/Cargo.toml b/tests/Cargo.toml index bbdea1978..1920822fa 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -13,6 +13,7 @@ name = "tests" [dependencies] alloy.workspace = true amp-client = { path = "../crates/clients/flight" } +amp-data-store = { path = "../crates/core/data-store" } amp-object-store = { path = "../crates/core/object-store" } ampctl = { path = "../crates/bin/ampctl" } arrow-flight.workspace = true diff --git a/tests/src/main.rs b/tests/src/main.rs index 771fb8201..315f39395 100644 --- a/tests/src/main.rs +++ b/tests/src/main.rs @@ -48,12 +48,13 @@ use std::{path::PathBuf, sync::Arc, time::Duration}; use amp_config::Config; +use amp_data_store::DataStore; use amp_dataset_store::{ DatasetStore, dataset_and_dependencies, manifests::DatasetManifestsStore, providers::ProviderConfigsStore, }; use clap::Parser; -use common::{BoxError, Store}; +use common::BoxError; use datasets_common::reference::Reference; use dump::consistency_check; use fs_err as fs; @@ -153,8 +154,12 @@ async fn main() { .await .expect("Failed to load config"), ); - let data_store = Store::new(sysdb.conn_pool().clone(), config.data_store_url.clone()) - .expect("Failed to create data store"); + let data_store = DataStore::new( + sysdb.conn_pool().clone(), + config.data_store_url.clone(), + config.parquet.cache_size_mb, + ) + .expect("Failed to create data store"); let dataset_store = { let provider_configs_store = ProviderConfigsStore::new( @@ -267,7 +272,7 @@ async fn main() { async fn bless( ampctl: &Ampctl, dataset_store: DatasetStore, - data_store: Store, + data_store: DataStore, dataset: Reference, end: u64, ) -> Result<(), BoxError> { @@ -321,7 +326,7 @@ async fn bless( .delete_stream(path_stream) .try_collect::>() .await - .map_err(|err| { + .map_err(|err: object_store::Error| { format!( "Failed to clear existing data for dataset '{}': {}", dataset, err diff --git a/tests/src/testlib/ctx.rs b/tests/src/testlib/ctx.rs index 8d6979ae8..a07faef50 100644 --- a/tests/src/testlib/ctx.rs +++ b/tests/src/testlib/ctx.rs @@ -30,7 +30,8 @@ use std::{collections::BTreeSet, path::Path, sync::Arc}; use amp_config::Config; -use common::{BoxError, store::Store}; +use amp_data_store::DataStore; +use common::BoxError; use datasets_common::reference::Reference; use worker::node_id::NodeId; @@ -342,9 +343,10 @@ impl TestCtxBuilder { let config = Arc::new(Config::load(daemon_state_dir.config_file(), false, None, true, None).await?); - let data_store = Store::new( + let data_store = DataStore::new( metadata_db.conn_pool().clone(), config.data_store_url.clone(), + config.parquet.cache_size_mb, )?; // Create shared DatasetStore instance (used by both server and worker) diff --git a/tests/src/testlib/fixtures/daemon_controller.rs b/tests/src/testlib/fixtures/daemon_controller.rs index 756179442..ffdd6f0e4 100644 --- a/tests/src/testlib/fixtures/daemon_controller.rs +++ b/tests/src/testlib/fixtures/daemon_controller.rs @@ -6,8 +6,11 @@ use std::{net::SocketAddr, sync::Arc}; -use common::{BoxError, BoxResult, store::Store}; +use amp_data_store::DataStore; +use amp_dataset_store::DatasetStore; +use common::{BoxError, BoxResult}; use controller::config::Config; +use metadata_db::MetadataDb; use opentelemetry::metrics::Meter; use tokio::task::JoinHandle; @@ -32,9 +35,9 @@ impl DaemonController { /// The controller will be automatically shut down when the fixture is dropped. pub async fn new( config: Arc, - metadata_db: metadata_db::MetadataDb, - data_store: Store, - dataset_store: amp_dataset_store::DatasetStore, + metadata_db: MetadataDb, + data_store: DataStore, + dataset_store: DatasetStore, meter: Option, ) -> Result { // Convert common config to controller config @@ -71,12 +74,12 @@ impl DaemonController { } /// Get a reference to the metadata database. - pub fn metadata_db(&self) -> &metadata_db::MetadataDb { + pub fn metadata_db(&self) -> &MetadataDb { &self.metadata_db } /// Get a reference to the dataset store. - pub fn dataset_store(&self) -> &_dataset_store::DatasetStore { + pub fn dataset_store(&self) -> &DatasetStore { &self.dataset_store } diff --git a/tests/src/testlib/fixtures/daemon_server.rs b/tests/src/testlib/fixtures/daemon_server.rs index a3fba97cb..f6a1cda8f 100644 --- a/tests/src/testlib/fixtures/daemon_server.rs +++ b/tests/src/testlib/fixtures/daemon_server.rs @@ -6,8 +6,9 @@ use std::{net::SocketAddr, sync::Arc}; +use amp_data_store::DataStore; use amp_dataset_store::DatasetStore; -use common::{BoxError, store::Store}; +use common::BoxError; use metadata_db::MetadataDb; use opentelemetry::metrics::Meter; use server::{config::Config, service::BoundAddrs}; @@ -21,7 +22,7 @@ use tokio::task::JoinHandle; pub struct DaemonServer { config: Arc, metadata_db: MetadataDb, - data_store: Store, + data_store: DataStore, dataset_store: DatasetStore, server_addrs: BoundAddrs, @@ -38,7 +39,7 @@ impl DaemonServer { pub async fn new( config: Arc, metadb: MetadataDb, - data_store: Store, + data_store: DataStore, dataset_store: DatasetStore, meter: Option, enable_flight: bool, @@ -93,7 +94,7 @@ impl DaemonServer { } /// Get a reference to the data store. - pub fn data_store(&self) -> &Store { + pub fn data_store(&self) -> &DataStore { &self.data_store } @@ -151,6 +152,5 @@ fn server_config_from_common(config: &_config::Config) -> Config { max_mem_mb: config.max_mem_mb, query_max_mem_mb: config.query_max_mem_mb, spill_location: config.spill_location.clone(), - parquet_cache_size_mb: config.parquet.cache_size_mb, } } diff --git a/tests/src/testlib/fixtures/daemon_worker.rs b/tests/src/testlib/fixtures/daemon_worker.rs index 339d89b81..6cc9c81c9 100644 --- a/tests/src/testlib/fixtures/daemon_worker.rs +++ b/tests/src/testlib/fixtures/daemon_worker.rs @@ -6,8 +6,9 @@ use std::sync::Arc; +use amp_data_store::DataStore; use amp_dataset_store::DatasetStore; -use common::{BoxError, store::Store}; +use common::BoxError; use metadata_db::MetadataDb; use opentelemetry::metrics::Meter; use tokio::task::JoinHandle; @@ -21,7 +22,7 @@ use worker::{config::Config, node_id::NodeId, service::RuntimeError as WorkerRun pub struct DaemonWorker { config: Config, metadata_db: MetadataDb, - data_store: Store, + data_store: DataStore, dataset_store: DatasetStore, node_id: NodeId, @@ -36,7 +37,7 @@ impl DaemonWorker { pub async fn new( config: Arc, metadata_db: MetadataDb, - data_store: Store, + data_store: DataStore, dataset_store: DatasetStore, meter: Option, node_id: NodeId, @@ -84,7 +85,7 @@ impl DaemonWorker { } /// Get a reference to the data store. - pub fn data_store(&self) -> &Store { + pub fn data_store(&self) -> &DataStore { &self.data_store } diff --git a/tests/src/testlib/fixtures/snapshot_ctx.rs b/tests/src/testlib/fixtures/snapshot_ctx.rs index 7f4581c89..87657d63a 100644 --- a/tests/src/testlib/fixtures/snapshot_ctx.rs +++ b/tests/src/testlib/fixtures/snapshot_ctx.rs @@ -11,8 +11,9 @@ use std::sync::Arc; +use amp_data_store::DataStore; use common::{ - BoxError, LogicalCatalog, QueryContext, Store, + BoxError, LogicalCatalog, QueryContext, catalog::physical::{Catalog, PhysicalTable}, }; use server::config::Config; @@ -33,7 +34,7 @@ impl SnapshotContext { /// from dataset snapshot reference data or fresh ETL extraction pipeline dumps. pub async fn from_tables( config: &Config, - store: Store, + store: DataStore, tables: Vec>, ) -> Result { let logical = LogicalCatalog::from_tables(tables.iter().map(|t| t.table())); diff --git a/tests/src/testlib/helpers.rs b/tests/src/testlib/helpers.rs index 43dc57c32..1be8b3628 100644 --- a/tests/src/testlib/helpers.rs +++ b/tests/src/testlib/helpers.rs @@ -9,9 +9,10 @@ pub mod git; use std::{collections::BTreeMap, sync::Arc, time::Duration}; +use amp_data_store::DataStore; use amp_dataset_store::DatasetStore; use common::{ - BoxError, LogicalCatalog, Store, + BoxError, LogicalCatalog, arrow::array::RecordBatch, catalog::physical::{Catalog, PhysicalTable}, metadata::segments::BlockRange, @@ -174,7 +175,7 @@ pub async fn deploy_and_wait( /// - Any physical table is not found in the data store pub async fn load_physical_tables( dataset_store: &DatasetStore, - data_store: &Store, + data_store: &DataStore, dataset_ref: &Reference, ) -> Result>, BoxError> { let hash_ref = dataset_store @@ -208,7 +209,7 @@ pub async fn load_physical_tables( /// or synchronization issues between metadata and storage. pub async fn check_table_consistency( table: &Arc, - store: &Store, + store: &DataStore, ) -> Result<(), BoxError> { consistency_check(table, store).await.map_err(Into::into) } @@ -227,7 +228,7 @@ pub async fn check_table_consistency( pub async fn restore_dataset_snapshot( ampctl: &super::fixtures::Ampctl, dataset_store: &DatasetStore, - data_store: &Store, + data_store: &DataStore, dataset_ref: &Reference, ) -> Result>, BoxError> { // 1. Restore via Admin API (indexes files into metadata DB) @@ -409,7 +410,7 @@ pub async fn assert_snapshots_eq(left: &SnapshotContext, right: &SnapshotContext pub async fn catalog_for_dataset( dataset_name: &str, dataset_store: &DatasetStore, - data_store: &Store, + data_store: &DataStore, ) -> Result { let dataset_ref: Reference = format!("_/{dataset_name}@latest") .parse() diff --git a/tests/src/tests/it_dump.rs b/tests/src/tests/it_dump.rs index a1a01512b..96aa0a624 100644 --- a/tests/src/tests/it_dump.rs +++ b/tests/src/tests/it_dump.rs @@ -1,6 +1,6 @@ use std::time::Duration; -use common::Store; +use amp_data_store::DataStore; use datasets_common::reference::Reference; use monitoring::logging; @@ -210,7 +210,7 @@ struct TestCtx { impl TestCtx { /// Get the data store from the daemon server. - fn data_store(&self) -> &Store { + fn data_store(&self) -> &DataStore { self.ctx.daemon_server().data_store() } diff --git a/tests/src/tests/it_sql_dataset_batch_size.rs b/tests/src/tests/it_sql_dataset_batch_size.rs index 0432d59aa..5a186a7ae 100644 --- a/tests/src/tests/it_sql_dataset_batch_size.rs +++ b/tests/src/tests/it_sql_dataset_batch_size.rs @@ -5,7 +5,7 @@ use std::{ }; use amp_dataset_store::DatasetStore; -use common::{BoxError, ParquetFooterCache, metadata::Generation}; +use common::{BoxError, metadata::Generation}; use datasets_common::reference::Reference; use dump::{ compaction::{AmpCompactor, SegmentSizeLimit}, @@ -87,7 +87,6 @@ async fn sql_dataset_input_batch_size() { /// and collection workflows with specific batch size configurations. struct TestCtx { ctx: testlib::ctx::TestCtx, - cache: ParquetFooterCache, } impl TestCtx { @@ -107,11 +106,6 @@ impl TestCtx { .await .expect("Failed to create test context"); - let cache = ParquetFooterCache::builder( - (ctx.daemon_worker().config().parquet.cache_size_mb * 1024 * 1024) as usize, - ) - .build(); - // Deploy the TypeScript dataset let sql_stream_ds = DatasetPackage::new("sql_stream_ds", Some("amp.config.ts")); let cli = ctx.new_amp_cli(); @@ -120,7 +114,7 @@ impl TestCtx { .await .expect("Failed to register sql_stream_ds dataset"); - Self { ctx, cache } + Self { ctx } } /// Get reference to the dataset store. @@ -174,12 +168,10 @@ impl TestCtx { opts_mut.compactor.interval = Duration::ZERO; opts_mut.compactor.algorithm.cooldown_duration = Duration::ZERO; opts_mut.partition = SegmentSizeLimit::new(100, 0, 0, 0, Generation::default(), 10); - let cache = self.cache.clone(); let metadata_db = self.ctx.daemon_worker().metadata_db().clone(); let data_store = self.ctx.daemon_server().data_store().clone(); - let cached_store = common::CachedStore::from_parts(data_store, cache); let mut task = - AmpCompactor::start(metadata_db, cached_store, opts.clone(), table.clone(), None); + AmpCompactor::start(metadata_db, data_store, opts.clone(), table.clone(), None); task.join_current_then_spawn_new().await.unwrap(); while !task.is_finished() { tokio::task::yield_now().await; @@ -205,12 +197,10 @@ impl TestCtx { opts_mut.collector.interval = Duration::ZERO; opts_mut.compactor.interval = Duration::ZERO; opts_mut.partition = SegmentSizeLimit::new(1, 1, 1, 0, Generation::default(), 1.5); - let cache = self.cache.clone(); let metadata_db = self.ctx.daemon_worker().metadata_db().clone(); let data_store = self.ctx.daemon_server().data_store().clone(); - let cached_store = common::CachedStore::from_parts(data_store, cache); let mut task = - AmpCompactor::start(metadata_db, cached_store, opts.clone(), table.clone(), None); + AmpCompactor::start(metadata_db, data_store, opts.clone(), table.clone(), None); task.join_current_then_spawn_new().await.unwrap(); while !task.is_finished() { tokio::task::yield_now().await;