From 8c1eaf9baf9156cff8fac198ce1e90ac3c142dc4 Mon Sep 17 00:00:00 2001 From: Shiyas Mohammed Date: Mon, 22 Dec 2025 12:33:28 +0530 Subject: [PATCH] refactor(dataset-store): rename dataset-store crate to amp-dataset-store --- Cargo.lock | 74 +++++++++---------- crates/bin/ampctl/Cargo.toml | 2 +- .../bin/ampctl/src/cmd/manifest/generate.rs | 12 +-- crates/bin/ampd/Cargo.toml | 2 +- crates/bin/ampd/src/controller_cmd.rs | 6 +- crates/bin/ampd/src/server_cmd.rs | 6 +- crates/bin/ampd/src/solo_cmd.rs | 6 +- crates/bin/ampd/src/worker_cmd.rs | 6 +- crates/core/dataset-store/Cargo.toml | 2 +- crates/core/dump/Cargo.toml | 2 +- crates/core/dump/src/derived_dataset.rs | 2 +- crates/core/dump/src/lib.rs | 2 +- crates/core/dump/src/raw_dataset.rs | 4 +- crates/core/dump/src/streaming_query.rs | 2 +- crates/core/monitoring/src/logging.rs | 2 +- crates/services/admin-api/Cargo.toml | 2 +- crates/services/admin-api/src/ctx.rs | 2 +- .../admin-api/src/handlers/datasets/delete.rs | 4 +- .../src/handlers/datasets/delete_version.rs | 6 +- .../admin-api/src/handlers/datasets/deploy.rs | 8 +- .../admin-api/src/handlers/datasets/get.rs | 14 ++-- .../src/handlers/datasets/get_manifest.rs | 14 ++-- .../src/handlers/datasets/list_all.rs | 2 +- .../src/handlers/datasets/list_jobs.rs | 2 +- .../src/handlers/datasets/list_versions.rs | 4 +- .../src/handlers/datasets/register.rs | 4 +- .../src/handlers/datasets/restore.rs | 4 +- .../src/handlers/manifests/delete_by_id.rs | 15 ++-- .../src/handlers/manifests/get_by_id.rs | 8 +- .../src/handlers/manifests/list_all.rs | 2 +- .../src/handlers/manifests/list_datasets.rs | 10 +-- .../admin-api/src/handlers/manifests/prune.rs | 4 +- .../src/handlers/manifests/register.rs | 8 +- .../src/handlers/providers/create.rs | 2 +- .../src/handlers/providers/delete_by_id.rs | 2 +- .../src/handlers/providers/provider_info.rs | 6 +- .../services/admin-api/src/handlers/schema.rs | 2 +- crates/services/admin-api/src/scheduler.rs | 2 +- crates/services/controller/Cargo.toml | 2 +- crates/services/controller/src/scheduler.rs | 2 +- crates/services/controller/src/service.rs | 2 +- crates/services/server/Cargo.toml | 2 +- crates/services/server/src/flight.rs | 2 +- crates/services/server/src/service.rs | 2 +- crates/services/worker/Cargo.toml | 2 +- crates/services/worker/src/job.rs | 2 +- crates/services/worker/src/service.rs | 2 +- .../services/worker/src/service/job_impl.rs | 2 +- tests/Cargo.toml | 2 +- tests/src/main.rs | 6 +- tests/src/testlib/ctx.rs | 2 +- .../src/testlib/fixtures/daemon_controller.rs | 6 +- tests/src/testlib/fixtures/daemon_server.rs | 2 +- tests/src/testlib/fixtures/daemon_worker.rs | 2 +- tests/src/testlib/helpers.rs | 2 +- tests/src/tests/it_sql_dataset_batch_size.rs | 2 +- 56 files changed, 154 insertions(+), 147 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4355f492f..3ecb9c6a7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -29,11 +29,11 @@ name = "admin-api" version = "0.1.0" dependencies = [ "amp-config", + "amp-dataset-store", "async-trait", "axum 0.8.7", "common", "datafusion", - "dataset-store", "datasets-common", "datasets-derived", "dump", @@ -939,6 +939,35 @@ dependencies = [ "tokio", ] +[[package]] +name = "amp-dataset-store" +version = "0.1.0" +dependencies = [ + "async-stream", + "common", + "datafusion", + "datasets-common", + "datasets-derived", + "eth-beacon-datasets", + "evm-rpc-datasets", + "firehose-datasets", + "futures", + "indoc", + "metadata-db", + "monitoring", + "object_store", + "parking_lot", + "rand 0.9.2", + "serde", + "serde_json", + "solana-datasets", + "thiserror 2.0.17", + "tokio", + "toml 0.8.23", + "tracing", + "url", +] + [[package]] name = "amp-object-store" version = "0.1.0" @@ -979,12 +1008,12 @@ name = "ampctl" version = "0.1.0" dependencies = [ "admin-client", + "amp-dataset-store", "amp-object-store", "anyhow", "clap", "common", "console 0.16.1", - "dataset-store", "datasets-common", "dump", "eth-beacon-datasets", @@ -1011,12 +1040,12 @@ name = "ampd" version = "0.1.0" dependencies = [ "amp-config", + "amp-dataset-store", "amp-object-store", "clap", "common", "console-subscriber", "controller", - "dataset-store", "metadata-db", "monitoring", "server", @@ -2847,10 +2876,10 @@ version = "0.1.0" dependencies = [ "admin-api", "amp-config", + "amp-dataset-store", "async-trait", "axum 0.8.7", "common", - "dataset-store", "datasets-common", "dump", "metadata-db", @@ -3981,35 +4010,6 @@ dependencies = [ "unicode-width 0.2.0", ] -[[package]] -name = "dataset-store" -version = "0.1.0" -dependencies = [ - "async-stream", - "common", - "datafusion", - "datasets-common", - "datasets-derived", - "eth-beacon-datasets", - "evm-rpc-datasets", - "firehose-datasets", - "futures", - "indoc", - "metadata-db", - "monitoring", - "object_store", - "parking_lot", - "rand 0.9.2", - "serde", - "serde_json", - "solana-datasets", - "thiserror 2.0.17", - "tokio", - "toml 0.8.23", - "tracing", - "url", -] - [[package]] name = "datasets-common" version = "0.1.0" @@ -4388,10 +4388,10 @@ name = "dump" version = "0.1.0" dependencies = [ "alloy", + "amp-dataset-store", "async-stream", "common", "datafusion", - "dataset-store", "datasets-common", "datasets-derived", "futures", @@ -9112,6 +9112,7 @@ dependencies = [ name = "server" version = "0.1.0" dependencies = [ + "amp-dataset-store", "arrow-flight", "async-stream", "async-trait", @@ -9120,7 +9121,6 @@ dependencies = [ "bytes", "common", "datafusion", - "dataset-store", "dump", "futures", "metadata-db", @@ -11884,6 +11884,7 @@ dependencies = [ "alloy", "amp-client", "amp-config", + "amp-dataset-store", "amp-object-store", "ampctl", "arrow-flight", @@ -11893,7 +11894,6 @@ dependencies = [ "common", "controller", "ctor", - "dataset-store", "datasets-common", "datasets-derived", "dotenvy", @@ -13593,10 +13593,10 @@ name = "worker" version = "0.1.0" dependencies = [ "amp-config", + "amp-dataset-store", "backon", "chrono", "common", - "dataset-store", "datasets-common", "dump", "futures", diff --git a/crates/bin/ampctl/Cargo.toml b/crates/bin/ampctl/Cargo.toml index 520e296d3..41a4b59c0 100644 --- a/crates/bin/ampctl/Cargo.toml +++ b/crates/bin/ampctl/Cargo.toml @@ -20,7 +20,7 @@ anyhow.workspace = true clap.workspace = true common = { path = "../../core/common" } console = "0.16.1" -dataset-store = { path = "../../core/dataset-store" } +amp-dataset-store = { path = "../../core/dataset-store" } datasets-common = { path = "../../core/datasets-common" } dump = { path = "../../core/dump" } eth-beacon-datasets = { path = "../../extractors/eth-beacon" } diff --git a/crates/bin/ampctl/src/cmd/manifest/generate.rs b/crates/bin/ampctl/src/cmd/manifest/generate.rs index 9d1a41b0b..cc671b1dc 100644 --- a/crates/bin/ampctl/src/cmd/manifest/generate.rs +++ b/crates/bin/ampctl/src/cmd/manifest/generate.rs @@ -24,7 +24,7 @@ use std::path::PathBuf; -use dataset_store::DatasetKind; +use amp_dataset_store::DatasetKind; use datasets_common::manifest::{ArrowSchema, Field, TableSchema}; use monitoring::logging; @@ -96,7 +96,7 @@ where let kind = kind.into(); let dataset_bytes = match kind { - dataset_store::DatasetKind::EvmRpc => { + DatasetKind::EvmRpc => { let tables = evm_rpc_datasets::tables::all(&network) .iter() .map(|table| { @@ -114,7 +114,7 @@ where }; serde_json::to_vec_pretty(&manifest).map_err(Error::Serialization)? } - dataset_store::DatasetKind::Solana => { + DatasetKind::Solana => { let tables = solana_datasets::tables::all(&network) .iter() .map(|table| { @@ -132,7 +132,7 @@ where }; serde_json::to_vec_pretty(&manifest).map_err(Error::Serialization)? } - dataset_store::DatasetKind::EthBeacon => { + DatasetKind::EthBeacon => { let tables = eth_beacon_datasets::all_tables(network.clone()) .iter() .map(|table| { @@ -150,7 +150,7 @@ where }; serde_json::to_vec_pretty(&manifest).map_err(Error::Serialization)? } - dataset_store::DatasetKind::Firehose => { + DatasetKind::Firehose => { let tables = firehose_datasets::evm::tables::all(&network) .iter() .map(|table| { @@ -169,7 +169,7 @@ where }; serde_json::to_vec_pretty(&manifest).map_err(Error::Serialization)? } - dataset_store::DatasetKind::Derived => { + DatasetKind::Derived => { return Err(Error::DerivedNotSupported); } }; diff --git a/crates/bin/ampd/Cargo.toml b/crates/bin/ampd/Cargo.toml index 181737a53..912b38529 100644 --- a/crates/bin/ampd/Cargo.toml +++ b/crates/bin/ampd/Cargo.toml @@ -20,7 +20,7 @@ common = { path = "../../core/common" } amp-config = { path = "../../config" } console-subscriber = { version = "0.4.1", default-features = false, optional = true } controller = { path = "../../services/controller" } -dataset-store = { path = "../../core/dataset-store" } +amp-dataset-store = { path = "../../core/dataset-store" } metadata-db = { path = "../../core/metadata-db" } monitoring = { path = "../../core/monitoring" } server = { path = "../../services/server" } diff --git a/crates/bin/ampd/src/controller_cmd.rs b/crates/bin/ampd/src/controller_cmd.rs index 7d131d1a5..3eab12ce4 100644 --- a/crates/bin/ampd/src/controller_cmd.rs +++ b/crates/bin/ampd/src/controller_cmd.rs @@ -1,12 +1,12 @@ use std::{net::SocketAddr, sync::Arc}; use amp_config::Config as CommonConfig; +use amp_dataset_store::{ + DatasetStore, manifests::DatasetManifestsStore, providers::ProviderConfigsStore, +}; use amp_object_store::ObjectStoreCreationError; use common::{BoxError, store::Store}; use controller::config::Config; -use dataset_store::{ - DatasetStore, manifests::DatasetManifestsStore, providers::ProviderConfigsStore, -}; use monitoring::telemetry::metrics::Meter; /// Run the controller service (Admin API server) diff --git a/crates/bin/ampd/src/server_cmd.rs b/crates/bin/ampd/src/server_cmd.rs index aecc19f1a..7c1257791 100644 --- a/crates/bin/ampd/src/server_cmd.rs +++ b/crates/bin/ampd/src/server_cmd.rs @@ -1,11 +1,11 @@ use std::sync::Arc; use amp_config::{Addrs, Config as CommonConfig}; -use amp_object_store::ObjectStoreCreationError; -use common::{BoxError, store::Store}; -use dataset_store::{ +use amp_dataset_store::{ DatasetStore, manifests::DatasetManifestsStore, providers::ProviderConfigsStore, }; +use amp_object_store::ObjectStoreCreationError; +use common::{BoxError, store::Store}; use monitoring::telemetry::metrics::Meter; use server::config::Config as ServerConfig; diff --git a/crates/bin/ampd/src/solo_cmd.rs b/crates/bin/ampd/src/solo_cmd.rs index dbf01c226..e9c3b9fd7 100644 --- a/crates/bin/ampd/src/solo_cmd.rs +++ b/crates/bin/ampd/src/solo_cmd.rs @@ -1,11 +1,11 @@ use std::{future::Future, pin::Pin, sync::Arc}; use amp_config::Config as CommonConfig; -use amp_object_store::ObjectStoreCreationError; -use common::{BoxError, store::Store}; -use dataset_store::{ +use amp_dataset_store::{ DatasetStore, manifests::DatasetManifestsStore, providers::ProviderConfigsStore, }; +use amp_object_store::ObjectStoreCreationError; +use common::{BoxError, store::Store}; use monitoring::telemetry::metrics::Meter; use crate::{controller_cmd, server_cmd, worker_cmd}; diff --git a/crates/bin/ampd/src/worker_cmd.rs b/crates/bin/ampd/src/worker_cmd.rs index e906fb21a..fa0214778 100644 --- a/crates/bin/ampd/src/worker_cmd.rs +++ b/crates/bin/ampd/src/worker_cmd.rs @@ -1,9 +1,9 @@ use amp_config::Config; -use amp_object_store::ObjectStoreCreationError; -use common::store::Store; -use dataset_store::{ +use amp_dataset_store::{ DatasetStore, manifests::DatasetManifestsStore, providers::ProviderConfigsStore, }; +use amp_object_store::ObjectStoreCreationError; +use common::store::Store; use monitoring::telemetry::metrics::Meter; use worker::node_id::NodeId; diff --git a/crates/core/dataset-store/Cargo.toml b/crates/core/dataset-store/Cargo.toml index dfb452bfd..e39fac446 100644 --- a/crates/core/dataset-store/Cargo.toml +++ b/crates/core/dataset-store/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "dataset-store" +name = "amp-dataset-store" edition.workspace = true version.workspace = true license-file.workspace = true diff --git a/crates/core/dump/Cargo.toml b/crates/core/dump/Cargo.toml index abd9e1068..b8d98d7ce 100644 --- a/crates/core/dump/Cargo.toml +++ b/crates/core/dump/Cargo.toml @@ -9,7 +9,7 @@ async-stream.workspace = true metadata-db = { path = "../metadata-db" } common = { path = "../common" } datafusion.workspace = true -dataset-store = { path = "../dataset-store" } +amp-dataset-store = { path = "../dataset-store" } datasets-derived = { path = "../datasets-derived" } datasets-common = { path = "../datasets-common" } futures.workspace = true diff --git a/crates/core/dump/src/derived_dataset.rs b/crates/core/dump/src/derived_dataset.rs index b6a6b7686..5e3c66986 100644 --- a/crates/core/dump/src/derived_dataset.rs +++ b/crates/core/dump/src/derived_dataset.rs @@ -243,7 +243,7 @@ pub enum Error { /// /// The manifest is required to determine table definitions and SQL queries. #[error("Failed to get derived manifest")] - GetDerivedManifest(#[source] dataset_store::GetDerivedManifestError), + GetDerivedManifest(#[source] amp_dataset_store::GetDerivedManifestError), /// Failed to dump individual table /// diff --git a/crates/core/dump/src/lib.rs b/crates/core/dump/src/lib.rs index 3f7eb4a3c..d505ae078 100644 --- a/crates/core/dump/src/lib.rs +++ b/crates/core/dump/src/lib.rs @@ -2,12 +2,12 @@ use std::sync::Arc; +use amp_dataset_store::{DatasetKind, DatasetStore}; use common::{ catalog::physical::PhysicalTable, parquet::file::properties::WriterProperties as ParquetWriterProperties, store::Store as DataStore, }; -use dataset_store::{DatasetKind, DatasetStore}; use datasets_common::hash_reference::HashReference; use metadata_db::{MetadataDb, NotificationMultiplexerHandle}; diff --git a/crates/core/dump/src/raw_dataset.rs b/crates/core/dump/src/raw_dataset.rs index 685784dd7..9a96a2af2 100644 --- a/crates/core/dump/src/raw_dataset.rs +++ b/crates/core/dump/src/raw_dataset.rs @@ -289,7 +289,7 @@ pub enum Error { /// - Manifest parsing errors /// - Missing required manifest fields #[error("Failed to get dataset")] - GetDataset(#[source] dataset_store::GetDatasetError), + GetDataset(#[source] amp_dataset_store::GetDatasetError), /// Failed consistency check for table /// @@ -316,7 +316,7 @@ pub enum Error { /// - Manifest not registered in metadata database /// - Network connectivity issues to provider #[error("Failed to get blockchain client for dataset")] - GetClient(#[source] dataset_store::GetClientError), + GetClient(#[source] amp_dataset_store::GetClientError), /// Failed to resolve end block number /// diff --git a/crates/core/dump/src/streaming_query.rs b/crates/core/dump/src/streaming_query.rs index 5ed50aeb2..372502c0a 100644 --- a/crates/core/dump/src/streaming_query.rs +++ b/crates/core/dump/src/streaming_query.rs @@ -7,6 +7,7 @@ use std::{ }; use alloy::{hex::ToHexExt as _, primitives::BlockHash}; +use amp_dataset_store::DatasetStore; use common::{ BlockNum, BoxError, Dataset, DetachedLogicalPlan, LogicalCatalog, PlanningContext, QueryContext, SPECIAL_BLOCK_NUM, Store, @@ -19,7 +20,6 @@ use common::{ sql_str::SqlStr, }; use datafusion::{common::cast::as_fixed_size_binary_array, error::DataFusionError}; -use dataset_store::DatasetStore; use datasets_common::{ hash::Hash, name::Name, partial_reference::PartialReference, revision::Revision, }; diff --git a/crates/core/monitoring/src/logging.rs b/crates/core/monitoring/src/logging.rs index a675d3c19..370546083 100644 --- a/crates/core/monitoring/src/logging.rs +++ b/crates/core/monitoring/src/logging.rs @@ -67,6 +67,7 @@ const AMP_CRATES: &[&str] = &[ "admin_client", "amp_client", "amp_config", + "amp_dataset_store", "amp_object_store", "ampcc", "ampctl", @@ -77,7 +78,6 @@ const AMP_CRATES: &[&str] = &[ "auth_http", "common", "controller", - "dataset_store", "datasets_common", "datasets_derived", "datasets_raw", diff --git a/crates/services/admin-api/Cargo.toml b/crates/services/admin-api/Cargo.toml index 925aacfb5..dacdc3207 100644 --- a/crates/services/admin-api/Cargo.toml +++ b/crates/services/admin-api/Cargo.toml @@ -10,7 +10,7 @@ axum.workspace = true common = { path = "../../core/common" } amp-config = { path = "../../config" } datafusion.workspace = true -dataset-store = { path = "../../core/dataset-store" } +amp-dataset-store = { path = "../../core/dataset-store" } datasets-common = { path = "../../core/datasets-common" } datasets-derived = { path = "../../core/datasets-derived" } dump = { path = "../../core/dump" } diff --git a/crates/services/admin-api/src/ctx.rs b/crates/services/admin-api/src/ctx.rs index a4373d0db..a4c6528d5 100644 --- a/crates/services/admin-api/src/ctx.rs +++ b/crates/services/admin-api/src/ctx.rs @@ -2,8 +2,8 @@ use std::sync::Arc; use amp_config::BuildInfo; +use amp_dataset_store::DatasetStore; use common::store::Store; -use dataset_store::DatasetStore; use metadata_db::MetadataDb; use crate::scheduler::Scheduler; diff --git a/crates/services/admin-api/src/handlers/datasets/delete.rs b/crates/services/admin-api/src/handlers/datasets/delete.rs index 4407550a2..c3c0ddca8 100644 --- a/crates/services/admin-api/src/handlers/datasets/delete.rs +++ b/crates/services/admin-api/src/handlers/datasets/delete.rs @@ -1,8 +1,8 @@ +use amp_dataset_store::DeleteManifestError; use axum::{ extract::{Path, State, rejection::PathRejection}, http::StatusCode, }; -use dataset_store::DeleteManifestError; use datasets_common::{name::Name, namespace::Namespace}; use futures::stream::{FuturesUnordered, StreamExt}; use monitoring::logging; @@ -178,7 +178,7 @@ pub enum Error { /// - Failed to delete dataset manifest links from database /// - Database connection or transaction issues #[error("Failed to unlink dataset manifests: {0}")] - UnlinkDatasetManifests(#[source] dataset_store::UnlinkDatasetManifestsError), + UnlinkDatasetManifests(#[source] amp_dataset_store::UnlinkDatasetManifestsError), } impl IntoErrorResponse for Error { diff --git a/crates/services/admin-api/src/handlers/datasets/delete_version.rs b/crates/services/admin-api/src/handlers/datasets/delete_version.rs index 8fc718699..73bc488c4 100644 --- a/crates/services/admin-api/src/handlers/datasets/delete_version.rs +++ b/crates/services/admin-api/src/handlers/datasets/delete_version.rs @@ -137,7 +137,7 @@ pub enum Error { /// - Database connection issues /// - Internal database errors during resolution #[error("Failed to resolve latest revision: {0}")] - ResolveLatestRevision(#[source] dataset_store::ResolveRevisionError), + ResolveLatestRevision(#[source] amp_dataset_store::ResolveRevisionError), /// Failed to resolve the version revision /// /// This occurs when: @@ -145,7 +145,7 @@ pub enum Error { /// - Database connection issues /// - Internal database errors during resolution #[error("Failed to resolve version revision: {0}")] - ResolveVersionRevision(#[source] dataset_store::ResolveRevisionError), + ResolveVersionRevision(#[source] amp_dataset_store::ResolveRevisionError), /// Cannot delete the version currently tagged as "latest" /// /// This occurs when: @@ -160,7 +160,7 @@ pub enum Error { /// - Database connection issues /// - Internal database errors during deletion #[error("Failed to delete version tag: {0}")] - DeleteVersionTag(#[source] dataset_store::DeleteVersionTagError), + DeleteVersionTag(#[source] amp_dataset_store::DeleteVersionTagError), } impl IntoErrorResponse for Error { diff --git a/crates/services/admin-api/src/handlers/datasets/deploy.rs b/crates/services/admin-api/src/handlers/datasets/deploy.rs index ed91e25ea..12c1cdb7c 100644 --- a/crates/services/admin-api/src/handlers/datasets/deploy.rs +++ b/crates/services/admin-api/src/handlers/datasets/deploy.rs @@ -1,5 +1,6 @@ use std::fmt::Debug; +use amp_dataset_store::DatasetKind; use axum::{ Json, extract::{ @@ -8,7 +9,6 @@ use axum::{ }, http::StatusCode, }; -use dataset_store::DatasetKind; use datasets_common::{name::Name, namespace::Namespace, reference::Reference, revision::Revision}; use monitoring::logging; use worker::{job::JobId, node_id::NodeId}; @@ -279,7 +279,7 @@ pub enum Error { /// - Database connection issues /// - Internal database errors #[error("Failed to list version tags: {0}")] - ListVersionTags(#[source] dataset_store::ListVersionTagsError), + ListVersionTags(#[source] amp_dataset_store::ListVersionTagsError), /// Dataset store operation error when resolving revision /// /// This occurs when: @@ -287,7 +287,7 @@ pub enum Error { /// - Database connection issues /// - Internal database errors #[error("Failed to resolve revision: {0}")] - ResolveRevision(#[source] dataset_store::ResolveRevisionError), + ResolveRevision(#[source] amp_dataset_store::ResolveRevisionError), /// Dataset store operation error when loading dataset /// /// This occurs when: @@ -295,7 +295,7 @@ pub enum Error { /// - Manifest parsing errors /// - Invalid dataset structure #[error("Failed to load dataset: {0}")] - GetDataset(#[source] dataset_store::GetDatasetError), + GetDataset(#[source] amp_dataset_store::GetDatasetError), /// Scheduler error /// /// This occurs when: diff --git a/crates/services/admin-api/src/handlers/datasets/get.rs b/crates/services/admin-api/src/handlers/datasets/get.rs index 73bccea25..0d8da8796 100644 --- a/crates/services/admin-api/src/handlers/datasets/get.rs +++ b/crates/services/admin-api/src/handlers/datasets/get.rs @@ -96,8 +96,10 @@ pub async fn handler( .get_manifest(reference.hash()) .await .map_err(|err| match err { - dataset_store::GetManifestError::MetadataDbQueryPath(_) => Error::GetManifestPath(err), - dataset_store::GetManifestError::ObjectStoreError(_) => Error::ReadManifest(err), + amp_dataset_store::GetManifestError::MetadataDbQueryPath(_) => { + Error::GetManifestPath(err) + } + amp_dataset_store::GetManifestError::ObjectStoreError(_) => Error::ReadManifest(err), })? .ok_or_else(|| Error::NotFound { namespace: namespace.clone(), @@ -168,7 +170,7 @@ pub enum Error { /// - Database connection issues /// - Internal database errors during revision resolution #[error("Failed to resolve revision: {0}")] - ResolveRevision(#[source] dataset_store::ResolveRevisionError), + ResolveRevision(#[source] amp_dataset_store::ResolveRevisionError), /// Failed to query manifest path from metadata database /// /// This occurs when: @@ -176,7 +178,7 @@ pub enum Error { /// - Database connection issues /// - Internal database errors #[error("Failed to query manifest path: {0}")] - GetManifestPath(#[source] dataset_store::GetManifestError), + GetManifestPath(#[source] amp_dataset_store::GetManifestError), /// Failed to read manifest from object store /// /// This occurs when: @@ -185,7 +187,7 @@ pub enum Error { /// - Permissions issues accessing object store /// - Network errors #[error("Failed to read manifest from object store: {0}")] - ReadManifest(#[source] dataset_store::GetManifestError), + ReadManifest(#[source] amp_dataset_store::GetManifestError), /// Failed to parse manifest JSON /// /// This occurs when: @@ -193,7 +195,7 @@ pub enum Error { /// - Manifest structure doesn't match expected schema /// - Required fields are missing #[error("Failed to parse manifest: {0}")] - ParseManifest(#[source] dataset_store::ManifestParseError), + ParseManifest(#[source] amp_dataset_store::ManifestParseError), } impl IntoErrorResponse for Error { diff --git a/crates/services/admin-api/src/handlers/datasets/get_manifest.rs b/crates/services/admin-api/src/handlers/datasets/get_manifest.rs index b0193188d..b51f73f99 100644 --- a/crates/services/admin-api/src/handlers/datasets/get_manifest.rs +++ b/crates/services/admin-api/src/handlers/datasets/get_manifest.rs @@ -97,8 +97,10 @@ pub async fn handler( .get_manifest(reference.hash()) .await .map_err(|err| match err { - dataset_store::GetManifestError::MetadataDbQueryPath(_) => Error::GetManifestPath(err), - dataset_store::GetManifestError::ObjectStoreError(_) => Error::ReadManifest(err), + amp_dataset_store::GetManifestError::MetadataDbQueryPath(_) => { + Error::GetManifestPath(err) + } + amp_dataset_store::GetManifestError::ObjectStoreError(_) => Error::ReadManifest(err), })? .ok_or_else(|| Error::ManifestNotFound { hash: reference.hash().to_string(), @@ -149,7 +151,7 @@ pub enum Error { /// - Database connection issues /// - Internal database errors during revision resolution #[error("Failed to resolve revision: {0}")] - ResolveRevision(#[source] dataset_store::ResolveRevisionError), + ResolveRevision(#[source] amp_dataset_store::ResolveRevisionError), /// Failed to query manifest path from metadata database /// /// This occurs when: @@ -157,7 +159,7 @@ pub enum Error { /// - Database connection issues /// - Internal database errors #[error("Failed to query manifest path: {0}")] - GetManifestPath(#[source] dataset_store::GetManifestError), + GetManifestPath(#[source] amp_dataset_store::GetManifestError), /// Failed to read manifest from object store /// /// This occurs when: @@ -166,7 +168,7 @@ pub enum Error { /// - Permissions issues accessing object store /// - Network errors #[error("Failed to read manifest from object store: {0}")] - ReadManifest(#[source] dataset_store::GetManifestError), + ReadManifest(#[source] amp_dataset_store::GetManifestError), /// Failed to parse manifest JSON /// /// This occurs when: @@ -174,7 +176,7 @@ pub enum Error { /// - Manifest structure doesn't match expected schema /// - Required fields are missing #[error("Failed to parse manifest: {0}")] - ParseManifest(#[source] dataset_store::ManifestParseError), + ParseManifest(#[source] amp_dataset_store::ManifestParseError), } impl IntoErrorResponse for Error { diff --git a/crates/services/admin-api/src/handlers/datasets/list_all.rs b/crates/services/admin-api/src/handlers/datasets/list_all.rs index 2e2f9055d..26f3a3019 100644 --- a/crates/services/admin-api/src/handlers/datasets/list_all.rs +++ b/crates/services/admin-api/src/handlers/datasets/list_all.rs @@ -136,7 +136,7 @@ pub enum Error { /// - Database connection issues /// - Internal database errors #[error("Failed to list all datasets: {0}")] - ListAllDatasets(#[source] dataset_store::ListAllDatasetsError), + ListAllDatasets(#[source] amp_dataset_store::ListAllDatasetsError), } impl IntoErrorResponse for Error { diff --git a/crates/services/admin-api/src/handlers/datasets/list_jobs.rs b/crates/services/admin-api/src/handlers/datasets/list_jobs.rs index 6df622ee1..0db89700c 100644 --- a/crates/services/admin-api/src/handlers/datasets/list_jobs.rs +++ b/crates/services/admin-api/src/handlers/datasets/list_jobs.rs @@ -152,7 +152,7 @@ pub enum Error { /// - Query execution encounters an internal database error /// - Metadata database query for revision fails #[error("failed to resolve dataset revision: {0}")] - ResolveRevision(#[source] dataset_store::ResolveRevisionError), + ResolveRevision(#[source] amp_dataset_store::ResolveRevisionError), /// Dataset revision does not exist /// diff --git a/crates/services/admin-api/src/handlers/datasets/list_versions.rs b/crates/services/admin-api/src/handlers/datasets/list_versions.rs index e7551e2a7..0d3dbc04f 100644 --- a/crates/services/admin-api/src/handlers/datasets/list_versions.rs +++ b/crates/services/admin-api/src/handlers/datasets/list_versions.rs @@ -184,7 +184,7 @@ pub enum Error { /// - Database connection issues /// - Internal database errors #[error("Failed to list version tags: {0}")] - ListVersionTags(#[source] dataset_store::ListVersionTagsError), + ListVersionTags(#[source] amp_dataset_store::ListVersionTagsError), /// Dataset store operation error when resolving revision /// @@ -193,7 +193,7 @@ pub enum Error { /// - Database connection issues /// - Internal database errors #[error("Failed to resolve revision: {0}")] - ResolveRevision(#[source] dataset_store::ResolveRevisionError), + ResolveRevision(#[source] amp_dataset_store::ResolveRevisionError), } impl IntoErrorResponse for Error { diff --git a/crates/services/admin-api/src/handlers/datasets/register.rs b/crates/services/admin-api/src/handlers/datasets/register.rs index 5a7fbafe1..e7dc5e274 100644 --- a/crates/services/admin-api/src/handlers/datasets/register.rs +++ b/crates/services/admin-api/src/handlers/datasets/register.rs @@ -1,10 +1,12 @@ +use amp_dataset_store::{ + DatasetKind, LinkManifestError, RegisterManifestError, SetVersionTagError, +}; use axum::{ Json, extract::{State, rejection::JsonRejection}, http::StatusCode, }; use common::BoxError; -use dataset_store::{DatasetKind, LinkManifestError, RegisterManifestError, SetVersionTagError}; use datasets_common::{ hash::{Hash, hash}, manifest::Manifest as CommonManifest, diff --git a/crates/services/admin-api/src/handlers/datasets/restore.rs b/crates/services/admin-api/src/handlers/datasets/restore.rs index 9b003e52f..864f60e42 100644 --- a/crates/services/admin-api/src/handlers/datasets/restore.rs +++ b/crates/services/admin-api/src/handlers/datasets/restore.rs @@ -238,7 +238,7 @@ pub enum Error { /// - Database connection issues /// - Internal database errors #[error("Failed to resolve revision: {0}")] - ResolveRevision(#[source] dataset_store::ResolveRevisionError), + ResolveRevision(#[source] amp_dataset_store::ResolveRevisionError), /// Dataset store operation error when loading dataset /// @@ -247,7 +247,7 @@ pub enum Error { /// - Manifest parsing errors /// - Invalid dataset structure #[error("Failed to load dataset: {0}")] - GetDataset(#[source] dataset_store::GetDatasetError), + GetDataset(#[source] amp_dataset_store::GetDatasetError), /// Failed to restore table from storage /// diff --git a/crates/services/admin-api/src/handlers/manifests/delete_by_id.rs b/crates/services/admin-api/src/handlers/manifests/delete_by_id.rs index fdfb979b4..72a621a73 100644 --- a/crates/services/admin-api/src/handlers/manifests/delete_by_id.rs +++ b/crates/services/admin-api/src/handlers/manifests/delete_by_id.rs @@ -1,5 +1,6 @@ //! Manifests delete by ID handler +use amp_dataset_store::DeleteManifestError; use axum::{ extract::{Path, State, rejection::PathRejection}, http::StatusCode, @@ -87,14 +88,14 @@ pub async fn handler( "successfully deleted manifest" ); } - Err(dataset_store::DeleteManifestError::ManifestLinked) => { + Err(DeleteManifestError::ManifestLinked) => { tracing::debug!( manifest_hash = %hash, "manifest is linked to datasets, cannot delete" ); return Err(Error::ManifestLinked { hash }.into()); } - Err(dataset_store::DeleteManifestError::TransactionBegin(err)) => { + Err(DeleteManifestError::TransactionBegin(err)) => { tracing::error!( manifest_hash = %hash, error = %err, error_source = logging::error_source(&err), @@ -102,7 +103,7 @@ pub async fn handler( ); return Err(Error::TransactionBeginError(err).into()); } - Err(dataset_store::DeleteManifestError::MetadataDbCheckLinks(err)) => { + Err(DeleteManifestError::MetadataDbCheckLinks(err)) => { tracing::error!( manifest_hash = %hash, error = %err, error_source = logging::error_source(&err), @@ -110,7 +111,7 @@ pub async fn handler( ); return Err(Error::CheckLinksError(err).into()); } - Err(dataset_store::DeleteManifestError::MetadataDbDelete(err)) => { + Err(DeleteManifestError::MetadataDbDelete(err)) => { tracing::error!( manifest_hash = %hash, error = %err, error_source = logging::error_source(&err), @@ -118,7 +119,7 @@ pub async fn handler( ); return Err(Error::MetadataDbDeleteError(err).into()); } - Err(dataset_store::DeleteManifestError::ObjectStoreError(err)) => { + Err(DeleteManifestError::ObjectStoreError(err)) => { tracing::error!( manifest_hash = %hash, error = %err, error_source = logging::error_source(&err), @@ -126,7 +127,7 @@ pub async fn handler( ); return Err(Error::ObjectStoreDeleteError(err).into()); } - Err(dataset_store::DeleteManifestError::TransactionCommit(err)) => { + Err(DeleteManifestError::TransactionCommit(err)) => { tracing::error!( manifest_hash = %hash, error = %err, error_source = logging::error_source(&err), @@ -198,7 +199,7 @@ pub enum Error { /// - Network errors prevent deletion from remote storage /// - File does not exist in object store (non-critical if DB delete succeeded) #[error("failed to delete manifest from object store: {0}")] - ObjectStoreDeleteError(#[source] dataset_store::manifests::DeleteError), + ObjectStoreDeleteError(#[source] amp_dataset_store::manifests::DeleteError), /// Failed to commit transaction /// diff --git a/crates/services/admin-api/src/handlers/manifests/get_by_id.rs b/crates/services/admin-api/src/handlers/manifests/get_by_id.rs index 20d28390e..1a38c2258 100644 --- a/crates/services/admin-api/src/handlers/manifests/get_by_id.rs +++ b/crates/services/admin-api/src/handlers/manifests/get_by_id.rs @@ -1,11 +1,11 @@ //! Manifests get by ID handler +use amp_dataset_store::{GetManifestError, manifests::ManifestContent}; use axum::{ Json, extract::{Path, State, rejection::PathRejection}, http::StatusCode, }; -use dataset_store::manifests::ManifestContent; use datasets_common::hash::Hash; use monitoring::logging; use serde_json::value::RawValue as JsonRawValue; @@ -88,7 +88,7 @@ pub async fn handler( ); return Err(Error::NotFound { hash }.into()); } - Err(dataset_store::GetManifestError::MetadataDbQueryPath(err)) => { + Err(GetManifestError::MetadataDbQueryPath(err)) => { tracing::error!( manifest_hash=%hash, error = %err, error_source = logging::error_source(&err), @@ -96,7 +96,7 @@ pub async fn handler( ); return Err(Error::MetadataDbQueryError(err).into()); } - Err(dataset_store::GetManifestError::ObjectStoreError(err)) => { + Err(GetManifestError::ObjectStoreError(err)) => { tracing::error!( manifest_hash=%hash, error = %err, error_source = logging::error_source(&err), @@ -173,7 +173,7 @@ pub enum Error { /// - Network errors prevent reading from remote storage /// - File corruption or invalid data in the object store #[error("failed to read manifest from object store: {0}")] - ObjectStoreReadError(#[source] dataset_store::manifests::GetError), + ObjectStoreReadError(#[source] amp_dataset_store::manifests::GetError), } impl IntoErrorResponse for Error { diff --git a/crates/services/admin-api/src/handlers/manifests/list_all.rs b/crates/services/admin-api/src/handlers/manifests/list_all.rs index 6bfd322d6..c6b1ae60c 100644 --- a/crates/services/admin-api/src/handlers/manifests/list_all.rs +++ b/crates/services/admin-api/src/handlers/manifests/list_all.rs @@ -87,7 +87,7 @@ pub enum Error { /// - Database connection issues /// - Internal database errors #[error("Failed to list all manifests: {0}")] - ListAllManifests(#[source] dataset_store::ListAllManifestsError), + ListAllManifests(#[source] amp_dataset_store::ListAllManifestsError), } impl IntoErrorResponse for Error { diff --git a/crates/services/admin-api/src/handlers/manifests/list_datasets.rs b/crates/services/admin-api/src/handlers/manifests/list_datasets.rs index e4948dffc..07ca33530 100644 --- a/crates/services/admin-api/src/handlers/manifests/list_datasets.rs +++ b/crates/services/admin-api/src/handlers/manifests/list_datasets.rs @@ -110,8 +110,8 @@ pub struct Dataset { pub version: Version, } -impl From for Dataset { - fn from(tag: dataset_store::DatasetTag) -> Self { +impl From for Dataset { + fn from(tag: amp_dataset_store::DatasetTag) -> Self { Self { namespace: tag.namespace, name: tag.name, @@ -151,7 +151,7 @@ pub enum Error { /// - SQL query to retrieve dataset tags fails /// - Database schema inconsistencies prevent tag retrieval #[error("failed to list dataset tags: {0}")] - ListDatasetTags(#[source] dataset_store::ListDatasetsUsingManifestError), + ListDatasetTags(#[source] amp_dataset_store::ListDatasetsUsingManifestError), } impl IntoErrorResponse for Error { @@ -160,10 +160,10 @@ impl IntoErrorResponse for Error { Error::InvalidHash(_) => "INVALID_HASH", Error::ManifestNotFound { .. } => "MANIFEST_NOT_FOUND", Error::ListDatasetTags(err) => match err { - dataset_store::ListDatasetsUsingManifestError::MetadataDbQueryPath(_) => { + amp_dataset_store::ListDatasetsUsingManifestError::MetadataDbQueryPath(_) => { "QUERY_MANIFEST_PATH_ERROR" } - dataset_store::ListDatasetsUsingManifestError::MetadataDbListTags(_) => { + amp_dataset_store::ListDatasetsUsingManifestError::MetadataDbListTags(_) => { "LIST_DATASET_TAGS_ERROR" } }, diff --git a/crates/services/admin-api/src/handlers/manifests/prune.rs b/crates/services/admin-api/src/handlers/manifests/prune.rs index c5acbe036..149fbc04a 100644 --- a/crates/services/admin-api/src/handlers/manifests/prune.rs +++ b/crates/services/admin-api/src/handlers/manifests/prune.rs @@ -1,7 +1,7 @@ //! Manifests prune handler +use amp_dataset_store::DeleteManifestError; use axum::{Json, extract::State, http::StatusCode}; -use dataset_store::DeleteManifestError; use futures::stream::{FuturesUnordered, StreamExt}; use monitoring::logging; @@ -163,7 +163,7 @@ pub struct PruneResponse { /// - Database errors during query #[derive(Debug, thiserror::Error)] #[error("failed to list orphaned manifests")] -pub struct Error(#[source] pub dataset_store::ListOrphanedManifestsError); +pub struct Error(#[source] pub amp_dataset_store::ListOrphanedManifestsError); impl IntoErrorResponse for Error { fn error_code(&self) -> &'static str { diff --git a/crates/services/admin-api/src/handlers/manifests/register.rs b/crates/services/admin-api/src/handlers/manifests/register.rs index 64efbb567..53c8b6021 100644 --- a/crates/services/admin-api/src/handlers/manifests/register.rs +++ b/crates/services/admin-api/src/handlers/manifests/register.rs @@ -1,11 +1,11 @@ //! Manifests register handler +use amp_dataset_store::DatasetKind; use axum::{ Json, extract::{State, rejection::JsonRejection}, http::StatusCode, }; -use dataset_store::DatasetKind; use datasets_common::{ hash::{Hash, hash}, manifest::Manifest as CommonManifest, @@ -150,10 +150,10 @@ pub async fn handler( "failed to register manifest" ); return Err(match err { - dataset_store::RegisterManifestError::ManifestStorage(e) => { + amp_dataset_store::RegisterManifestError::ManifestStorage(e) => { Error::ObjectStoreWriteError(e) } - dataset_store::RegisterManifestError::MetadataRegistration(e) => { + amp_dataset_store::RegisterManifestError::MetadataRegistration(e) => { Error::MetadataDbError(e) } } @@ -227,7 +227,7 @@ pub enum Error { /// - Storage quota is exceeded /// - Network errors prevent writing to remote storage #[error("failed to write manifest to object store: {0}")] - ObjectStoreWriteError(#[source] dataset_store::StoreError), + ObjectStoreWriteError(#[source] amp_dataset_store::StoreError), /// Failed to register manifest in metadata database /// diff --git a/crates/services/admin-api/src/handlers/providers/create.rs b/crates/services/admin-api/src/handlers/providers/create.rs index a3ed2e584..a75da8a2c 100644 --- a/crates/services/admin-api/src/handlers/providers/create.rs +++ b/crates/services/admin-api/src/handlers/providers/create.rs @@ -1,11 +1,11 @@ //! Provider create handler +use amp_dataset_store::providers::{ProviderConfig, RegisterError}; use axum::{ Json, extract::{State, rejection::JsonRejection}, http::StatusCode, }; -use dataset_store::providers::{ProviderConfig, RegisterError}; use monitoring::logging; use super::{convert, provider_info::ProviderInfo}; diff --git a/crates/services/admin-api/src/handlers/providers/delete_by_id.rs b/crates/services/admin-api/src/handlers/providers/delete_by_id.rs index d736cb767..000d685e5 100644 --- a/crates/services/admin-api/src/handlers/providers/delete_by_id.rs +++ b/crates/services/admin-api/src/handlers/providers/delete_by_id.rs @@ -1,10 +1,10 @@ //! Providers delete handler +use amp_dataset_store::providers::DeleteError; use axum::{ extract::{Path, State, rejection::PathRejection}, http::StatusCode, }; -use dataset_store::providers::DeleteError; use monitoring::logging; use crate::{ diff --git a/crates/services/admin-api/src/handlers/providers/provider_info.rs b/crates/services/admin-api/src/handlers/providers/provider_info.rs index ef2c743b3..9b46c4287 100644 --- a/crates/services/admin-api/src/handlers/providers/provider_info.rs +++ b/crates/services/admin-api/src/handlers/providers/provider_info.rs @@ -24,7 +24,7 @@ pub struct ProviderInfo { /// The type of provider (e.g., "evm-rpc", "firehose") #[serde_as(as = "serde_with::DisplayFromStr")] #[cfg_attr(feature = "utoipa", schema(value_type = String))] - pub kind: dataset_store::DatasetKind, + pub kind: amp_dataset_store::DatasetKind, /// The blockchain network (e.g., "mainnet", "goerli", "polygon") #[cfg_attr(feature = "utoipa", schema(value_type = String))] pub network: NonEmptyString, @@ -33,11 +33,11 @@ pub struct ProviderInfo { pub rest: serde_json::Map, } -impl TryFrom<(String, dataset_store::providers::ProviderConfig)> for ProviderInfo { +impl TryFrom<(String, amp_dataset_store::providers::ProviderConfig)> for ProviderInfo { type Error = serde_json::Error; fn try_from( - (name, config): (String, dataset_store::providers::ProviderConfig), + (name, config): (String, amp_dataset_store::providers::ProviderConfig), ) -> Result { // SAFETY: Provider names from the dataset store are guaranteed to be non-empty // as they are validated during provider registration and storage. diff --git a/crates/services/admin-api/src/handlers/schema.rs b/crates/services/admin-api/src/handlers/schema.rs index 12282b85f..acea6d2a8 100644 --- a/crates/services/admin-api/src/handlers/schema.rs +++ b/crates/services/admin-api/src/handlers/schema.rs @@ -1,5 +1,6 @@ use std::collections::BTreeMap; +use amp_dataset_store::GetDatasetError; use axum::{ Json, extract::{State, rejection::JsonRejection}, @@ -16,7 +17,6 @@ use common::{ sql_str::SqlStr, }; use datafusion::sql::parser::Statement; -use dataset_store::GetDatasetError; use datasets_common::{ deps::{ alias::{DepAlias, DepAliasOrSelfRef}, diff --git a/crates/services/admin-api/src/scheduler.rs b/crates/services/admin-api/src/scheduler.rs index b2367306c..1b05eed7c 100644 --- a/crates/services/admin-api/src/scheduler.rs +++ b/crates/services/admin-api/src/scheduler.rs @@ -23,8 +23,8 @@ //! - Worker information queries //! - Worker status retrieval +use amp_dataset_store::DatasetKind; use async_trait::async_trait; -use dataset_store::DatasetKind; use datasets_common::{ hash::Hash, hash_reference::HashReference, name::Name, namespace::Namespace, }; diff --git a/crates/services/controller/Cargo.toml b/crates/services/controller/Cargo.toml index f3b47237e..67b75bba5 100644 --- a/crates/services/controller/Cargo.toml +++ b/crates/services/controller/Cargo.toml @@ -10,7 +10,7 @@ async-trait.workspace = true axum.workspace = true common = { path = "../../core/common" } amp-config = { path = "../../config" } -dataset-store = { version = "0.1.0", path = "../../core/dataset-store" } +amp-dataset-store = { version = "0.1.0", path = "../../core/dataset-store" } datasets-common = { version = "0.1.0", path = "../../core/datasets-common" } dump = { version = "0.1.0", path = "../../core/dump" } metadata-db = { version = "0.1.0", path = "../../core/metadata-db" } diff --git a/crates/services/controller/src/scheduler.rs b/crates/services/controller/src/scheduler.rs index cf634d181..4c14b4615 100644 --- a/crates/services/controller/src/scheduler.rs +++ b/crates/services/controller/src/scheduler.rs @@ -31,8 +31,8 @@ use admin_api::scheduler::{ ListJobsError, ListWorkersError, ScheduleJobError, SchedulerJobs, SchedulerWorkers, StopJobError, }; +use amp_dataset_store::DatasetKind; use async_trait::async_trait; -use dataset_store::DatasetKind; use datasets_common::{ hash::Hash, hash_reference::HashReference, name::Name, namespace::Namespace, }; diff --git a/crates/services/controller/src/service.rs b/crates/services/controller/src/service.rs index e1f398ab2..ec60b4963 100644 --- a/crates/services/controller/src/service.rs +++ b/crates/services/controller/src/service.rs @@ -1,6 +1,7 @@ use std::{future::Future, net::SocketAddr, sync::Arc, time::Duration}; use admin_api::ctx::Ctx; +use amp_dataset_store::DatasetStore; use axum::{ Router, http::StatusCode, @@ -8,7 +9,6 @@ use axum::{ serve::{Listener as _, ListenerExt as _}, }; use common::{BoxError, store::Store}; -use dataset_store::DatasetStore; use metadata_db::MetadataDb; use monitoring::telemetry::metrics::Meter; use opentelemetry_instrumentation_tower::HTTPMetricsLayerBuilder; diff --git a/crates/services/server/Cargo.toml b/crates/services/server/Cargo.toml index 87bfc301d..d18eeedb3 100644 --- a/crates/services/server/Cargo.toml +++ b/crates/services/server/Cargo.toml @@ -13,7 +13,7 @@ bincode.workspace = true bytes.workspace = true common = { path = "../../core/common" } datafusion.workspace = true -dataset-store = { path = "../../core/dataset-store" } +amp-dataset-store = { path = "../../core/dataset-store" } dump = { path = "../../core/dump" } futures.workspace = true metadata-db = { path = "../../core/metadata-db" } diff --git a/crates/services/server/src/flight.rs b/crates/services/server/src/flight.rs index 2658ae26c..8fe54a660 100644 --- a/crates/services/server/src/flight.rs +++ b/crates/services/server/src/flight.rs @@ -6,6 +6,7 @@ use std::{collections::BTreeMap, pin::Pin, sync::Arc}; +use amp_dataset_store::{DatasetStore, GetDatasetError}; use arrow_flight::{ ActionType, FlightData, FlightDescriptor, FlightEndpoint, FlightInfo, HandshakeRequest, HandshakeResponse, PutResult, SchemaAsIpc, Ticket, @@ -40,7 +41,6 @@ use common::{ use datafusion::{ common::DFSchema, error::DataFusionError, physical_plan::stream::RecordBatchStreamAdapter, }; -use dataset_store::{DatasetStore, GetDatasetError}; use dump::streaming_query::{QueryMessage, StreamingQuery}; use futures::{ Stream, StreamExt as _, TryStreamExt, diff --git a/crates/services/server/src/service.rs b/crates/services/server/src/service.rs index 0fe205f5a..38e601d75 100644 --- a/crates/services/server/src/service.rs +++ b/crates/services/server/src/service.rs @@ -5,6 +5,7 @@ use std::{future::Future, net::SocketAddr, sync::Arc}; +use amp_dataset_store::DatasetStore; use axum::{ Router, http::StatusCode, @@ -13,7 +14,6 @@ use axum::{ }; use common::{BoxError, store::Store}; use datafusion::error::DataFusionError; -use dataset_store::DatasetStore; use futures::FutureExt; use metadata_db::MetadataDb; use monitoring::{logging, telemetry::metrics::Meter}; diff --git a/crates/services/worker/Cargo.toml b/crates/services/worker/Cargo.toml index aa7db0811..4460b75b6 100644 --- a/crates/services/worker/Cargo.toml +++ b/crates/services/worker/Cargo.toml @@ -9,7 +9,7 @@ backon.workspace = true chrono.workspace = true common = { path = "../../core/common" } amp-config = { path = "../../config" } -dataset-store = { path = "../../core/dataset-store" } +amp-dataset-store = { path = "../../core/dataset-store" } datasets-common = { path = "../../core/datasets-common" } dump = { path = "../../core/dump" } futures.workspace = true diff --git a/crates/services/worker/src/job.rs b/crates/services/worker/src/job.rs index 4b345177c..0f8dda93a 100644 --- a/crates/services/worker/src/job.rs +++ b/crates/services/worker/src/job.rs @@ -1,5 +1,5 @@ +use amp_dataset_store::DatasetKind; use chrono::{DateTime, Utc}; -use dataset_store::DatasetKind; use datasets_common::{hash::Hash, name::Name, namespace::Namespace}; pub use dump::Ctx; use dump::EndBlock; diff --git a/crates/services/worker/src/service.rs b/crates/services/worker/src/service.rs index 34cd8596b..953fced45 100644 --- a/crates/services/worker/src/service.rs +++ b/crates/services/worker/src/service.rs @@ -1,8 +1,8 @@ use std::{future::Future, sync::Arc, time::Duration}; +use amp_dataset_store::DatasetStore; use backon::{ExponentialBuilder, Retryable}; use common::{CachedParquetData, ParquetFooterCache, store::Store as DataStore}; -use dataset_store::DatasetStore; use futures::TryStreamExt as _; use metadata_db::{ Error as MetadataDbError, MetadataDb, NotificationMultiplexerHandle, diff --git a/crates/services/worker/src/service/job_impl.rs b/crates/services/worker/src/service/job_impl.rs index cd56918d6..f7a1669aa 100644 --- a/crates/services/worker/src/service/job_impl.rs +++ b/crates/services/worker/src/service/job_impl.rs @@ -144,7 +144,7 @@ pub enum JobInitError { FetchDataset { reference: HashReference, #[source] - source: dataset_store::GetDatasetError, + source: amp_dataset_store::GetDatasetError, }, /// Failed to get or create active physical table diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 892e6d0e8..bbdea1978 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -25,7 +25,7 @@ common = { path = "../crates/core/common" } amp-config = { path = "../crates/config" } controller = { path = "../crates/services/controller" } ctor = "0.5.0" -dataset-store = { path = "../crates/core/dataset-store" } +amp-dataset-store = { path = "../crates/core/dataset-store" } datasets-common = { path = "../crates/core/datasets-common" } datasets-derived = { path = "../crates/core/datasets-derived" } dotenvy.workspace = true diff --git a/tests/src/main.rs b/tests/src/main.rs index 85a2dd1d0..59df3fc80 100644 --- a/tests/src/main.rs +++ b/tests/src/main.rs @@ -48,12 +48,12 @@ use std::{path::PathBuf, sync::Arc}; use amp_config::Config; -use clap::Parser; -use common::{BoxError, Store}; -use dataset_store::{ +use amp_dataset_store::{ DatasetStore, dataset_and_dependencies, manifests::DatasetManifestsStore, providers::ProviderConfigsStore, }; +use clap::Parser; +use common::{BoxError, Store}; use datasets_common::reference::Reference; use dump::consistency_check; use fs_err as fs; diff --git a/tests/src/testlib/ctx.rs b/tests/src/testlib/ctx.rs index 2faca087a..8d6979ae8 100644 --- a/tests/src/testlib/ctx.rs +++ b/tests/src/testlib/ctx.rs @@ -349,7 +349,7 @@ impl TestCtxBuilder { // Create shared DatasetStore instance (used by both server and worker) let dataset_store = { - use dataset_store::{ + use amp_dataset_store::{ DatasetStore, manifests::DatasetManifestsStore, providers::ProviderConfigsStore, }; let provider_configs_store = diff --git a/tests/src/testlib/fixtures/daemon_controller.rs b/tests/src/testlib/fixtures/daemon_controller.rs index 7647f100b..756179442 100644 --- a/tests/src/testlib/fixtures/daemon_controller.rs +++ b/tests/src/testlib/fixtures/daemon_controller.rs @@ -19,7 +19,7 @@ use tokio::task::JoinHandle; pub struct DaemonController { config: Arc, metadata_db: metadata_db::MetadataDb, - dataset_store: dataset_store::DatasetStore, + dataset_store: amp_dataset_store::DatasetStore, admin_api_addr: SocketAddr, _task: JoinHandle>, @@ -34,7 +34,7 @@ impl DaemonController { config: Arc, metadata_db: metadata_db::MetadataDb, data_store: Store, - dataset_store: dataset_store::DatasetStore, + dataset_store: amp_dataset_store::DatasetStore, meter: Option, ) -> Result { // Convert common config to controller config @@ -76,7 +76,7 @@ impl DaemonController { } /// Get a reference to the dataset store. - pub fn dataset_store(&self) -> &dataset_store::DatasetStore { + pub fn dataset_store(&self) -> &_dataset_store::DatasetStore { &self.dataset_store } diff --git a/tests/src/testlib/fixtures/daemon_server.rs b/tests/src/testlib/fixtures/daemon_server.rs index 074c07c75..a3fba97cb 100644 --- a/tests/src/testlib/fixtures/daemon_server.rs +++ b/tests/src/testlib/fixtures/daemon_server.rs @@ -6,8 +6,8 @@ use std::{net::SocketAddr, sync::Arc}; +use amp_dataset_store::DatasetStore; use common::{BoxError, store::Store}; -use dataset_store::DatasetStore; use metadata_db::MetadataDb; use opentelemetry::metrics::Meter; use server::{config::Config, service::BoundAddrs}; diff --git a/tests/src/testlib/fixtures/daemon_worker.rs b/tests/src/testlib/fixtures/daemon_worker.rs index da095329f..339d89b81 100644 --- a/tests/src/testlib/fixtures/daemon_worker.rs +++ b/tests/src/testlib/fixtures/daemon_worker.rs @@ -6,8 +6,8 @@ use std::sync::Arc; +use amp_dataset_store::DatasetStore; use common::{BoxError, store::Store}; -use dataset_store::DatasetStore; use metadata_db::MetadataDb; use opentelemetry::metrics::Meter; use tokio::task::JoinHandle; diff --git a/tests/src/testlib/helpers.rs b/tests/src/testlib/helpers.rs index 77376396a..765c6f5ca 100644 --- a/tests/src/testlib/helpers.rs +++ b/tests/src/testlib/helpers.rs @@ -9,6 +9,7 @@ pub mod git; use std::{collections::BTreeMap, sync::Arc}; +use amp_dataset_store::{DatasetKind, DatasetStore}; use common::{ BoxError, CachedParquetData, LogicalCatalog, ParquetFooterCache, Store, arrow::array::RecordBatch, @@ -17,7 +18,6 @@ use common::{ sql, sql_str::SqlStr, }; -use dataset_store::{DatasetKind, DatasetStore}; use datasets_common::{reference::Reference, table_name::TableName}; use dump::{EndBlock, compaction::AmpCompactor, consistency_check}; use metadata_db::{MetadataDb, notification_multiplexer}; diff --git a/tests/src/tests/it_sql_dataset_batch_size.rs b/tests/src/tests/it_sql_dataset_batch_size.rs index 903df8f74..850525d18 100644 --- a/tests/src/tests/it_sql_dataset_batch_size.rs +++ b/tests/src/tests/it_sql_dataset_batch_size.rs @@ -4,8 +4,8 @@ use std::{ time::Duration, }; +use amp_dataset_store::DatasetStore; use common::{BoxError, ParquetFooterCache, metadata::Generation}; -use dataset_store::DatasetStore; use datasets_common::reference::Reference; use dump::{ compaction::{AmpCompactor, SegmentSizeLimit},