[WIP] Move in-memory providers to diskann-inmem crate - blocked by orphan rules

Cargo.toml

@@ -14,6 +14,7 @@ members = [
     "diskann",
     # Providers
     "diskann-providers",
+    "diskann-inmem",
     "diskann-disk",
     "diskann-label-filter",
     # Infrastructure
@@ -55,6 +56,7 @@ diskann-platform = { path = "diskann-platform", version = "0.41.0" }
 diskann = { path = "diskann", version = "0.41.0" }
 # Providers
 diskann-providers = { path = "diskann-providers", default-features = false, version = "0.41.0" }
+diskann-inmem = { path = "diskann-inmem", version = "0.41.0" }
 diskann-disk = { path = "diskann-disk", version = "0.41.0" }
 diskann-label-filter = { path = "diskann-label-filter", version = "0.41.0" }
 # Infra

diskann-benchmark/Cargo.toml

@@ -15,6 +15,7 @@ bytemuck.workspace = true
 roaring.workspace = true
 clap = { workspace = true, features = ["derive"] }
 diskann-providers = { workspace = true }
+diskann-inmem = { workspace = true }
 diskann.workspace = true
 diskann-utils.workspace = true
 half = { workspace = true, features = ["rand_distr", "num-traits"] }

diskann-benchmark/src/backend/index/benchmarks.rs

@@ -18,12 +18,12 @@ use diskann_benchmark_runner::{
     Any, Checkpoint,
 };
 use diskann_providers::{
-    index::diskann_async,
     model::{
         configuration::IndexConfiguration,
-        graph::provider::async_::{common, inmem::DefaultProvider},
+        graph::provider::async_::common,
     },
 };
+use diskann_inmem::{diskann_async, DefaultProvider};
 use diskann_utils::{
     future::AsyncFriendly,
     views::{Matrix, MatrixView},

diskann-benchmark/src/backend/index/build.rs

@@ -23,10 +23,10 @@ use diskann_providers::{
     self,
     model::{
         configuration::IndexConfiguration,
-        graph::provider::async_::inmem::{DefaultProvider, SetStartPoints},
     },
     storage::{AsyncIndexMetadata, LoadWith, SaveWith},
 };
+use diskann_inmem::{DefaultProvider, SetStartPoints};
 use diskann_utils::{
     future::AsyncFriendly,
     views::{Matrix, MatrixView},

diskann-benchmark/src/backend/index/product.rs

@@ -37,9 +37,9 @@ mod imp {
     use std::{io::Write, sync::Arc};
 
     use diskann_providers::{
-        index::diskann_async::{self},
         model::{graph::provider::async_::common, IndexConfiguration},
     };
+    use diskann_inmem::diskann_async;
     use diskann_utils::views::{Matrix, MatrixView};
 
     use diskann_benchmark_runner::{

diskann-benchmark/src/backend/index/scalar.rs

@@ -82,12 +82,12 @@ mod imp {
         Any, Checkpoint, Output,
     };
     use diskann_providers::{
-        index::diskann_async::{self},
         model::{
             configuration::IndexConfiguration,
-            graph::provider::async_::{common, inmem},
+            graph::provider::async_::common,
         },
     };
+    use diskann_inmem::{self as inmem, diskann_async};
     use diskann_utils::views::{Matrix, MatrixView};
     use half::f16;
 

diskann-benchmark/src/backend/index/spherical.rs

@@ -64,9 +64,9 @@ mod imp {
         Any, Checkpoint, Output,
     };
     use diskann_providers::{
-        index::diskann_async::{self},
-        model::graph::provider::async_::{common::NoDeletes, inmem},
+        model::graph::provider::async_::common::NoDeletes,
     };
+    use diskann_inmem::{self as inmem, diskann_async};
     use diskann_quantization::alloc::GlobalAllocator;
     use diskann_utils::views::Matrix;
     use rand::SeedableRng;

diskann-benchmark/src/backend/index/update.rs

@@ -15,7 +15,7 @@ use diskann::{
     utils::async_tools,
 };
 use diskann_benchmark_runner::utils::{percentiles, MicroSeconds};
-use diskann_providers::model::graph::provider::async_::inmem::DefaultProvider;
+use diskann_inmem::DefaultProvider;
 use diskann_utils::views::Matrix;
 use serde::Serialize;
 

diskann-benchmark/src/inputs/async_.rs

@@ -17,10 +17,10 @@ use diskann_benchmark_runner::{
 use diskann_providers::{
     model::{
         configuration::IndexConfiguration,
-        graph::provider::async_::inmem::DefaultProviderParameters,
     },
     utils::load_metadata_from_file,
 };
+use diskann_inmem::DefaultProviderParameters;
 use serde::{Deserialize, Serialize};
 
 use crate::{

diskann-disk/Cargo.toml

@@ -16,6 +16,7 @@ targets = ["x86_64-unknown-linux-gnu", "aarch64-pc-windows-msvc", "i686-pc-windo
 diskann = { workspace = true }
 diskann-utils = { workspace = true }
 diskann-providers = { workspace = true }
+diskann-inmem = { workspace = true }
 diskann-vector = { workspace = true }
 diskann-linalg = { workspace = true }
 diskann-quantization = { workspace = true, features = ["rayon"] }

diskann-disk/src/build/builder/build.rs

@@ -19,7 +19,6 @@ use diskann_providers::storage::{StorageReadProvider, StorageWriteProvider};
 use diskann_providers::{
     model::{
         graph::{
-            provider::async_::inmem::DefaultProviderParameters,
             traits::{AdHoc, GraphDataType},
         },
         IndexConfiguration, MAX_PQ_TRAINING_SET_SIZE, NUM_KMEANS_REPS_PQ, NUM_PQ_CENTROIDS,
@@ -30,6 +29,7 @@ use diskann_providers::{
         VectorDataIterator, MAX_MEDOID_SAMPLE_SIZE,
     },
 };
+use diskann_inmem::DefaultProviderParameters;
 use tokio::task::JoinSet;
 use tracing::{debug, info};
 

diskann-inmem/Cargo.toml

@@ -0,0 +1,72 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+[package]
+name = "diskann-inmem"
+version.workspace = true
+authors.workspace = true
+description.workspace = true
+documentation.workspace = true
+license.workspace = true
+edition = "2024"
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+bincode.workspace = true
+bytemuck = { workspace = true, features = ["must_cast"]}
+byteorder.workspace = true
+half = { workspace = true, features = ["bytemuck", "num-traits"] }
+hashbrown.workspace = true
+num-traits.workspace = true
+once_cell.workspace = true
+rand.workspace = true
+rayon.workspace = true
+serde = { workspace = true, features = ["derive"] }
+thiserror.workspace = true
+diskann-platform = { workspace = true }
+diskann-vector = { workspace = true }
+rand_distr.workspace = true
+cfg-if.workspace = true
+diskann-wide = { workspace = true }
+arc-swap.workspace = true
+tracing.workspace = true
+diskann-linalg = { workspace = true }
+vfs = { workspace = true }
+diskann = { workspace = true }
+diskann-utils = { workspace = true }
+diskann-quantization = { workspace = true, features = ["rayon"] }
+tokio = { workspace = true, features = ["rt", "rt-multi-thread"] }
+anyhow.workspace = true
+prost = "0.14.1"
+futures-util.workspace = true
+serde_json.workspace = true
+diskann-providers = { workspace = true }
+
+[dev-dependencies]
+approx.workspace = true
+tempfile.workspace = true
+rstest.workspace = true
+serde_json.workspace = true
+tokio = { workspace = true, features = ["full"] }
+proptest.workspace = true
+
+[package.metadata.docs.rs]
+default-target = "x86_64-pc-windows-msvc"
+targets = [
+    "x86_64-unknown-linux-gnu",
+    "aarch64-pc-windows-msvc",
+    "i686-pc-windows-msvc",
+    "x86_64-pc-windows-msvc",
+]
+
+[features]
+default = []
+experimental_diversity_search = ["diskann/experimental_diversity_search"]
+
+# Some 'cfg's in the source tree will be flagged by `cargo clippy -j 2 --workspace --no-deps --all-targets -- -D warnings`
+[lints.rust]
+unexpected_cfgs = { level = "warn", check-cfg = ['cfg(coverage)'] }
+
+[lints.clippy]
+# Commenting out for now. It would give warnings.
+# undocumented_unsafe_blocks = "warn"
+uninlined_format_args = "allow"