GraphQL improvements for v4

.github/workflows/bench-graphql.yml

@@ -38,7 +38,7 @@ jobs:
       - name: Run GraphQL benchmarks
         run: cd graphql-bench && make bench-local
       - name: Restore metadata file
-        run: git restore graphql-bench/data/apache/master/.raph # otherwise github-action-benchmark fails to create the commit
+        run: git restore graphql-bench/data/apache/master # otherwise github-action-benchmark fails to create the commit
       - name: Print bench results
         run: cat graphql-bench/output.json
       - name: Store benchmark results from master branch

Makefile

@@ -12,17 +12,13 @@ build-all: rust-build
 
 test-all: rust-test-all python-test
 
-test-all-public: rust-test-all-public python-test-public
-
 # Tidying
 
 tidy: rust-fmt build-python stubs python-fmt
 
-tidy-public: rust-fmt build-python-public stubs python-fmt
-
 python-tidy: stubs python-fmt test-graphql-schema
 
-check-pr: tidy-public test-all
+check-pr: tidy test-all
 
 gen-graphql-schema:
 	raphtory schema > raphtory-graphql/schema.graphql
@@ -31,7 +27,6 @@ test-graphql-schema: install-node-tools
 	npx graphql-schema-linter --rules fields-have-descriptions,types-have-descriptions raphtory-graphql/schema.graphql
 
 # Utilities
-
 activate-storage:
 	./scripts/activate_private_storage.py
 
@@ -71,13 +66,12 @@ run-graphql:
 rust-test:
 	cargo test -q
 
-rust-test-all: activate-storage
-	cargo nextest run --all --features=storage
+rust-check:
 	cargo hack check --workspace --all-targets --each-feature  --skip extension-module,default
 
-rust-test-all-public:
+rust-test-all: rust-check
 	cargo nextest run --all
-	cargo hack check --workspace --all-targets --each-feature  --skip extension-module,default,storage
+
 
 ##########
 # Python #
@@ -86,32 +80,23 @@ rust-test-all-public:
 install-python:
 	cd python && maturin build && pip install ../target/wheels/*.whl
 
-build-python-public: deactivate-storage
+build-python:
 	cd python && maturin develop -r --extras=dev
 
-build-python: activate-storage
-	cd python && maturin develop -r --features=storage --extras=dev
+debug-python:
+	cd python && maturin develop --profile=debug --extras=dev
 
 # Testing
-
-python-test: activate-storage
-	cd python && tox run && tox run -e storage
-
-python-test-public:
+python-test:
 	cd python && tox run
 
 python-fmt:
 	cd python && black .
 
-debug-python-public: deactivate-storage
-	cd python && maturin develop --profile=debug
 
 build-python-rtd:
 	cd python && maturin build --profile=build-fast && pip install ../target/wheels/*.whl
 
-debug-python: activate-storage
-	cd python && maturin develop --features=storage,extension-module --extras=dev
-
 ########
 # Docs #
 ########

db4-graph/src/lib.rs

@@ -8,7 +8,7 @@ use std::{
 };
 
 use raphtory_api::core::{
-    entities::{self, properties::meta::Meta},
+    entities::{self, properties::meta::Meta, GidType},
     input::input_node::InputNode,
 };
 use raphtory_core::{
@@ -149,9 +149,10 @@ impl<EXT: PersistentStrategy<NS = NS<EXT>, ES = ES<EXT>, GS = GS<EXT>>> Temporal
     pub fn load_from_path(path: impl AsRef<Path>) -> Result<Self, StorageError> {
         let path = path.as_ref();
         let storage = Layer::load(path)?;
+        let id_type = storage.nodes().id_type();
 
         let gid_resolver_dir = path.join("gid_resolver");
-        let resolver = GIDResolver::new_with_path(&gid_resolver_dir)?;
+        let resolver = GIDResolver::new_with_path(&gid_resolver_dir, id_type)?;
         let node_count = AtomicUsize::new(storage.nodes().num_nodes());
         let wal_dir = path.join("wal");
         let wal = Arc::new(WalImpl::new(Some(wal_dir))?);
@@ -184,9 +185,15 @@ impl<EXT: PersistentStrategy<NS = NS<EXT>, ES = ES<EXT>, GS = GS<EXT>>> Temporal
             std::fs::create_dir_all(dir)?
         }
 
+        let id_type = node_meta
+            .metadata_mapper()
+            .d_types()
+            .first()
+            .and_then(|dtype| GidType::from_prop_type(dtype));
+
         let gid_resolver_dir = graph_dir.as_ref().map(|dir| dir.gid_resolver_dir());
         let logical_to_physical = match gid_resolver_dir {
-            Some(gid_resolver_dir) => GIDResolver::new_with_path(gid_resolver_dir)?,
+            Some(gid_resolver_dir) => GIDResolver::new_with_path(gid_resolver_dir, id_type)?,
             None => GIDResolver::new()?,
         }
         .into();
@@ -212,8 +219,9 @@ impl<EXT: PersistentStrategy<NS = NS<EXT>, ES = ES<EXT>, GS = GS<EXT>>> Temporal
         })
     }
 
-    pub fn disk_storage_enabled(&self) -> bool {
-        self.graph_dir().is_some() && Extension::disk_storage_enabled()
+    pub fn disk_storage_path(&self) -> Option<&Path> {
+        self.graph_dir()
+            .filter(|_| Extension::disk_storage_enabled())
     }
 
     pub fn extension(&self) -> &EXT {

db4-storage/src/pages/edge_page/writer.rs

@@ -2,12 +2,8 @@ use crate::{
     LocalPOS, api::edges::EdgeSegmentOps, error::StorageError, pages::layer_counter::GraphStats,
     segments::edge::segment::MemEdgeSegment,
 };
-use arrow_array::{ArrayRef, BooleanArray};
 use raphtory_api::core::entities::{VID, properties::prop::Prop};
-use raphtory_core::{
-    entities::EID,
-    storage::timeindex::{AsTime, TimeIndexEntry},
-};
+use raphtory_core::storage::timeindex::{AsTime, TimeIndexEntry};
 use std::ops::DerefMut;
 
 pub struct EdgeWriter<

db4-storage/src/pages/layer_counter.rs

@@ -96,8 +96,6 @@ impl GraphStats {
             }
         } else {
             // we need to create the layer
-            self.layers.reserve(layer_id + 1 - self.layers.count());
-
             loop {
                 let new_layer_id = self.layers.push_with(|_| Default::default());
                 if new_layer_id >= layer_id {

db4-storage/src/pages/mod.rs

@@ -450,7 +450,8 @@ fn write_graph_config<EXT: Config>(
     config: &EXT,
 ) -> Result<(), StorageError> {
     let config_file = graph_dir.as_ref().join("graph_config.json");
-    let config_file = std::fs::File::create(config_file).unwrap();
+    let config_file = std::fs::File::create(&config_file)?;
+
     serde_json::to_writer_pretty(config_file, config)?;
     Ok(())
 }
@@ -459,7 +460,7 @@ fn read_graph_config<EXT: PersistentStrategy>(
     graph_dir: impl AsRef<Path>,
 ) -> Result<EXT, StorageError> {
     let config_file = graph_dir.as_ref().join("graph_config.json");
-    let config_file = std::fs::File::open(config_file).unwrap();
+    let config_file = std::fs::File::open(config_file)?;
     let config = serde_json::from_reader(config_file)?;
     Ok(config)
 }
@@ -484,7 +485,7 @@ mod test {
             make_nodes,
         },
     };
-    use chrono::{DateTime, NaiveDateTime, Utc};
+    use chrono::DateTime;
     use proptest::prelude::*;
     use raphtory_api::core::entities::properties::prop::Prop;
     use raphtory_core::{entities::VID, storage::timeindex::TimeIndexOps};
@@ -498,10 +499,7 @@ mod test {
             .collect();
 
         check_edges_support(edges, par_load, false, |graph_dir| {
-            Layer::new(
-                Some(graph_dir),
-                crate::Extension::new(chunk_size, chunk_size),
-            )
+            Layer::new(Some(graph_dir), Extension::new(chunk_size, chunk_size))
         })
     }
 
@@ -511,10 +509,7 @@ mod test {
         par_load: bool,
     ) {
         check_edges_support(edges, par_load, false, |graph_dir| {
-            Layer::new(
-                Some(graph_dir),
-                crate::Extension::new(chunk_size, chunk_size),
-            )
+            Layer::new(Some(graph_dir), Extension::new(chunk_size, chunk_size))
         })
     }
 
@@ -766,14 +761,11 @@ mod test {
                     ("857".to_owned(), Prop::F64(2.56)),
                     (
                         "296".to_owned(),
-                        Prop::NDTime(NaiveDateTime::from_timestamp(1334043671, 0)),
+                        Prop::NDTime(DateTime::from_timestamp(1334043671, 0).unwrap().naive_utc()),
                     ),
                     (
                         "92".to_owned(),
-                        Prop::DTime(DateTime::<Utc>::from_utc(
-                            NaiveDateTime::from_timestamp(994032315, 0),
-                            Utc,
-                        )),
+                        Prop::DTime(DateTime::from_timestamp(994032315, 0).unwrap()),
                     ),
                 ],
             )],

db4-storage/src/pages/node_store.rs

@@ -11,7 +11,10 @@ use crate::{
     segments::node::segment::MemNodeSegment,
 };
 use parking_lot::RwLockWriteGuard;
-use raphtory_api::core::entities::properties::meta::Meta;
+use raphtory_api::core::entities::{
+    GidType,
+    properties::{meta::Meta, prop::PropType},
+};
 use raphtory_core::{
     entities::{EID, VID},
     storage::timeindex::AsTime,
@@ -22,7 +25,6 @@ use std::{
     path::{Path, PathBuf},
     sync::Arc,
 };
-
 // graph // (nodes|edges) // graph segments // layers // chunks
 
 #[derive(Debug)]
@@ -226,6 +228,14 @@ impl<NS: NodeSegmentOps<Extension = EXT>, EXT: Config> NodeStorageInner<NS, EXT>
         Some(NodeWriter::new(segment, &self.stats, head))
     }
 
+    pub fn id_type(&self) -> Option<GidType> {
+        self.node_meta
+            .metadata_mapper()
+            .d_types()
+            .first()
+            .and_then(|dtype| GidType::from_prop_type(dtype))
+    }
+
     pub fn load(
         nodes_path: impl AsRef<Path>,
         edge_meta: Arc<Meta>,

db4-storage/src/properties/mod.rs

@@ -8,8 +8,8 @@ use bigdecimal::ToPrimitive;
 use raphtory_api::core::entities::properties::{
     meta::PropMapper,
     prop::{
-        Prop, PropType, SerdeList, SerdeMap, arrow_dtype_from_prop_type, list_array_from_props,
-        struct_array_from_props,
+        Prop, PropType, SerdeArrowList, SerdeArrowMap, arrow_dtype_from_prop_type,
+        list_array_from_props, struct_array_from_props,
     },
 };
 use raphtory_core::{
@@ -206,7 +206,7 @@ impl Properties {
                     .unwrap();
                 let array_iter = indices
                     .map(|i| lazy_vec.get_opt(i))
-                    .map(|e| e.map(|m| SerdeMap(m)));
+                    .map(|e| e.map(|m| SerdeArrowMap(m)));
 
                 let struct_array = struct_array_from_props(&dt, array_iter);
 
@@ -221,7 +221,7 @@ impl Properties {
 
                 let array_iter = indices
                     .map(|i| lazy_vec.get_opt(i))
-                    .map(|opt_list| opt_list.map(SerdeList));
+                    .map(|opt_list| opt_list.map(SerdeArrowList));
 
                 let list_array = list_array_from_props(&dt, array_iter);
 

db4-storage/src/resolver/mapping_resolver.rs

@@ -27,8 +27,20 @@ impl GIDResolverOps for MappingResolver {
         })
     }
 
-    fn new_with_path(_path: impl AsRef<Path>) -> Result<Self, StorageError> {
-        Self::new()
+    fn new_with_path(
+        _path: impl AsRef<Path>,
+        dtype: Option<GidType>,
+    ) -> Result<Self, StorageError> {
+        match dtype {
+            None => Self::new(),
+            Some(dtype) => {
+                let mapping = match dtype {
+                    GidType::U64 => Mapping::new_u64(),
+                    GidType::Str => Mapping::new_str(),
+                };
+                Ok(Self { mapping })
+            }
+        }
     }
 
     fn len(&self) -> usize {

db4-storage/src/resolver/mod.rs

@@ -11,7 +11,7 @@ pub trait GIDResolverOps {
     fn new() -> Result<Self, StorageError>
     where
         Self: Sized;
-    fn new_with_path(path: impl AsRef<Path>) -> Result<Self, StorageError>
+    fn new_with_path(path: impl AsRef<Path>, dtype: Option<GidType>) -> Result<Self, StorageError>
     where
         Self: Sized;
     fn len(&self) -> usize;

db4-storage/src/segments/graph_prop/segment.rs

@@ -1,5 +1,4 @@
 use crate::{
-    LocalPOS,
     error::StorageError,
     segments::{HasRow, SegmentContainer},
 };

db4-storage/src/segments/node/segment.rs

@@ -550,21 +550,19 @@ impl<P: PersistentStrategy<NS = NodeSegmentView<P>>> NodeSegmentOps for NodeSegm
 
 #[cfg(test)]
 mod test {
-    use std::sync::Arc;
-
-    use raphtory_api::core::entities::properties::{
-        meta::Meta,
-        prop::{Prop, PropType},
-    };
-    use raphtory_core::entities::{EID, ELID, VID};
-    use tempfile::tempdir;
-
     use crate::{
         LocalPOS, NodeSegmentView,
         api::nodes::NodeSegmentOps,
         pages::{layer_counter::GraphStats, node_page::writer::NodeWriter},
         persist::strategy::NoOpStrategy,
     };
+    use raphtory_api::core::entities::properties::{
+        meta::Meta,
+        prop::{Prop, PropType},
+    };
+    use raphtory_core::entities::{EID, ELID, VID};
+    use std::sync::Arc;
+    use tempfile::tempdir;
 
     #[test]
     fn est_size_changes() {