From 7a43d50246946a32317b18cfea70dc88e6fa7855 Mon Sep 17 00:00:00 2001 From: Lucas Jeub Date: Wed, 13 May 2026 10:03:41 +0200 Subject: [PATCH 01/24] cache wip --- Cargo.lock | 24 ++- Cargo.toml | 5 +- raphtory-graphql/Cargo.toml | 2 + raphtory-graphql/src/cache.rs | 155 ++++++++++++++++++++ raphtory-graphql/src/config/cache_config.rs | 3 - raphtory-graphql/src/data.rs | 37 +---- raphtory-graphql/src/lib.rs | 1 + raphtory-graphql/src/rayon.rs | 13 +- 8 files changed, 199 insertions(+), 41 deletions(-) create mode 100644 raphtory-graphql/src/cache.rs diff --git a/Cargo.lock b/Cargo.lock index 1a5ce66f44..98784d3686 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2651,8 +2651,9 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] name = "dynamic-graphql" -version = "0.10.1" -source = "git+https://github.com/miratepuffin/dynamic-graphql?branch=add-arg-descriptions#69a07c5fe3c16b4baf76f676c96cde5865cae1de" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d58055cecef1736f42bee8819059054e18239cda3d5de56b0c8836c2a2322b" dependencies = [ "async-graphql", "dynamic-graphql-derive", @@ -2661,8 +2662,9 @@ dependencies = [ [[package]] name = "dynamic-graphql-derive" -version = "0.10.1" -source = "git+https://github.com/miratepuffin/dynamic-graphql?branch=add-arg-descriptions#69a07c5fe3c16b4baf76f676c96cde5865cae1de" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff1e2aa3d6affef7ce88263e83c0110b11f57409cf25aa81689bcc5f023a602e" dependencies = [ "Inflector", "darling 0.20.11", @@ -6121,6 +6123,18 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quick_cache" +version = "0.6.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a70b1b8b47e31d0498ecbc3c5470bb931399a8bfed1fd79d1717a61ce7f96e3" +dependencies = [ + "ahash", + "equivalent", + "hashbrown 0.16.1", + "parking_lot", +] + [[package]] name = "quinn" version = "0.11.9" @@ -6536,6 +6550,7 @@ dependencies = [ "clap", "config", "crossbeam-channel", + "dashmap", "dynamic-graphql", "futures-util", "itertools 0.13.0", @@ -6551,6 +6566,7 @@ dependencies = [ "poem", "pretty_assertions", "pyo3", + "quick_cache", "raphtory", "raphtory-api", "raphtory-storage", diff --git a/Cargo.toml b/Cargo.toml index 4baf275165..c953d1e2c0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,10 +70,11 @@ raphtory-storage = { version = "0.18.0", path = "raphtory-storage", default-feat raphtory-itertools = { version = "0.18.0", path = "raphtory-itertools" } clam-core = { version = "0.18.0", path = "clam-core" } optd-core = { version = "0.18.0", path = "optd/optd/core" } + async-graphql = { version = "7.2.1", features = ["dynamic-schema"] } bincode = { version = "2", features = ["serde"] } async-graphql-poem = "7.2.1" -dynamic-graphql = { git = "https://github.com/miratepuffin/dynamic-graphql", branch = "add-arg-descriptions" } +dynamic-graphql = "0.10.2" derive_more = "2.1.1" tikv-jemallocator = "0.6.1" reqwest = { version = "0.12.28", default-features = false, features = [ @@ -167,7 +168,6 @@ pest = "2.8.6" pest_derive = "2.8.6" minijinja = "2.18.0" minijinja-contrib = { version = "2.18.0", features = ["datetime"] } - lancedb = { version = "0.27.2", features = [] } heed = "0.22.0" sqlparser = "0.59.0" @@ -185,6 +185,7 @@ arrow-data = { version = "57.3.0" } arrow-select = { version = "57.3.0" } serde_arrow = { version = "0.13.7", features = ["arrow-57"] } moka = { version = "0.12.15", features = ["future"] } +quick_cache = "0.6.21" indexmap = { version = "2.13.0", features = ["rayon"] } fake = { version = "3.1.0", features = ["chrono"] } strsim = { version = "0.11.1" } diff --git a/raphtory-graphql/Cargo.toml b/raphtory-graphql/Cargo.toml index 081ea82f76..390424e323 100644 --- a/raphtory-graphql/Cargo.toml +++ b/raphtory-graphql/Cargo.toml @@ -42,6 +42,7 @@ tracing-opentelemetry = { workspace = true } tracing-subscriber = { workspace = true } walkdir = { workspace = true } ordered-float = { workspace = true } +dashmap.workspace = true chrono = { workspace = true } config = { workspace = true } url = { workspace = true } @@ -54,6 +55,7 @@ ahash = { workspace = true } strum = { workspace = true } strum_macros = { workspace = true } bigdecimal = { workspace = true, features = ["serde"] } +quick_cache = { workspace = true } # python binding optional dependencies pyo3 = { workspace = true, optional = true } diff --git a/raphtory-graphql/src/cache.rs b/raphtory-graphql/src/cache.rs new file mode 100644 index 0000000000..bd71097abd --- /dev/null +++ b/raphtory-graphql/src/cache.rs @@ -0,0 +1,155 @@ +use crate::{graph::GraphWithVectors, paths::ValidGraphPaths, rayon::EVICT_POOL}; +use ahash::HashMap; +use dashmap::{DashMap, Entry}; +use parking_lot::Mutex; +use quick_cache::{unsync::Cache, DefaultHashBuilder, Lifecycle, UnitWeighter, Weighter}; +use raphtory::{ + db::api::{storage::storage::PersistenceStrategy, view::internal::InternalStorageOps}, + prelude::AdditionOps, +}; +use raphtory_storage::core_ops::CoreGraphOps; +use std::{future::Future, marker::PhantomData, sync::Arc}; +use tokio::join; +use tracing::{debug, error}; + +#[derive(Default, Clone)] +pub struct ArcPinned { + dropping: Arc>>, +} + +pub struct CacheShard { + dropping: ArcPinned, + cache: Cache, UnitWeighter, DefaultHashBuilder, ArcPinned>, +} + +fn drop_graph(val: Arc) -> () { + let graph = val.graph; + if let Err(e) = graph.flush() { + error!("Failed to flush graph {}: {e}", val.folder.local_path()) + } + if let Err(e) = val.folder.replace_graph_data(graph) { + error!("Failed to write graph {}: {e}", val.folder.local_path()) + } +} + +impl Lifecycle> for ArcPinned { + type RequestState = (); + + #[inline] + fn is_pinned(&self, _key: &String, val: &Arc) -> bool { + Arc::strong_count(val) > 1 + } + + #[inline] + fn begin_request(&self) -> Self::RequestState { + () + } + + fn on_evict(&self, _state: &mut Self::RequestState, key: String, val: Arc) { + if val.is_dirty() { + self.dropping.insert(key.clone(), val.clone()); + let dropping_map = self.dropping.clone(); + EVICT_POOL.spawn(move || { + debug!( + "Graph {} removed from cache (flushing)", + val.folder.local_path() + ); + drop_graph(val); + dropping_map.remove(&key); + }) + } else { + debug!( + "Graph {} removed from cache (clean)", + val.folder.local_path() + ) + } + } +} + +pub struct GraphCache { + cache: Cache, UnitWeighter, DefaultHashBuilder, ArcPinned>, + dropping: ArcPinned, +} + +impl GraphCache { + pub fn new(items_capacity: usize) -> Self { + let dropping = ArcPinned::default(); + let cache = Cache::with( + items_capacity, + items_capacity as u64, + Default::default(), + Default::default(), + dropping.clone(), + ); + Self { cache, dropping } + } + + /// Get item for key, resurrecting it if it is currently being dropped or looking it up in the cache + pub fn get(&self, key: String) -> Option> { + match self.dropping.dropping.entry(key) { + Entry::Occupied(entry) => { + let (key, value) = entry.remove_entry(); + self.cache.insert(key, value.clone()); + Some(value) + } + Entry::Vacant(entry) => self.cache.get(&entry.into_key()), + } + } + + /// Get item for key, resurrecting it if it is currently being dropped or looking it up in the cache. + /// If the item is not found, insert it using the provided future + pub async fn get_or_insert( + &self, + key: String, + with: impl Future, E>>, + ) -> Result, E> { + match self.dropping.dropping.entry(key) { + Entry::Occupied(entry) => { + let (key, value) = entry.remove_entry(); + self.cache.insert(key, value.clone()); + Ok(value) + } + Entry::Vacant(entry) => { + self.cache + .get_or_insert_async(&entry.into_key(), with) + .await + } + } + } + + pub async fn insert_with( + &self, + key: String, + with: impl Future>, E>, + ) -> Result<(), E> { + self.dropping.dropping.remove(&key); // make sure we don't resurrect the old graph if it is still being dropped + let cache_guard = tokio::spawn( + self.cache + .entry_async(&key, |key, value| EntryAction::<()>::ReplaceWithGuard), + ); + let new_graph = tokio::spawn(with); + let (guard, graph) = join!(cache_guard, new_graph); + + match res { + EntryResult::Replaced(guard, _) | EntryResult::Vacant(guard) => {} + _ => { + unreachable!() + } + } + } + + /// drain all items from the cache + pub fn drain( + &self, + ) -> Drain<'_, String, Arc, UnitWeighter, DefaultHashBuilder, ArcPinned> { + self.cache.drain() + } + + /// remove a graph from the cache without triggering the eviction drop logic + pub fn remove(&self, key: &str) -> Option> { + self.cache + .remove(key) + .or_else(|| self.dropping.dropping.remove(key)) + .map(|(_, v)| v) + } +} diff --git a/raphtory-graphql/src/config/cache_config.rs b/raphtory-graphql/src/config/cache_config.rs index da97275717..49fbb974ed 100644 --- a/raphtory-graphql/src/config/cache_config.rs +++ b/raphtory-graphql/src/config/cache_config.rs @@ -1,19 +1,16 @@ use serde::Deserialize; pub const DEFAULT_CAPACITY: u64 = 30; -pub const DEFAULT_TTI_SECONDS: u64 = 1000000000; #[derive(Debug, Deserialize, PartialEq, Clone, serde::Serialize)] pub struct CacheConfig { pub capacity: u64, - pub tti_seconds: u64, } impl Default for CacheConfig { fn default() -> Self { Self { capacity: DEFAULT_CAPACITY, - tti_seconds: DEFAULT_TTI_SECONDS, } } } diff --git a/raphtory-graphql/src/data.rs b/raphtory-graphql/src/data.rs index 989a706eaf..bb0cdf7b23 100644 --- a/raphtory-graphql/src/data.rs +++ b/raphtory-graphql/src/data.rs @@ -1,6 +1,7 @@ use crate::{ auth::ContextValidation, auth_policy::{AuthorizationPolicy, GraphPermission, NamespacePermission}, + cache::GraphCache, config::app_config::AppConfig, graph::GraphWithVectors, model::{ @@ -142,7 +143,7 @@ pub(crate) fn get_relative_path( /// Inner struct with a drop implementation that cleans up the graphs pub struct DataInner { pub(crate) work_dir: PathBuf, - pub(crate) cache: Cache, + pub(crate) cache: GraphCache, pub(crate) vector_cache: LazyDiskVectorCache, pub(crate) graph_conf: Config, pub(crate) auth_policy: Option>, @@ -167,25 +168,7 @@ impl Data { pub fn new(work_dir: &Path, configs: &AppConfig, graph_conf: Config) -> Self { let cache_configs = &configs.cache; - let cache = Cache::::builder() - .max_capacity(cache_configs.capacity) - .time_to_idle(std::time::Duration::from_secs(cache_configs.tti_seconds)) - .async_eviction_listener(|_, graph, cause| { - // The eviction listener gets called any time a graph is removed from the cache, - // not just when it is evicted. Only serialize on evictions. - async move { - if !cause.was_evicted() { - return; - } - if let Err(e) = - blocking_compute(move || graph.folder.replace_graph_data(graph.graph)).await - { - error!("Error encoding graph to disk on eviction: {e}"); - } - } - .boxed() - }) - .build(); + let cache = GraphCache::new(cache_configs.capacity as usize); #[cfg(feature = "search")] let create_index = configs.index.create_index; @@ -213,8 +196,7 @@ impl Data { } async fn invalidate(&self, path: &str) { - self.cache.invalidate(path).await; - self.cache.run_pending_tasks().await; // make sure the item is actually dropped + self.cache.remove(path); } pub fn validate_path_for_insert( @@ -232,9 +214,9 @@ impl Data { /// # ⚠ Bypasses all permission checks — do not call from resolvers directly. /// Use `get_graph_with_read_permission`, `get_raw_graph_with_read_permission`, or /// `get_graph_with_write_permission` instead. - async fn get_graph(&self, path: &str) -> Result> { + async fn get_graph(&self, path: &str) -> Result, GQLError> { self.cache - .try_get_with(path.into(), self.read_graph_from_disk(path)) + .get_or_insert(path.into(), self.read_graph_from_disk(path)) .await } @@ -248,12 +230,7 @@ impl Data { } pub async fn get_cached_graph(&self, path: &str) -> Option { - self.cache.get(path).await - } - - pub fn has_graph(&self, path: &str) -> bool { - self.cache.contains_key(path) - || ExistingGraphFolder::try_from(self.work_dir.clone(), path).is_ok() + self.cache.get(path.into()).await } pub async fn insert_graph( diff --git a/raphtory-graphql/src/lib.rs b/raphtory-graphql/src/lib.rs index 5a4c7b0cf6..4ab71b0e9f 100644 --- a/raphtory-graphql/src/lib.rs +++ b/raphtory-graphql/src/lib.rs @@ -20,6 +20,7 @@ mod routes; pub mod server; pub mod url_encode; +pub mod cache; pub mod cli; pub mod config; #[cfg(feature = "python")] diff --git a/raphtory-graphql/src/rayon.rs b/raphtory-graphql/src/rayon.rs index cb9352ab98..7bba4dcad4 100644 --- a/raphtory-graphql/src/rayon.rs +++ b/raphtory-graphql/src/rayon.rs @@ -2,14 +2,14 @@ use rayon::{ThreadPool, ThreadPoolBuilder}; use std::sync::LazyLock; use tokio::sync::oneshot; -static WRITE_POOL: LazyLock = LazyLock::new(|| { +pub static WRITE_POOL: LazyLock = LazyLock::new(|| { ThreadPoolBuilder::new() .thread_name(|t| format!("RAP-write-{t}")) .build() .unwrap() }); -static COMPUTE_POOL: LazyLock = LazyLock::new(|| { +pub static COMPUTE_POOL: LazyLock = LazyLock::new(|| { ThreadPoolBuilder::new() .stack_size(16 * 1024 * 1024) .thread_name(|t| format!("RAP-compute-{t}")) @@ -17,6 +17,15 @@ static COMPUTE_POOL: LazyLock = LazyLock::new(|| { .unwrap() }); +pub static EVICT_POOL: LazyLock = LazyLock::new(|| { + ThreadPoolBuilder::new() + .stack_size(16 * 1024 * 1024) + .num_threads(1) + .thread_name(|t| format!("RAP-evict-{t}")) + .build() + .unwrap() +}); + /// Use the rayon threadpool to execute a task /// /// Use this for long-running, compute-heavy work From cb4b78bf0c79a643fe8dbf492d0855ff63ad136e Mon Sep 17 00:00:00 2001 From: Lucas Jeub Date: Tue, 12 May 2026 18:04:03 +0200 Subject: [PATCH 02/24] attempt to make this work with dashmap but it is getting complicated --- raphtory-graphql/src/cache.rs | 152 +++++++++++++++++++++++++++------- 1 file changed, 124 insertions(+), 28 deletions(-) diff --git a/raphtory-graphql/src/cache.rs b/raphtory-graphql/src/cache.rs index bd71097abd..48bf60d8b4 100644 --- a/raphtory-graphql/src/cache.rs +++ b/raphtory-graphql/src/cache.rs @@ -2,19 +2,71 @@ use crate::{graph::GraphWithVectors, paths::ValidGraphPaths, rayon::EVICT_POOL}; use ahash::HashMap; use dashmap::{DashMap, Entry}; use parking_lot::Mutex; -use quick_cache::{unsync::Cache, DefaultHashBuilder, Lifecycle, UnitWeighter, Weighter}; +use quick_cache::{ + sync::{Cache, Drain, EntryAction, EntryResult}, + DefaultHashBuilder, Lifecycle, UnitWeighter, Weighter, +}; use raphtory::{ db::api::{storage::storage::PersistenceStrategy, view::internal::InternalStorageOps}, prelude::AdditionOps, }; use raphtory_storage::core_ops::CoreGraphOps; use std::{future::Future, marker::PhantomData, sync::Arc}; -use tokio::join; +use tokio::{join, sync::Notify}; use tracing::{debug, error}; +#[derive(Clone)] +enum DroppingState { + Dropping { + wait: Arc, + graph: Arc, + }, + Replacing { + wait: Arc, + }, + DroppedWhileReplacing { + wait: Arc, + }, +} + +impl DroppingState { + fn into_wait(self) -> Arc { + match self { + Self::Dropping { wait, .. } + | Self::Replacing { wait } + | Self::DroppedWhileReplacing { wait } => wait, + } + } + + fn as_wait(&self) -> &Arc { + match self { + Self::Dropping { wait, .. } + | Self::Replacing { wait } + | Self::DroppedWhileReplacing { wait } => wait, + } + } + + fn new_dropping(graph: Arc) -> Self { + let wait = Arc::new(Notify::new()); + Self::Dropping { wait, graph } + } + + fn as_dropping(&mut self, dropping_graph: Arc) { + match self { + DroppingState::Dropping { graph, .. } => { + *graph = dropping_graph; + } + DroppingState::Replacing { wait } => { + *self = DroppingState::DroppedWhileReplacing { wait: wait.clone() } + } + DroppingState::DroppedWhileReplacing { .. } => {} + } + } +} + #[derive(Default, Clone)] pub struct ArcPinned { - dropping: Arc>>, + dropping: Arc>, } pub struct CacheShard { @@ -37,7 +89,14 @@ impl Lifecycle> for ArcPinned { #[inline] fn is_pinned(&self, _key: &String, val: &Arc) -> bool { - Arc::strong_count(val) > 1 + if Arc::strong_count(val) > 1 { + return true; + } + if val.is_dirty() { + + return true; + } + false } #[inline] @@ -45,22 +104,36 @@ impl Lifecycle> for ArcPinned { () } - fn on_evict(&self, _state: &mut Self::RequestState, key: String, val: Arc) { - if val.is_dirty() { - self.dropping.insert(key.clone(), val.clone()); + fn on_evict(&self, _state: &mut Self::RequestState, key: String, graph: Arc) { + debug_assert_eq!( + Arc::strong_count(&graph), + 1, + "We should have the only reference to the graph on eviction" + ); + if graph.is_dirty() { + match self.dropping.entry(key.clone()) { + Entry::Occupied(mut entry) => { + entry.get_mut().as_dropping(graph.clone()); + } + Entry::Vacant(entry) => { + entry.insert(DroppingState::new_dropping(graph.clone())); + } + }; let dropping_map = self.dropping.clone(); EVICT_POOL.spawn(move || { debug!( "Graph {} removed from cache (flushing)", - val.folder.local_path() + graph.folder.local_path() ); - drop_graph(val); - dropping_map.remove(&key); + drop_graph(graph); + if let Some((_, state)) = dropping_map.remove(&key) { + state.into_wait().notify_waiters() // this makes sure graph is fully dropped before waking up other tasks + }; }) } else { debug!( "Graph {} removed from cache (clean)", - val.folder.local_path() + graph.folder.local_path() ) } } @@ -84,18 +157,36 @@ impl GraphCache { Self { cache, dropping } } - /// Get item for key, resurrecting it if it is currently being dropped or looking it up in the cache - pub fn get(&self, key: String) -> Option> { - match self.dropping.dropping.entry(key) { - Entry::Occupied(entry) => { - let (key, value) = entry.remove_entry(); - self.cache.insert(key, value.clone()); - Some(value) - } - Entry::Vacant(entry) => self.cache.get(&entry.into_key()), + fn resurrect(&self, key: &str, graph: &Arc) { + // resurrect the graph + let entry = self + .cache + .entry(&key, None, |key, graph| EntryAction::Retain(())); + if let EntryResult::Vacant(placeholder) = entry { + placeholder.insert(graph.clone()).unwrap_or_else(|graph| { + error!("Failed to resurrect graph {}", graph.folder.local_path()); + }); } } + /// Get item for key, resurrecting it if it is currently being dropped or looking it up in the cache + pub async fn get(&self, key: String) -> Option> { + let wait = match self.dropping.dropping.entry(key.clone()) { + Entry::Occupied(entry) => match entry.get() { + DroppingState::Dropping { graph, .. } => { + self.resurrect(&key, graph); + return Some(graph.clone()); + } + DroppingState::Replacing { wait } + | DroppingState::DroppedWhileReplacing { wait } => wait, + }, + Entry::Vacant(entry) => return self.cache.get(&entry.into_key()), + }; + // have to wait for replacement to finish before trying again + wait.notified().await; + self.get(key).await + } + /// Get item for key, resurrecting it if it is currently being dropped or looking it up in the cache. /// If the item is not found, insert it using the provided future pub async fn get_or_insert( @@ -103,11 +194,16 @@ impl GraphCache { key: String, with: impl Future, E>>, ) -> Result, E> { - match self.dropping.dropping.entry(key) { - Entry::Occupied(entry) => { - let (key, value) = entry.remove_entry(); - self.cache.insert(key, value.clone()); - Ok(value) + let wait = match self.dropping.dropping.entry(key) { + Entry::Occupied(mut entry) => { + match entry.get() { + DroppingState::Dropping { graph, .. } => { + self.resurrect(entry.key(), graph); + return Ok(graph.clone()) + } + DroppingState::Replacing { wait } | + DroppingState::DroppedWhileReplacing { wait } => {wait} + } } Entry::Vacant(entry) => { self.cache @@ -120,7 +216,7 @@ impl GraphCache { pub async fn insert_with( &self, key: String, - with: impl Future>, E>, + with: impl Future, E>>, ) -> Result<(), E> { self.dropping.dropping.remove(&key); // make sure we don't resurrect the old graph if it is still being dropped let cache_guard = tokio::spawn( @@ -130,8 +226,8 @@ impl GraphCache { let new_graph = tokio::spawn(with); let (guard, graph) = join!(cache_guard, new_graph); - match res { - EntryResult::Replaced(guard, _) | EntryResult::Vacant(guard) => {} + match guard? { + EntryResult::Replaced(guard, _) | EntryResult::Vacant(guard) => guard.insert(graph??), _ => { unreachable!() } From 938721d91d68bc4c13373495c08f8e72fd7858fc Mon Sep 17 00:00:00 2001 From: Lucas Jeub Date: Wed, 13 May 2026 14:20:25 +0200 Subject: [PATCH 03/24] implement dirty graph handling using pinning only --- raphtory-graphql/src/cache.rs | 308 ++++++++---------- raphtory-graphql/src/cli.rs | 6 +- raphtory-graphql/src/config/app_config.rs | 7 +- raphtory-graphql/src/config/mod.rs | 6 +- raphtory-graphql/src/data.rs | 151 ++++----- raphtory-graphql/src/graph.rs | 67 ++-- raphtory-graphql/src/model/graph/graph.rs | 5 +- .../src/model/graph/meta_graph.rs | 2 +- .../src/model/graph/mutable_graph.rs | 25 +- raphtory-graphql/src/model/mod.rs | 2 +- raphtory-graphql/src/server.rs | 13 - 11 files changed, 268 insertions(+), 324 deletions(-) diff --git a/raphtory-graphql/src/cache.rs b/raphtory-graphql/src/cache.rs index 48bf60d8b4..de3f7bf599 100644 --- a/raphtory-graphql/src/cache.rs +++ b/raphtory-graphql/src/cache.rs @@ -1,4 +1,10 @@ -use crate::{graph::GraphWithVectors, paths::ValidGraphPaths, rayon::EVICT_POOL}; +use crate::{ + data::{InsertionError, MutationErrorInner}, + graph::GraphWithVectors, + paths::ValidGraphPaths, + rayon::{blocking_compute, EVICT_POOL}, + GQLError, +}; use ahash::HashMap; use dashmap::{DashMap, Entry}; use parking_lot::Mutex; @@ -15,237 +21,191 @@ use std::{future::Future, marker::PhantomData, sync::Arc}; use tokio::{join, sync::Notify}; use tracing::{debug, error}; -#[derive(Clone)] -enum DroppingState { - Dropping { - wait: Arc, - graph: Arc, - }, - Replacing { - wait: Arc, - }, - DroppedWhileReplacing { - wait: Arc, - }, -} - -impl DroppingState { - fn into_wait(self) -> Arc { - match self { - Self::Dropping { wait, .. } - | Self::Replacing { wait } - | Self::DroppedWhileReplacing { wait } => wait, - } - } - - fn as_wait(&self) -> &Arc { - match self { - Self::Dropping { wait, .. } - | Self::Replacing { wait } - | Self::DroppedWhileReplacing { wait } => wait, - } - } - - fn new_dropping(graph: Arc) -> Self { - let wait = Arc::new(Notify::new()); - Self::Dropping { wait, graph } - } - - fn as_dropping(&mut self, dropping_graph: Arc) { - match self { - DroppingState::Dropping { graph, .. } => { - *graph = dropping_graph; - } - DroppingState::Replacing { wait } => { - *self = DroppingState::DroppedWhileReplacing { wait: wait.clone() } - } - DroppingState::DroppedWhileReplacing { .. } => {} - } - } -} - -#[derive(Default, Clone)] -pub struct ArcPinned { - dropping: Arc>, -} +#[derive(Default, Copy, Clone)] +pub struct ArcPinned; pub struct CacheShard { - dropping: ArcPinned, - cache: Cache, UnitWeighter, DefaultHashBuilder, ArcPinned>, + cache: Cache, } -fn drop_graph(val: Arc) -> () { - let graph = val.graph; +fn flush_graph(val: GraphWithVectors) -> () { + val.set_flushing(true); + val.set_dirty(false); // make sure this is reset before the flush so any mutation that gets triggered afterwards will set the graph back to dirty + let graph = val.graph(); if let Err(e) = graph.flush() { - error!("Failed to flush graph {}: {e}", val.folder.local_path()) + error!("Failed to flush graph {}: {e}", val.folder().local_path()) } - if let Err(e) = val.folder.replace_graph_data(graph) { - error!("Failed to write graph {}: {e}", val.folder.local_path()) + if let Err(e) = val.folder().replace_graph_data(graph.clone()) { + error!("Failed to write graph {}: {e}", val.folder().local_path()) } + val.set_flushing(false); } -impl Lifecycle> for ArcPinned { +impl Lifecycle for ArcPinned { type RequestState = (); #[inline] - fn is_pinned(&self, _key: &String, val: &Arc) -> bool { - if Arc::strong_count(val) > 1 { + fn begin_request(&self) -> Self::RequestState {} + + #[inline] + fn is_pinned(&self, _key: &String, val: &GraphWithVectors) -> bool { + if val.ref_count() > 1 { return true; } - if val.is_dirty() { + if val.is_dirty() { + if !val.is_flushing() { + let graph = val.clone(); + EVICT_POOL.spawn(move || { + debug!("Flushing graph {}", graph.folder().local_path()); + flush_graph(graph); + }) + } return true; } - false - } - #[inline] - fn begin_request(&self) -> Self::RequestState { - () + val.is_flushing() } - fn on_evict(&self, _state: &mut Self::RequestState, key: String, graph: Arc) { + #[inline] + fn on_evict(&self, state: &mut Self::RequestState, key: String, graph: GraphWithVectors) { debug_assert_eq!( - Arc::strong_count(&graph), + graph.ref_count(), 1, "We should have the only reference to the graph on eviction" ); - if graph.is_dirty() { - match self.dropping.entry(key.clone()) { - Entry::Occupied(mut entry) => { - entry.get_mut().as_dropping(graph.clone()); - } - Entry::Vacant(entry) => { - entry.insert(DroppingState::new_dropping(graph.clone())); - } - }; - let dropping_map = self.dropping.clone(); - EVICT_POOL.spawn(move || { - debug!( - "Graph {} removed from cache (flushing)", - graph.folder.local_path() - ); - drop_graph(graph); - if let Some((_, state)) = dropping_map.remove(&key) { - state.into_wait().notify_waiters() // this makes sure graph is fully dropped before waking up other tasks - }; - }) - } else { - debug!( - "Graph {} removed from cache (clean)", - graph.folder.local_path() - ) - } + debug_assert!(!graph.is_dirty(), "Graph should be clean on eviction"); + debug_assert!( + !graph.is_flushing(), + "Graph should be already flushed on eviction" + ); + + debug!( + "Graph {} removed from cache (clean)", + graph.folder().local_path() + ); } } pub struct GraphCache { - cache: Cache, UnitWeighter, DefaultHashBuilder, ArcPinned>, - dropping: ArcPinned, + cache: Cache, } impl GraphCache { pub fn new(items_capacity: usize) -> Self { - let dropping = ArcPinned::default(); let cache = Cache::with( items_capacity, items_capacity as u64, Default::default(), Default::default(), - dropping.clone(), + Default::default(), ); - Self { cache, dropping } + Self { cache } } - fn resurrect(&self, key: &str, graph: &Arc) { - // resurrect the graph - let entry = self - .cache - .entry(&key, None, |key, graph| EntryAction::Retain(())); - if let EntryResult::Vacant(placeholder) = entry { - placeholder.insert(graph.clone()).unwrap_or_else(|graph| { - error!("Failed to resurrect graph {}", graph.folder.local_path()); - }); - } + /// Get item for key if it is cached + pub fn get(&self, key: &str) -> Option { + self.cache.get(key) } - /// Get item for key, resurrecting it if it is currently being dropped or looking it up in the cache - pub async fn get(&self, key: String) -> Option> { - let wait = match self.dropping.dropping.entry(key.clone()) { - Entry::Occupied(entry) => match entry.get() { - DroppingState::Dropping { graph, .. } => { - self.resurrect(&key, graph); - return Some(graph.clone()); - } - DroppingState::Replacing { wait } - | DroppingState::DroppedWhileReplacing { wait } => wait, - }, - Entry::Vacant(entry) => return self.cache.get(&entry.into_key()), - }; - // have to wait for replacement to finish before trying again - wait.notified().await; - self.get(key).await + pub fn contains_key(&self, key: &str) -> bool { + self.cache.contains_key(key) } - /// Get item for key, resurrecting it if it is currently being dropped or looking it up in the cache. - /// If the item is not found, insert it using the provided future + pub fn iter(&self) -> impl Iterator + use<'_> { + self.cache.iter() + } + + /// Get item for key. If the item is not found, insert it using the provided future pub async fn get_or_insert( &self, - key: String, - with: impl Future, E>>, - ) -> Result, E> { - let wait = match self.dropping.dropping.entry(key) { - Entry::Occupied(mut entry) => { - match entry.get() { - DroppingState::Dropping { graph, .. } => { - self.resurrect(entry.key(), graph); - return Ok(graph.clone()) - } - DroppingState::Replacing { wait } | - DroppingState::DroppedWhileReplacing { wait } => {wait} - } - } - Entry::Vacant(entry) => { - self.cache - .get_or_insert_async(&entry.into_key(), with) - .await - } - } + key: &str, + with: impl Future>, + ) -> Result { + self.cache.get_or_insert_async(key, with).await } + /// Insert a new item into the cache, replacing an existing item if it exists pub async fn insert_with( &self, - key: String, - with: impl Future, E>>, - ) -> Result<(), E> { - self.dropping.dropping.remove(&key); // make sure we don't resurrect the old graph if it is still being dropped - let cache_guard = tokio::spawn( - self.cache - .entry_async(&key, |key, value| EntryAction::<()>::ReplaceWithGuard), - ); - let new_graph = tokio::spawn(with); - let (guard, graph) = join!(cache_guard, new_graph); - - match guard? { - EntryResult::Replaced(guard, _) | EntryResult::Vacant(guard) => guard.insert(graph??), + key: &str, + with: impl Future>, + ) -> Result<(), InsertionError> + where + InsertionError: From, + { + let new_graph = with.await?; + let cache_guard = self + .cache + .entry_async(key, |key, value| EntryAction::<()>::ReplaceWithGuard) + .await; + match cache_guard { + EntryResult::Replaced(guard, old_graph) => { + drop(old_graph); + guard.insert(new_graph) + } + EntryResult::Vacant(guard) => guard.insert(new_graph), _ => { unreachable!() } } + .map_err(|_| InsertionError::Insertion { + graph: key.to_string(), + error: MutationErrorInner::CacheReplacementError, + })?; + Ok(()) } - /// drain all items from the cache - pub fn drain( - &self, - ) -> Drain<'_, String, Arc, UnitWeighter, DefaultHashBuilder, ArcPinned> { - self.cache.drain() + /// clear all items from the cache, flushing them if needed + pub fn flush_and_clear(&self) { + for (_, graph) in self.cache.drain() { + flush_graph(graph); + } } /// remove a graph from the cache without triggering the eviction drop logic - pub fn remove(&self, key: &str) -> Option> { - self.cache - .remove(key) - .or_else(|| self.dropping.dropping.remove(key)) - .map(|(_, v)| v) + /// Note that the cache entry is available again immediately! + pub async fn remove(&self, key: &str) -> Option { + let res = self + .cache + .entry_async(key, |key, graph| EntryAction::<()>::Remove) + .await; + match res { + EntryResult::Removed(_, graph) => Some(graph), + _ => None, + } + } + + /// remove a graph from the cache, locking the cache entry until the graph is dropped + /// this is different from remove which returns the graph and unlocks the entry immediately + pub async fn delete(&self, key: &str) { + let res = self + .cache + .entry_async(key, |key, graph| EntryAction::<()>::ReplaceWithGuard) + .await; + + match res { + EntryResult::Replaced(_guard, graph) => { + blocking_compute(move || drop(graph)).await; + } + _ => {} + } + } + + /// remove a graph from the cache, locking the cache entry until the graph is dropped and the future has completed. + /// if the graph exists, it is dropped first before the future runs + pub async fn invalidate_with(&self, key: &str, with: impl Future) -> E { + let guard = self + .cache + .entry_async(key, |key, graph| EntryAction::<()>::ReplaceWithGuard) + .await; + + match guard { + EntryResult::Replaced(_guard, graph) => { + blocking_compute(move || drop(graph)).await; + with.await + } + _ => with.await, + } } } diff --git a/raphtory-graphql/src/cli.rs b/raphtory-graphql/src/cli.rs index 56a7bbe242..5f303ddeed 100644 --- a/raphtory-graphql/src/cli.rs +++ b/raphtory-graphql/src/cli.rs @@ -4,7 +4,7 @@ use crate::{ config::{ app_config::AppConfigBuilder, auth_config::{DEFAULT_REQUIRE_AUTH_FOR_READS, PUBLIC_KEY_DECODING_ERR_MSG}, - cache_config::{DEFAULT_CAPACITY, DEFAULT_TTI_SECONDS}, + cache_config::DEFAULT_CAPACITY, concurrency_config::{ DEFAULT_DISABLE_BATCHING, DEFAULT_DISABLE_LISTS, DEFAULT_EXCLUSIVE_WRITES, }, @@ -55,9 +55,6 @@ struct ServerArgs { #[arg(long, env = "RAPHTORY_CACHE_CAPACITY", default_value_t = DEFAULT_CAPACITY, help = "Cache capacity")] cache_capacity: u64, - #[arg(long, env = "RAPHTORY_CACHE_TTI_SECONDS", default_value_t = DEFAULT_TTI_SECONDS, help = "Cache time-to-idle in seconds")] - cache_tti_seconds: u64, - #[arg(long, env = "RAPHTORY_LOG_LEVEL", default_value = DEFAULT_LOG_LEVEL, help = "Log level")] log_level: String, @@ -199,7 +196,6 @@ where Commands::Server(server_args) => { let mut builder = AppConfigBuilder::new() .with_cache_capacity(server_args.cache_capacity) - .with_cache_tti_seconds(server_args.cache_tti_seconds) .with_log_level(server_args.log_level) .with_tracing(server_args.tracing) .with_tracing_level(server_args.tracing_level) diff --git a/raphtory-graphql/src/config/app_config.rs b/raphtory-graphql/src/config/app_config.rs index 4157dd84df..bc66d5cff4 100644 --- a/raphtory-graphql/src/config/app_config.rs +++ b/raphtory-graphql/src/config/app_config.rs @@ -98,11 +98,6 @@ impl AppConfigBuilder { self } - pub fn with_cache_tti_seconds(mut self, tti_seconds: u64) -> Self { - self.cache.tti_seconds = tti_seconds; - self - } - pub fn with_auth_public_key( mut self, public_key: Option, @@ -254,7 +249,7 @@ pub fn load_config( app_config_builder = app_config_builder.with_cache_capacity(cache_capacity); } if let Ok(cache_tti_seconds) = settings.get::("cache.tti_seconds") { - app_config_builder = app_config_builder.with_cache_tti_seconds(cache_tti_seconds); + app_config_builder = app_config_builder; } if let Ok(public_key) = settings.get::>("auth.public_key") { diff --git a/raphtory-graphql/src/config/mod.rs b/raphtory-graphql/src/config/mod.rs index 9ef7311dec..4cd7d851f5 100644 --- a/raphtory-graphql/src/config/mod.rs +++ b/raphtory-graphql/src/config/mod.rs @@ -42,7 +42,6 @@ mod tests { .with_tracing(true) .with_tracing_level(TracingLevel::ESSENTIAL) .with_cache_capacity(30) - .with_cache_tti_seconds(1000) .with_auth_public_key(Some( "MCowBQYDK2VwAyEADdrWr1kTLj+wSHlr45eneXmOjlHo3N1DjLIvDa2ozno=".to_owned(), )) @@ -57,10 +56,7 @@ mod tests { #[test] fn test_load_config_with_custom_cache() { - let app_config = AppConfigBuilder::new() - .with_cache_capacity(50) - .with_cache_tti_seconds(1200) - .build(); + let app_config = AppConfigBuilder::new().with_cache_capacity(50).build(); let result = load_config(Some(app_config.clone()), None); diff --git a/raphtory-graphql/src/data.rs b/raphtory-graphql/src/data.rs index bb0cdf7b23..ff4a871e0d 100644 --- a/raphtory-graphql/src/data.rs +++ b/raphtory-graphql/src/data.rs @@ -21,7 +21,6 @@ use crate::{ use async_graphql::Context; use dynamic_graphql::Enum; use futures_util::FutureExt; -use moka::future::Cache; use raphtory::{ db::{ api::{ @@ -57,6 +56,8 @@ pub enum MutationErrorInner { IO(#[from] io::Error), #[error(transparent)] InvalidInternal(#[from] InternalPathValidationError), + #[error("Cache operation failed, simultaneous mutation occurred")] + CacheReplacementError, } #[derive(thiserror::Error, Debug)] @@ -195,10 +196,6 @@ impl Data { .auth_policy = Some(policy); } - async fn invalidate(&self, path: &str) { - self.cache.remove(path); - } - pub fn validate_path_for_insert( &self, path: &str, @@ -214,7 +211,7 @@ impl Data { /// # ⚠ Bypasses all permission checks — do not call from resolvers directly. /// Use `get_graph_with_read_permission`, `get_raw_graph_with_read_permission`, or /// `get_graph_with_write_permission` instead. - async fn get_graph(&self, path: &str) -> Result, GQLError> { + async fn get_graph(&self, path: &str) -> Result { self.cache .get_or_insert(path.into(), self.read_graph_from_disk(path)) .await @@ -225,12 +222,12 @@ impl Data { pub(crate) async fn get_graph_for_test( &self, path: &str, - ) -> Result> { + ) -> Result { self.get_graph(path).await } pub async fn get_cached_graph(&self, path: &str) -> Option { - self.cache.get(path.into()).await + self.cache.get(path.into()) } pub async fn insert_graph( @@ -238,25 +235,19 @@ impl Data { writeable_folder: ValidWriteableGraphFolder, graph: MaterializedGraph, ) -> Result<(), InsertionError> { - self.invalidate(writeable_folder.local_path()).await; + let key = writeable_folder.local_path().to_owned(); let config = self.graph_conf.clone(); - let graph = blocking_compute(move || { - writeable_folder.write_graph_data(graph.clone(), config)?; - let folder = writeable_folder.finish()?; - let graph = GraphWithVectors::new(graph, None, folder.as_existing()?); - Ok::<_, InsertionError>(graph) - }) - .await?; self.cache - .insert(graph.folder.local_path().into(), graph) - .await; - // moka's `insert(..).await` is eventually consistent — the entry is - // queued and may not be visible to `cache.get(..)` immediately. Force - // the pending insert through so a follow-up `MetaGraph.metadata` - // hitting the listing path sees the cached graph instead of falling - // through to `read_constant_graph_properties`, which would read the - // on-disk graph_props before the writer has flushed them. - self.cache.run_pending_tasks().await; + .insert_with( + &key, + blocking_compute(move || { + writeable_folder.write_graph_data(graph.clone(), config)?; + let folder = writeable_folder.finish()?; + let graph = GraphWithVectors::new(graph, None, folder.as_existing()?); + Ok::<_, InsertionError>(graph) + }), + ) + .await?; Ok(()) } @@ -266,13 +257,16 @@ impl Data { folder: ValidWriteableGraphFolder, bytes: R, ) -> Result<(), InsertionError> { - self.invalidate(folder.local_path()).await; let conf = self.graph_conf.clone(); - blocking_io(move || { - folder.write_graph_bytes(bytes, conf)?; - folder.finish() - }) - .await?; + self.cache + .invalidate_with( + &folder.local_path().to_string(), + blocking_io(move || { + folder.write_graph_bytes(bytes, conf)?; + folder.finish() + }), + ) + .await?; Ok(()) } @@ -280,14 +274,18 @@ impl Data { &self, graph_folder: ExistingGraphFolder, ) -> Result<(), MutationErrorInner> { + let key = graph_folder.local_path().to_string(); let dirty_file = mark_dirty(graph_folder.root())?; - self.invalidate(graph_folder.local_path()).await; - blocking_io(move || { - fs::remove_dir_all(graph_folder.root())?; - fs::remove_file(dirty_file)?; - Ok::<_, MutationErrorInner>(()) - }) - .await?; + self.cache + .invalidate_with( + &key, + blocking_io(move || { + fs::remove_dir_all(graph_folder.root())?; + fs::remove_file(dirty_file)?; + Ok::<_, MutationErrorInner>(()) + }), + ) + .await?; Ok(()) } @@ -332,10 +330,14 @@ impl Data { model: CachedEmbeddingModel, ) -> Result<(), GQLError> { let graph = match self.get_cached_graph(folder.local_path()).await { - None => self.read_graph_from_disk_inner(folder.clone()).await?, - Some(graph) => graph, + None => self + .read_graph_from_disk_inner(folder.clone()) + .await? + .graph() + .clone(), + Some(graph) => graph.graph().clone(), }; - self.vectorise_with_template(graph.graph, folder, template, model) + self.vectorise_with_template(graph, folder, template, model) .await; self.cache.remove(folder.local_path()).await; Ok(()) @@ -565,19 +567,21 @@ impl Data { ) -> async_graphql::Result<(ExistingGraphFolder, DynamicGraph)> { let gwv = self.get_graph(path).await?; let typed_graph = match graph_type { - Some(GqlGraphType::Event) => match gwv.graph { - MaterializedGraph::EventGraph(g) => MaterializedGraph::EventGraph(g), + Some(GqlGraphType::Event) => match gwv.graph() { + MaterializedGraph::EventGraph(g) => MaterializedGraph::EventGraph(g.clone()), MaterializedGraph::PersistentGraph(g) => { MaterializedGraph::EventGraph(g.event_graph()) } }, - Some(GqlGraphType::Persistent) => match gwv.graph { + Some(GqlGraphType::Persistent) => match gwv.graph() { MaterializedGraph::EventGraph(g) => { MaterializedGraph::PersistentGraph(g.persistent_graph()) } - MaterializedGraph::PersistentGraph(g) => MaterializedGraph::PersistentGraph(g), + MaterializedGraph::PersistentGraph(g) => { + MaterializedGraph::PersistentGraph(g.clone()) + } }, - None => gwv.graph, + None => gwv.graph().clone(), }; let raw = typed_graph.into_dynamic(); let graph = if let GraphPermission::Read { @@ -588,7 +592,7 @@ impl Data { } else { raw }; - Ok((gwv.folder, graph)) + Ok((gwv.folder().clone(), graph)) } /// For the `graph()` resolver: permission denial → `Ok(None)` (null to client, hides @@ -629,9 +633,8 @@ impl Data { path: &str, ) -> async_graphql::Result { require_at_least_read(ctx, &self.auth_policy, path)?; - self.get_graph(path) - .await - .map_err(|e| async_graphql::Error::new(e.to_string())) + let graph = self.get_graph(path).await?; + Ok(graph) } /// Checks write permission then returns the raw `GraphWithVectors` for mutation operations. @@ -641,9 +644,8 @@ impl Data { path: &str, ) -> async_graphql::Result { require_graph_write(ctx, &self.auth_policy, path)?; - self.get_graph(path) - .await - .map_err(|e| async_graphql::Error::new(e.to_string())) + let graph = self.get_graph(path).await?; + Ok(graph) } /// Checks read permission then returns the vectorised graph, if any. @@ -658,25 +660,14 @@ impl Data { if matches!(perm, GraphPermission::Read { filter: Some(_) }) { return Ok(None); } - Ok(self - .get_graph(path) - .await - .ok() - .and_then(|g| g.vectors) - .map(Into::into)) + let graph = self.get_graph(path).await?; + Ok(graph.vectors().cloned().map(|g| g.into())) } } impl Drop for DataInner { fn drop(&mut self) { - // On drop, serialize graphs that don't have underlying storage. - for (_, graph) in self.cache.iter() { - if graph.is_dirty() { - if let Err(e) = graph.folder.replace_graph_data(graph.graph) { - error!("Error encoding graph to disk on drop: {e}"); - } - } - } + self.cache.flush_and_clear(); } } @@ -753,10 +744,7 @@ pub(crate) mod data_tests { graph.encode(&tmp_work_dir.path().join("test_g")).unwrap(); graph.encode(&tmp_work_dir.path().join("test_g2")).unwrap(); - let configs = AppConfigBuilder::new() - .with_cache_capacity(1) - .with_cache_tti_seconds(2) - .build(); + let configs = AppConfigBuilder::new().with_cache_capacity(1).build(); let data = Data::new(tmp_work_dir.path(), &configs, Default::default()); @@ -769,7 +757,6 @@ pub(crate) mod data_tests { assert!(!data.cache.contains_key("test_g")); data.get_graph("test_g").await.unwrap(); // wait for any eviction - data.cache.run_pending_tasks().await; assert_eq!(data.cache.iter().count(), 1); sleep(Duration::from_secs(3)).await; @@ -813,10 +800,7 @@ pub(crate) mod data_tests { fs::create_dir_all(&g6_path).unwrap(); fs::write(g6_path.join("random-file"), "some-random-content").unwrap(); - let configs = AppConfigBuilder::new() - .with_cache_capacity(1) - .with_cache_tti_seconds(2) - .build(); + let configs = AppConfigBuilder::new().with_cache_capacity(1).build(); let data = Data::new(work_dir, &configs, Default::default()); @@ -877,10 +861,7 @@ pub(crate) mod data_tests { let graph1_original_time = graph1_metadata.modified().unwrap(); let graph2_original_time = graph2_metadata.modified().unwrap(); - let configs = AppConfigBuilder::new() - .with_cache_capacity(10) - .with_cache_tti_seconds(300) - .build(); + let configs = AppConfigBuilder::new().with_cache_capacity(10).build(); let data = Data::new(tmp_work_dir.path(), &configs, Default::default()); @@ -888,7 +869,7 @@ pub(crate) mod data_tests { let loaded_graph2 = data.get_graph("test_graph2").await.unwrap(); // TODO: This test doesn't work with disk storage right now, make sure modification dates actually update correctly! - if loaded_graph1.graph.disk_storage_path().is_some() { + if loaded_graph1.graph().disk_storage_path().is_some() { assert!( !loaded_graph1.is_dirty(), "Graph1 should not be dirty when loaded from disk" @@ -961,10 +942,7 @@ pub(crate) mod data_tests { let graph2_original_time = graph2_metadata.modified().unwrap(); // Create cache with time to idle 3 seconds to force eviction - let configs = AppConfigBuilder::new() - .with_cache_capacity(10) - .with_cache_tti_seconds(3) - .build(); + let configs = AppConfigBuilder::new().with_cache_capacity(10).build(); let data = Data::new(tmp_work_dir.path(), &configs, Default::default()); @@ -992,10 +970,9 @@ pub(crate) mod data_tests { // Sleep to trigger eviction sleep(Duration::from_secs(3)).await; - data.cache.run_pending_tasks().await; // TODO: This test doesn't work with disk storage right now, make sure modification dates actually update correctly! - if loaded_graph1.graph.disk_storage_path().is_some() { + if loaded_graph1.graph().disk_storage_path().is_some() { // Check modification times after eviction let graph1_metadata_after = fs::metadata(&graph1_path).unwrap(); let graph2_metadata_after = fs::metadata(&graph2_path).unwrap(); diff --git a/raphtory-graphql/src/graph.rs b/raphtory-graphql/src/graph.rs index 663adbcd23..02fd01f4fb 100644 --- a/raphtory-graphql/src/graph.rs +++ b/raphtory-graphql/src/graph.rs @@ -34,32 +34,62 @@ use raphtory::prelude::IndexMutationOps; #[derive(Clone)] pub struct GraphWithVectors { - pub graph: MaterializedGraph, - pub vectors: Option>, - pub(crate) folder: ExistingGraphFolder, - pub(crate) is_dirty: Arc, + inner: Arc, +} + +struct GraphWithVectorsInner { + graph: MaterializedGraph, + vectors: Option>, + folder: ExistingGraphFolder, + is_dirty: AtomicBool, + is_flushing: AtomicBool, } impl GraphWithVectors { - pub(crate) fn new( + pub fn new( graph: MaterializedGraph, vectors: Option>, folder: ExistingGraphFolder, ) -> Self { - Self { + let inner = Arc::new(GraphWithVectorsInner { graph, vectors, folder, - is_dirty: Arc::new(AtomicBool::new(false)), - } + is_dirty: AtomicBool::new(false), + is_flushing: AtomicBool::new(false), + }); + Self { inner } + } + + pub fn graph(&self) -> &MaterializedGraph { + &self.inner.graph + } + + pub fn vectors(&self) -> Option<&VectorisedGraph> { + self.inner.vectors.as_ref() } - pub(crate) fn set_dirty(&self, is_dirty: bool) { - self.is_dirty.store(is_dirty, Ordering::SeqCst); + pub fn folder(&self) -> &ExistingGraphFolder { + &self.inner.folder + } + pub fn set_dirty(&self, is_dirty: bool) { + self.inner.is_dirty.store(is_dirty, Ordering::Release); } - pub(crate) fn is_dirty(&self) -> bool { - self.is_dirty.load(Ordering::SeqCst) + pub fn is_dirty(&self) -> bool { + self.inner.is_dirty.load(Ordering::Acquire) + } + + pub fn is_flushing(&self) -> bool { + self.inner.is_flushing.load(Ordering::Acquire) + } + + pub fn set_flushing(&self, is_flushing: bool) { + self.inner.is_flushing.store(is_flushing, Ordering::Release) + } + + pub fn ref_count(&self) -> usize { + Arc::strong_count(&self.inner) } /// Generates and stores embeddings for a batch of nodes. @@ -67,7 +97,7 @@ impl GraphWithVectors { &self, nodes: Vec, ) -> GraphResult<()> { - if let Some(vectors) = &self.vectors { + if let Some(vectors) = &self.inner.vectors { vectors.update_nodes(nodes).await?; } @@ -79,7 +109,7 @@ impl GraphWithVectors { &self, edges: Vec<(T, T)>, ) -> GraphResult<()> { - if let Some(vectors) = &self.vectors { + if let Some(vectors) = &self.inner.vectors { vectors.update_edges(edges).await?; } @@ -116,12 +146,7 @@ impl GraphWithVectors { graph.create_index()?; } - Ok(Self { - graph: graph.clone(), - vectors, - folder: folder.clone().into(), - is_dirty: Arc::new(AtomicBool::new(false)), - }) + Ok(Self::new(graph, vectors, folder.clone())) } } @@ -129,7 +154,7 @@ impl Base for GraphWithVectors { type Base = MaterializedGraph; #[inline] fn base(&self) -> &Self::Base { - &self.graph + &self.inner.graph } } diff --git a/raphtory-graphql/src/model/graph/graph.rs b/raphtory-graphql/src/model/graph/graph.rs index e64ccc03d8..25bf609365 100644 --- a/raphtory-graphql/src/model/graph/graph.rs +++ b/raphtory-graphql/src/model/graph/graph.rs @@ -66,7 +66,7 @@ pub(crate) struct GqlGraph { impl From for GqlGraph { fn from(value: GraphWithVectors) -> Self { - GqlGraph::new(value.folder, value.graph) + GqlGraph::new(value.folder().clone(), value.graph().clone()) } } @@ -694,7 +694,8 @@ impl GqlGraph { let other_g = data .get_graph_with_write_permission(ctx, path.as_ref()) .await? - .graph; + .graph() + .clone(); let g = self.graph.clone(); blocking_compute(move || { other_g.import_nodes(g.nodes(), true)?; diff --git a/raphtory-graphql/src/model/graph/meta_graph.rs b/raphtory-graphql/src/model/graph/meta_graph.rs index e9da3277a4..7033a81d1e 100644 --- a/raphtory-graphql/src/model/graph/meta_graph.rs +++ b/raphtory-graphql/src/model/graph/meta_graph.rs @@ -116,7 +116,7 @@ impl MetaGraph { let data: &Data = ctx.data_unchecked(); if let Some(graph) = data.get_cached_graph(self.folder.local_path()).await { return Ok(graph - .graph + .graph() .metadata() .iter() .filter_map(|(key, value)| value.map(|prop| GqlProperty::new(key.into(), prop))) diff --git a/raphtory-graphql/src/model/graph/mutable_graph.rs b/raphtory-graphql/src/model/graph/mutable_graph.rs index 4dc852518a..6778097437 100644 --- a/raphtory-graphql/src/model/graph/mutable_graph.rs +++ b/raphtory-graphql/src/model/graph/mutable_graph.rs @@ -23,6 +23,7 @@ use raphtory_api::core::storage::arc_str::OptionAsStr; use std::{ error::Error, fmt::{Debug, Display, Formatter}, + sync::Arc, }; #[derive(Debug)] @@ -166,17 +167,17 @@ impl GqlMutableGraph { )] graph_type: Option, ) -> GqlGraph { - let folder = self.graph.folder.clone(); + let folder = self.graph.folder().clone(); match graph_type { - Some(GqlGraphType::Event) => match self.graph.graph.clone() { + Some(GqlGraphType::Event) => match self.graph.graph().clone() { MaterializedGraph::EventGraph(g) => GqlGraph::new(folder, g), MaterializedGraph::PersistentGraph(g) => GqlGraph::new(folder, g.event_graph()), }, - Some(GqlGraphType::Persistent) => match self.graph.graph.clone() { + Some(GqlGraphType::Persistent) => match self.graph.graph().clone() { MaterializedGraph::EventGraph(g) => GqlGraph::new(folder, g.persistent_graph()), MaterializedGraph::PersistentGraph(g) => GqlGraph::new(folder, g), }, - None => GqlGraph::new(folder, self.graph.graph.clone()), + None => GqlGraph::new(folder, self.graph.graph().clone()), } } @@ -524,8 +525,14 @@ impl GqlMutableGraph { async fn flush(&self) -> Result { let self_clone = self.clone(); blocking_write(move || { - self_clone.graph.graph.flush()?; - Ok(true) + self_clone.graph.set_flushing(true); + self_clone.graph.set_dirty(false); + let res = self_clone.graph.graph().flush(); + if res.is_err() { + self_clone.graph.set_dirty(true) + } + self_clone.graph.set_flushing(false); + res.map(|_| true) }) .await } @@ -959,7 +966,7 @@ mod tests { let limit = 5; let result = mutable_graph .graph - .vectors + .vectors() .unwrap() .nodes_by_similarity(&embedding.into(), limit, None) .execute() @@ -1035,7 +1042,7 @@ mod tests { let limit = 5; let result = mutable_graph .graph - .vectors + .vectors() .unwrap() .nodes_by_similarity(&embedding.into(), limit, None) .execute() @@ -1116,7 +1123,7 @@ mod tests { let limit = 5; let result = mutable_graph .graph - .vectors + .vectors() .unwrap() .edges_by_similarity(&embedding.into(), limit, None) .execute() diff --git a/raphtory-graphql/src/model/mod.rs b/raphtory-graphql/src/model/mod.rs index 41b846552a..d879b060f9 100644 --- a/raphtory-graphql/src/model/mod.rs +++ b/raphtory-graphql/src/model/mod.rs @@ -438,7 +438,7 @@ impl Mut { let overwrite = overwrite.unwrap_or(false); let src = data.get_raw_graph_with_read_permission(ctx, path).await?; let folder = data.validate_path_for_insert(new_path, overwrite)?; - data.insert_graph(folder, src.graph).await?; + data.insert_graph(folder, src.graph().clone()).await?; Ok(true) } diff --git a/raphtory-graphql/src/server.rs b/raphtory-graphql/src/server.rs index eb41886281..f0ec763eb5 100644 --- a/raphtory-graphql/src/server.rs +++ b/raphtory-graphql/src/server.rs @@ -263,16 +263,6 @@ impl GraphServer { let work_dir = self.data.work_dir.clone(); - // Otherwise evictions are only triggered when the cache is actively touched - let cache_clone = self.data.cache.clone(); - let cache_task: AbortOnDrop<()> = AbortOnDrop(tokio::spawn(async move { - let mut interval = tokio::time::interval(std::time::Duration::from_secs(1)); - loop { - interval.tick().await; - cache_clone.run_pending_tasks().await; - } - })); - // it is important that this runs after algorithms have been pushed to PLUGIN_ALGOS static variable let app = self .generate_endpoint(tp.clone().map(|tp| tp.tracer(tracer_name))) @@ -296,7 +286,6 @@ impl GraphServer { Ok(RunningGraphServer { signal_sender, server_result, - cache_task, }) } @@ -393,13 +382,11 @@ impl Future for AbortOnDrop { pub struct RunningGraphServer { signal_sender: Sender<()>, server_result: AbortOnDrop>, - cache_task: AbortOnDrop<()>, } impl RunningGraphServer { /// Stop the server. pub async fn stop(&self) { - self.cache_task.abort(); let _ignored = self.signal_sender.send(()).await; } From 2d8a856a6ef49055338d7953a600fade17156c72 Mon Sep 17 00:00:00 2001 From: Lucas Jeub Date: Wed, 13 May 2026 14:45:27 +0200 Subject: [PATCH 04/24] fix python --- raphtory-graphql/src/cache.rs | 1 - raphtory-graphql/src/python/server/server.rs | 5 ----- 2 files changed, 6 deletions(-) diff --git a/raphtory-graphql/src/cache.rs b/raphtory-graphql/src/cache.rs index de3f7bf599..c632f46e4b 100644 --- a/raphtory-graphql/src/cache.rs +++ b/raphtory-graphql/src/cache.rs @@ -7,7 +7,6 @@ use crate::{ }; use ahash::HashMap; use dashmap::{DashMap, Entry}; -use parking_lot::Mutex; use quick_cache::{ sync::{Cache, Drain, EntryAction, EntryResult}, DefaultHashBuilder, Lifecycle, UnitWeighter, Weighter, diff --git a/raphtory-graphql/src/python/server/server.rs b/raphtory-graphql/src/python/server/server.rs index 3fccbe0aa0..0969a95250 100644 --- a/raphtory-graphql/src/python/server/server.rs +++ b/raphtory-graphql/src/python/server/server.rs @@ -85,7 +85,6 @@ impl PyGraphServer { signature = ( work_dir, cache_capacity = None, - cache_tti_seconds = None, log_level = None, tracing=None, tracing_level=None, @@ -113,7 +112,6 @@ impl PyGraphServer { fn py_new( work_dir: PathBuf, cache_capacity: Option, - cache_tti_seconds: Option, log_level: Option, tracing: Option, tracing_level: Option, @@ -167,9 +165,6 @@ impl PyGraphServer { if let Some(cache_capacity) = cache_capacity { app_config_builder = app_config_builder.with_cache_capacity(cache_capacity); } - if let Some(cache_tti_seconds) = cache_tti_seconds { - app_config_builder = app_config_builder.with_cache_tti_seconds(cache_tti_seconds); - } app_config_builder = app_config_builder .with_auth_public_key(auth_public_key) .map_err(|_| PyValueError::new_err(PUBLIC_KEY_DECODING_ERR_MSG))?; From d306bf0aa0325a6191408c1707836517cf9c1eca Mon Sep 17 00:00:00 2001 From: Lucas Jeub Date: Wed, 13 May 2026 17:25:41 +0200 Subject: [PATCH 05/24] remove test of tti-based eviction as it is no longer a thing --- raphtory-graphql/src/data.rs | 12 ------------ raphtory-graphql/src/model/mod.rs | 7 +++++-- .../src/db/graph/views/property_redacted_graph.rs | 5 ++--- 3 files changed, 7 insertions(+), 17 deletions(-) diff --git a/raphtory-graphql/src/data.rs b/raphtory-graphql/src/data.rs index ff4a871e0d..8ffb92d221 100644 --- a/raphtory-graphql/src/data.rs +++ b/raphtory-graphql/src/data.rs @@ -758,18 +758,6 @@ pub(crate) mod data_tests { data.get_graph("test_g").await.unwrap(); // wait for any eviction assert_eq!(data.cache.iter().count(), 1); - - sleep(Duration::from_secs(3)).await; - assert!(!data.cache.contains_key("test_g")); - assert!(!data.cache.contains_key("test_g2")); - // FIXME: this test is not doing anything because calling cache.contains_key() runs - // any pending evictions. To actually test it we need this assertion: - // assert_eq!(data.cache.entry_count(), 0); - // Which currently does not work because the server task to trigger evictions is not running - // in this context. The problem is if we do run it by creating a server and calling - // server.start() the server gets consumed and we loose access to the cache to be able to run - // the check. If rework the server implementation and this becomes feasible we should change - // this test } #[tokio::test] diff --git a/raphtory-graphql/src/model/mod.rs b/raphtory-graphql/src/model/mod.rs index d879b060f9..b25419818a 100644 --- a/raphtory-graphql/src/model/mod.rs +++ b/raphtory-graphql/src/model/mod.rs @@ -2,7 +2,6 @@ use crate::{ auth::ContextValidation, auth_policy::{AuthorizationPolicy, NamespacePermission}, data::{parent_namespace, require_graph_write, Data, GqlGraphType, PermissionError}, - graph::GraphWithVectors, model::{ graph::{ collection::GqlCollection, graph::GqlGraph, index::IndexSpecInput, @@ -541,7 +540,11 @@ impl Mut { let data = ctx.data_unchecked::(); #[cfg(feature = "search")] { - let graph = data.get_graph_with_write_permission(ctx, path).await?.graph; + let graph = data + .get_graph_with_write_permission(ctx, path) + .await? + .graph() + .clone(); match index_spec { Some(index_spec) => { let index_spec = index_spec.to_index_spec(graph.clone())?; diff --git a/raphtory/src/db/graph/views/property_redacted_graph.rs b/raphtory/src/db/graph/views/property_redacted_graph.rs index e6eace9e47..c06aa001d7 100644 --- a/raphtory/src/db/graph/views/property_redacted_graph.rs +++ b/raphtory/src/db/graph/views/property_redacted_graph.rs @@ -1,8 +1,7 @@ use crate::db::api::{ properties::internal::{ - EdgePropertySchemaOps, InheritEdgePropertySchemaOps, InheritNodePropertySchemaOps, - InheritTemporalPropertyViewOps, InternalMetadataOps, InternalTemporalPropertiesOps, - NodePropertySchemaOps, + EdgePropertySchemaOps, InheritTemporalPropertyViewOps, InternalMetadataOps, + InternalTemporalPropertiesOps, NodePropertySchemaOps, }, view::{ internal::{ From 647cffe231496e23ae466d4f0a5173fd94dccafc Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 13 May 2026 19:31:55 +0000 Subject: [PATCH 06/24] chore: apply tidy-public auto-fixes --- python/python/raphtory/graphql/__init__.pyi | 1 - 1 file changed, 1 deletion(-) diff --git a/python/python/raphtory/graphql/__init__.pyi b/python/python/raphtory/graphql/__init__.pyi index b5c21e2146..39e07ceca4 100644 --- a/python/python/raphtory/graphql/__init__.pyi +++ b/python/python/raphtory/graphql/__init__.pyi @@ -87,7 +87,6 @@ class GraphServer(object): cls, work_dir: str | PathLike, cache_capacity: Optional[int] = None, - cache_tti_seconds: Optional[int] = None, log_level: Optional[str] = None, tracing: Optional[bool] = None, tracing_level: Optional[str] = None, From 3ce4ad8ae25e943bc41ea699ddd0d54b4dee7e4b Mon Sep 17 00:00:00 2001 From: Lucas Jeub Date: Thu, 14 May 2026 13:19:04 +0200 Subject: [PATCH 07/24] make the server startup work with port=0 and add fallback when the server is started without giving a specific port --- raphtory-graphql/src/python/server/mod.rs | 4 ++ raphtory-graphql/src/python/server/server.rs | 62 ++++++++++++-------- raphtory-graphql/src/server.rs | 50 +++++++++++++--- 3 files changed, 81 insertions(+), 35 deletions(-) diff --git a/raphtory-graphql/src/python/server/mod.rs b/raphtory-graphql/src/python/server/mod.rs index 4a33de6e01..5b9e24a7cb 100644 --- a/raphtory-graphql/src/python/server/mod.rs +++ b/raphtory-graphql/src/python/server/mod.rs @@ -10,6 +10,10 @@ pub(crate) enum BridgeCommand { StopListening, } +pub(crate) struct ServerStarted { + port: u16, +} + pub(crate) fn wait_server(running_server: &mut Option) -> PyResult<()> { let owned_running_server = running_server .take() diff --git a/raphtory-graphql/src/python/server/server.rs b/raphtory-graphql/src/python/server/server.rs index 0969a95250..4e25f87cdc 100644 --- a/raphtory-graphql/src/python/server/server.rs +++ b/raphtory-graphql/src/python/server/server.rs @@ -3,12 +3,15 @@ use crate::{ app_config::AppConfigBuilder, auth_config::PUBLIC_KEY_DECODING_ERR_MSG, otlp_config::TracingLevel, }, - python::server::{running_server::PyRunningGraphServer, wait_server, BridgeCommand}, + python::server::{ + running_server::PyRunningGraphServer, wait_server, BridgeCommand, ServerStarted, + }, server::apply_server_extension, GraphServer, }; +use crossbeam_channel::RecvTimeoutError; use pyo3::{ - exceptions::{PyAttributeError, PyException, PyValueError}, + exceptions::{PyAttributeError, PyException, PyRuntimeError, PyValueError}, prelude::*, }; use raphtory::{ @@ -19,7 +22,7 @@ use raphtory::{ }, vectors::template::{DocumentTemplate, DEFAULT_EDGE_TEMPLATE, DEFAULT_NODE_TEMPLATE}, }; -use std::{path::PathBuf, thread}; +use std::{path::PathBuf, thread, time::Duration}; /// A class for defining and running a Raphtory GraphQL server /// @@ -299,7 +302,8 @@ impl PyGraphServer { /// Start the server and return a handle to it. /// /// Arguments: - /// port (int): the port to use. Defaults to 1736. + /// port (int, optional): the port to use. If not specified, tries 1736 by default and if that is not available starts on an arbitrary port. + /// If specified and the port is in use, the server will fail to start. /// timeout_ms (int): wait for server to be online. Defaults to 5000. /// /// The server is stopped if not online within timeout_ms but manages to come online as soon as timeout_ms finishes! @@ -307,17 +311,28 @@ impl PyGraphServer { /// Returns: /// RunningGraphServer: The running server #[pyo3( - signature = (port = 1736, timeout_ms = 5000) + signature = (port = None, timeout_ms = 5000) )] - pub fn start(&self, py: Python, port: u16, timeout_ms: u64) -> PyResult { + pub fn start(&self, port: Option, timeout_ms: u64) -> PyResult { let (sender, receiver) = crossbeam_channel::bounded::(1); + let (start_sender, start_receiver) = crossbeam_channel::bounded::(1); let cloned_sender = sender.clone(); let server = self.0.clone(); let join_handle = thread::spawn(move || { block_on(async move { - let handler = server.start_with_port(port); - let running_server = handler.await?; + let running_server = match port { + None => server.start().await?, + Some(port) => server.start_with_port(port).await?, + }; + if let Err(_) = start_sender.send(ServerStarted { + port: running_server.port(), + }) { + // This happens if the other end of the channel doesn't exist + running_server.stop().await; + return Ok(()); + }; + let tokio_sender = running_server._get_sender().clone(); tokio::task::spawn_blocking(move || { match receiver.recv().expect("Failed to wait for cancellation") { @@ -333,35 +348,30 @@ impl PyGraphServer { }) }); - let mut server = PyRunningGraphServer::new(join_handle, sender, port)?; - if let Some(_server_handler) = &server.server_handler { - let url = format!("http://localhost:{port}"); - let result = server.wait_for_server_online(&url, timeout_ms); - match result { - Ok(_) => return Ok(server), - Err(e) => { - PyRunningGraphServer::stop_server(&mut server, py)?; - Err(e) - } - } - } else { - Err(PyException::new_err("Failed to start server")) - } + let ServerStarted { port } = start_receiver + .recv_timeout(Duration::from_millis(timeout_ms)) + .map_err(|err| { + let _ = sender.try_send(BridgeCommand::StopServer); // best effort cleanup + PyRuntimeError::new_err("Failed to start server") + })?; + let server = PyRunningGraphServer::new(join_handle, sender, port)?; + Ok(server) } /// Run the server until completion. /// /// Arguments: - /// port (int): The port to use. Defaults to 1736. + /// port (int, optional): The port to use. If not specified, tries 1736 by default and if that is not available starts on an arbitrary port. + /// If specified and the port is in use, the server will fail to start. /// timeout_ms (int): Timeout for waiting for the server to start. Defaults to 180000. /// /// Returns: /// None: #[pyo3( - signature = (port = 1736, timeout_ms = 180000) + signature = (port = None, timeout_ms = 180000) )] - pub fn run(&self, py: Python, port: u16, timeout_ms: u64) -> PyResult<()> { - let mut server = self.start(py, port, timeout_ms)?.server_handler; + pub fn run(&self, py: Python, port: Option, timeout_ms: u64) -> PyResult<()> { + let mut server = self.start(port, timeout_ms)?.server_handler; py.detach(|| wait_server(&mut server)) } } diff --git a/raphtory-graphql/src/server.rs b/raphtory-graphql/src/server.rs index f0ec763eb5..02760119a0 100644 --- a/raphtory-graphql/src/server.rs +++ b/raphtory-graphql/src/server.rs @@ -8,7 +8,7 @@ use crate::{ App, }, observability::open_telemetry::OpenTelemetry, - paths::ExistingGraphFolder, + paths::{ExistingGraphFolder, PathValidationError::IOError}, routes::{health, version, PublicFilesEndpoint}, server::ServerError::SchemaError, GQLError, @@ -19,7 +19,7 @@ use opentelemetry::trace::TracerProvider; use opentelemetry_sdk::trace::{Tracer, TracerProvider as TP}; use poem::{ get, - listener::TcpListener, + listener::{Acceptor, Listener, TcpListener}, middleware::{Compression, CompressionEndpoint, Cors, CorsEndpoint}, web::CompressionLevel, EndpointExt, Route, Server, @@ -32,6 +32,7 @@ use serde_json::json; use std::{ fs::create_dir_all, future::Future, + io::ErrorKind, ops::Deref, path::{Path, PathBuf}, pin::Pin, @@ -50,7 +51,7 @@ use tokio::{ task, task::JoinHandle, }; -use tracing::{debug, info}; +use tracing::{debug, info, warn}; use tracing_subscriber::{ fmt, fmt::format::FmtSpan, layer::SubscriberExt, util::SubscriberInitExt, Registry, }; @@ -232,12 +233,26 @@ impl GraphServer { } /// Start the server on the default port and return a handle to it. + /// If the default port is in use, pub async fn start(&self) -> IoResult { - self.start_with_port(DEFAULT_PORT).await + match self.start_with_port(DEFAULT_PORT).await { + Ok(server) => Ok(server), + Err(err) => { + if matches!(err.kind(), ErrorKind::AddrInUse) { + warn!("Default port {DEFAULT_PORT} already in use, retrying with port=0"); + self.start_with_port(0).await + } else { + Err(err) + } + } + } } /// Start the server on the given port and return a handle to it. pub async fn start_with_port(&self, port: u16) -> IoResult { + let acceptor = TcpListener::bind(format!("0.0.0.0:{port}")) + .into_acceptor() + .await?; // set up opentelemetry first of all let config = self.config.clone(); let filter = config.logging.get_log_env(); @@ -270,7 +285,22 @@ impl GraphServer { let (signal_sender, signal_receiver) = mpsc::channel(1); - info!("UI listening on 0.0.0.0:{port}, live at: http://localhost:{port}"); + let actual_port = acceptor + .local_addr() + .into_iter() + .next() + .unwrap() + .as_socket_addr() + .unwrap() + .port(); + let server_task = Server::new_with_acceptor(acceptor).run_with_graceful_shutdown( + app, + server_termination(signal_receiver, tp), + None, + ); + let server_result = AbortOnDrop(tokio::spawn(server_task)); + + info!("UI listening on 0.0.0.0:{actual_port}, live at: http://localhost:{actual_port}"); debug!( "Server configurations: {}", json!({ @@ -279,13 +309,10 @@ impl GraphServer { }) ); - let server_task = Server::new(TcpListener::bind(format!("0.0.0.0:{port}"))) - .run_with_graceful_shutdown(app, server_termination(signal_receiver, tp), None); - let server_result = AbortOnDrop(tokio::spawn(server_task)); - Ok(RunningGraphServer { signal_sender, server_result, + port: actual_port, }) } @@ -382,6 +409,7 @@ impl Future for AbortOnDrop { pub struct RunningGraphServer { signal_sender: Sender<()>, server_result: AbortOnDrop>, + port: u16, } impl RunningGraphServer { @@ -395,6 +423,10 @@ impl RunningGraphServer { self.server_result.await.expect("Server panicked") } + pub fn port(&self) -> u16 { + self.port + } + // TODO: make this optional with some python feature flag pub fn _get_sender(&self) -> &Sender<()> { &self.signal_sender From d7d1aa2ee9cb008f42ddfb000b130342eb09c159 Mon Sep 17 00:00:00 2001 From: Lucas Jeub Date: Thu, 14 May 2026 13:19:16 +0200 Subject: [PATCH 08/24] some cleanup --- db4-storage/src/pages/graph_prop_store.rs | 12 ++++------- .../test_graphql/test_namespace.py | 20 +++++++++---------- raphtory-graphql/src/cache.rs | 19 ++++-------------- raphtory-graphql/src/data.rs | 1 - .../src/model/graph/mutable_graph.rs | 1 - .../graph/storage_ops/property_schema.rs | 4 +--- raphtory/src/db/api/storage/storage.rs | 12 ++--------- raphtory/src/graph_loader/mod.rs | 2 +- 8 files changed, 22 insertions(+), 49 deletions(-) diff --git a/db4-storage/src/pages/graph_prop_store.rs b/db4-storage/src/pages/graph_prop_store.rs index c91f436dd5..05c2a42cdf 100644 --- a/db4-storage/src/pages/graph_prop_store.rs +++ b/db4-storage/src/pages/graph_prop_store.rs @@ -9,6 +9,7 @@ use crate::{ }; use raphtory_api::core::entities::properties::meta::Meta; use std::{ + marker::PhantomData, path::{Path, PathBuf}, sync::Arc, }; @@ -23,10 +24,7 @@ pub struct GraphPropStorageInner { /// Stores graph prop metadata (prop name -> prop id mappings). meta: Arc, - - path: Option, - - ext: EXT, + _ext: PhantomData, } impl, EXT: PersistenceStrategy> @@ -37,9 +35,8 @@ impl, EXT: PersistenceStrategy> Self { page, - path: path.map(|p| p.to_path_buf()), meta, - ext, + _ext: PhantomData, } } @@ -52,9 +49,8 @@ impl, EXT: PersistenceStrategy> path.as_ref(), ext.clone(), )?), - path: Some(path.as_ref().to_path_buf()), meta: graph_props_meta, - ext, + _ext: PhantomData, }) } diff --git a/python/tests/test_base_install/test_graphql/test_namespace.py b/python/tests/test_base_install/test_graphql/test_namespace.py index 35fbb6de11..5ab219149a 100644 --- a/python/tests/test_base_install/test_graphql/test_namespace.py +++ b/python/tests/test_base_install/test_graphql/test_namespace.py @@ -294,8 +294,8 @@ def test_wrong_paths(): with pytest.raises(Exception) as excinfo: client.query(query) assert ( - "Only relative paths are allowed to be used within the working_dir: /test" - in str(excinfo.value) + "Only relative paths are allowed to be used within the working_dir: /test" + in str(excinfo.value) ) query = """{ @@ -309,8 +309,8 @@ def test_wrong_paths(): with pytest.raises(Exception) as excinfo: client.query(query) assert ( - "References to the parent dir are not allowed within the path: test/../../" - in str(excinfo.value) + "References to the parent dir are not allowed within the path: test/../../" + in str(excinfo.value) ) query = """{ @@ -324,8 +324,8 @@ def test_wrong_paths(): with pytest.raises(Exception) as excinfo: client.query(query) assert ( - "References to the current dir are not allowed within the path: ./test" - in str(excinfo.value) + "References to the current dir are not allowed within the path: ./test" + in str(excinfo.value) ) query = """{ @@ -339,8 +339,8 @@ def test_wrong_paths(): with pytest.raises(Exception) as excinfo: client.query(query) assert ( - "The path to the graph contains a subpath to an existing graph: test/second/internal/graph1" - in str(excinfo.value) + "The path to the graph contains a subpath to an existing graph: test/second/internal/graph1" + in str(excinfo.value) ) @@ -441,8 +441,8 @@ def test_namespace_listing_does_not_load_each_graph(): graph_props segment).""" n_graphs = 200 work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() g = Graph() g.add_node(1, "alice", {"role": "engineer"}) diff --git a/raphtory-graphql/src/cache.rs b/raphtory-graphql/src/cache.rs index c632f46e4b..1a587c480d 100644 --- a/raphtory-graphql/src/cache.rs +++ b/raphtory-graphql/src/cache.rs @@ -3,30 +3,19 @@ use crate::{ graph::GraphWithVectors, paths::ValidGraphPaths, rayon::{blocking_compute, EVICT_POOL}, - GQLError, }; -use ahash::HashMap; -use dashmap::{DashMap, Entry}; use quick_cache::{ - sync::{Cache, Drain, EntryAction, EntryResult}, - DefaultHashBuilder, Lifecycle, UnitWeighter, Weighter, -}; -use raphtory::{ - db::api::{storage::storage::PersistenceStrategy, view::internal::InternalStorageOps}, - prelude::AdditionOps, + sync::{Cache, EntryAction, EntryResult}, + DefaultHashBuilder, Lifecycle, UnitWeighter, }; +use raphtory::prelude::AdditionOps; use raphtory_storage::core_ops::CoreGraphOps; -use std::{future::Future, marker::PhantomData, sync::Arc}; -use tokio::{join, sync::Notify}; +use std::future::Future; use tracing::{debug, error}; #[derive(Default, Copy, Clone)] pub struct ArcPinned; -pub struct CacheShard { - cache: Cache, -} - fn flush_graph(val: GraphWithVectors) -> () { val.set_flushing(true); val.set_dirty(false); // make sure this is reset before the flush so any mutation that gets triggered afterwards will set the graph back to dirty diff --git a/raphtory-graphql/src/data.rs b/raphtory-graphql/src/data.rs index 8ffb92d221..a75b13f66c 100644 --- a/raphtory-graphql/src/data.rs +++ b/raphtory-graphql/src/data.rs @@ -20,7 +20,6 @@ use crate::{ }; use async_graphql::Context; use dynamic_graphql::Enum; -use futures_util::FutureExt; use raphtory::{ db::{ api::{ diff --git a/raphtory-graphql/src/model/graph/mutable_graph.rs b/raphtory-graphql/src/model/graph/mutable_graph.rs index 6778097437..f5f429bc65 100644 --- a/raphtory-graphql/src/model/graph/mutable_graph.rs +++ b/raphtory-graphql/src/model/graph/mutable_graph.rs @@ -23,7 +23,6 @@ use raphtory_api::core::storage::arc_str::OptionAsStr; use std::{ error::Error, fmt::{Debug, Display, Formatter}, - sync::Arc, }; #[derive(Debug)] diff --git a/raphtory/src/db/api/storage/graph/storage_ops/property_schema.rs b/raphtory/src/db/api/storage/graph/storage_ops/property_schema.rs index 99e88ecc4e..94f44793bc 100644 --- a/raphtory/src/db/api/storage/graph/storage_ops/property_schema.rs +++ b/raphtory/src/db/api/storage/graph/storage_ops/property_schema.rs @@ -1,11 +1,9 @@ +use super::GraphStorage; use crate::db::api::{ properties::internal::{EdgePropertySchemaOps, NodePropertySchemaOps}, view::BoxedLIter, }; use raphtory_api::{core::storage::arc_str::ArcStr, iter::IntoDynBoxed}; -use raphtory_storage::core_ops::CoreGraphOps; - -use super::GraphStorage; impl NodePropertySchemaOps for GraphStorage { fn node_visible_temporal_prop_ids(&self) -> BoxedLIter<'_, usize> { diff --git a/raphtory/src/db/api/storage/storage.rs b/raphtory/src/db/api/storage/storage.rs index 273acd3589..e145f472ae 100644 --- a/raphtory/src/db/api/storage/storage.rs +++ b/raphtory/src/db/api/storage/storage.rs @@ -383,12 +383,10 @@ impl InheritViewOps for Storage {} #[derive(Clone)] pub struct StorageWriteSession<'a> { session: UnlockedSession<'a>, - storage: &'a Storage, } pub struct AtomicAddEdgeSession<'a> { session: AtomicAddEdge<'a, Extension>, - storage: &'a Storage, } impl EdgeWriteLock for AtomicAddEdgeSession<'_> { @@ -538,10 +536,7 @@ impl InternalAdditionOps for Storage { fn write_session(&self) -> Result, Self::Error> { let session = self.graph.write_session()?; - Ok(StorageWriteSession { - session, - storage: self, - }) + Ok(StorageWriteSession { session }) } fn atomic_add_edge( @@ -551,10 +546,7 @@ impl InternalAdditionOps for Storage { e_id: Option, ) -> Result, Self::Error> { let session = self.graph.atomic_add_edge(src, dst, e_id)?; - Ok(AtomicAddEdgeSession { - session, - storage: self, - }) + Ok(AtomicAddEdgeSession { session }) } fn internal_add_node( diff --git a/raphtory/src/graph_loader/mod.rs b/raphtory/src/graph_loader/mod.rs index 0234660bdb..586e994227 100644 --- a/raphtory/src/graph_loader/mod.rs +++ b/raphtory/src/graph_loader/mod.rs @@ -103,7 +103,7 @@ use std::{ path::{Path, PathBuf}, time::Duration, }; -use tempfile::{NamedTempFile, PersistError}; +use tempfile::NamedTempFile; use zip::read::ZipArchive; pub mod company_house; From a6d019f7b970966d681c127e93cdfb71c931a1af Mon Sep 17 00:00:00 2001 From: Lucas Jeub Date: Thu, 14 May 2026 13:51:59 +0200 Subject: [PATCH 09/24] add function to look up port on server --- raphtory-graphql/src/python/server/running_server.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/raphtory-graphql/src/python/server/running_server.rs b/raphtory-graphql/src/python/server/running_server.rs index 7b464d0bdd..62a93a758a 100644 --- a/raphtory-graphql/src/python/server/running_server.rs +++ b/raphtory-graphql/src/python/server/running_server.rs @@ -101,6 +101,11 @@ impl PyRunningGraphServer { }) } + /// Get the port the server is listening on + pub fn port(&self) -> PyResult { + self.apply_if_alive(|handler| Ok(handler.port)) + } + /// Stop the server and wait for it to finish. /// /// Returns: From 2074443da28b55811d61b5333e73de4a767f0782 Mon Sep 17 00:00:00 2001 From: Lucas Jeub Date: Thu, 14 May 2026 14:24:44 +0200 Subject: [PATCH 10/24] make the cli port behaviour consistent --- raphtory-graphql/src/cli.rs | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/raphtory-graphql/src/cli.rs b/raphtory-graphql/src/cli.rs index 5f303ddeed..36f009c1a5 100644 --- a/raphtory-graphql/src/cli.rs +++ b/raphtory-graphql/src/cli.rs @@ -24,14 +24,14 @@ use raphtory::db::api::storage::storage::Config; use std::path::PathBuf; use tokio::io::Result as IoResult; -#[derive(Parser)] +#[derive(Parser, Debug)] #[command(name = "raphtory", about = "Raphtory CLI", version = raphtory::version())] struct Args { #[command(subcommand)] command: Commands, } -#[derive(Subcommand)] +#[derive(Subcommand, Debug)] enum Commands { #[command(about = "Run the GraphQL server")] Server(ServerArgs), @@ -39,7 +39,7 @@ enum Commands { Schema, } -#[derive(clap::Args)] +#[derive(clap::Args, Debug)] struct ServerArgs { #[arg( long, @@ -49,8 +49,8 @@ struct ServerArgs { )] work_dir: PathBuf, - #[arg(long, env = "RAPHTORY_PORT", default_value_t = DEFAULT_PORT, help = "Port for Raphtory to run on")] - port: u16, + #[arg(long, env = "RAPHTORY_PORT", help = "Port for Raphtory to run on")] + port: Option, #[arg(long, env = "RAPHTORY_CACHE_CAPACITY", default_value_t = DEFAULT_CAPACITY, help = "Cache capacity")] cache_capacity: u64, @@ -234,7 +234,14 @@ where .await?; let server = apply_server_extension(server, server_args.permissions_store_path.as_deref()); - server.run_with_port(server_args.port).await?; + match server_args.port { + None => { + server.run().await?; + } + Some(port) => { + server.run_with_port(port).await?; + } + } } } Ok(()) From f55cca47566bfcd93ff52fdbcd9a3604cbc81904 Mon Sep 17 00:00:00 2001 From: Lucas Jeub Date: Thu, 14 May 2026 16:28:37 +0200 Subject: [PATCH 11/24] enable panic on drop errors for tests --- db4-storage/Cargo.toml | 5 +++-- db4-storage/src/pages/mod.rs | 29 ++++++++++++++++++++++++----- python/Cargo.toml | 1 + python/tox.ini | 4 +++- raphtory/Cargo.toml | 8 +++++++- 5 files changed, 38 insertions(+), 9 deletions(-) diff --git a/db4-storage/Cargo.toml b/db4-storage/Cargo.toml index fc03b2f476..3cf61c6661 100644 --- a/db4-storage/Cargo.toml +++ b/db4-storage/Cargo.toml @@ -46,5 +46,6 @@ rayon.workspace = true test-log.workspace = true [features] -test-utils = ["dep:proptest", "dep:chrono"] -default = ["test-utils"] +test-utils = ["dep:proptest", "dep:chrono", "panic-on-drop"] +panic-on-drop = [] + diff --git a/db4-storage/src/pages/mod.rs b/db4-storage/src/pages/mod.rs index 7fa0ab7114..1752db4526 100644 --- a/db4-storage/src/pages/mod.rs +++ b/db4-storage/src/pages/mod.rs @@ -42,8 +42,21 @@ pub mod session; #[cfg(any(test, feature = "test-utils"))] pub mod test_utils; -// graph // (node/edges) // segment // layer_ids (0, 1, 2, ...) // actual graphy bits +#[cfg(any(test, feature = "panic-on-drop"))] +macro_rules! drop_error { + ($($arg:tt)*) => {{ + panic!($($arg)*) + }}; +} + +#[cfg(not(any(test, feature = "panic-on-drop")))] +macro_rules! drop_error { + ($($arg:tt)*) => {{ + eprintln!($($arg)*) + }}; +} +// graph // (node/edges) // segment // layer_ids (0, 1, 2, ...) // actual graphy bits #[derive(Debug)] pub struct GraphStore< NS: NodeSegmentOps, @@ -372,7 +385,7 @@ impl< let checkpoint_lsn = match wal.log_shutdown_checkpoint() { Ok(lsn) => lsn, Err(err) => { - eprintln!("Failed to log shutdown checkpoint in drop: {err}"); + drop_error!("Failed to log shutdown checkpoint in drop: {err}"); return; } }; @@ -381,7 +394,7 @@ impl< let flush_lsn = wal.position(); if let Err(err) = wal.flush(flush_lsn) { - eprintln!("Failed to flush checkpoint record in drop: {err}"); + drop_error!("Failed to flush checkpoint record in drop: {err}"); return; } @@ -390,12 +403,12 @@ impl< control_file.set_db_state(DBState::Shutdown); if let Err(err) = control_file.save() { - eprintln!("Failed to save control file in drop: {err}"); + drop_error!("Failed to save control file in drop: {err}"); return; } } Err(err) => { - eprintln!("Failed to flush storage in drop: {err}") + drop_error!("Failed to flush storage in drop: {err}"); } } } @@ -458,4 +471,10 @@ mod test { assert_eq!(actual, expected); } + + #[test] + #[should_panic] + fn test_drop_error() { + drop_error!("failed"); + } } diff --git a/python/Cargo.toml b/python/Cargo.toml index 3fcef1949a..9a1a64579a 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -34,6 +34,7 @@ clam-core = { workspace = true, features = ["python"] } extension-module = ["pyo3/extension-module"] search = ["raphtory/search", "raphtory-graphql/search"] proto = ["raphtory/proto"] +test = ["raphtory/panic-on-drop", "extension-module"] [build-dependencies] diff --git a/python/tox.ini b/python/tox.ini index de2ce2de53..d600935672 100644 --- a/python/tox.ini +++ b/python/tox.ini @@ -18,6 +18,8 @@ pass_env = [testenv:.pkg] pass_env = MATURIN_PEP517_ARGS +set_env = + MATURIN_PEP517_ARGS="--features=test" [testenv:search] wheel_build_env = .pkg_search @@ -25,7 +27,7 @@ commands = pytest {posargs} {tty:--color=yes} tests/test_search [testenv:.pkg_search] set_env = - MATURIN_PEP517_ARGS="--features=search,extension-module" + MATURIN_PEP517_ARGS="--features=search,test" [testenv:export] diff --git a/raphtory/Cargo.toml b/raphtory/Cargo.toml index 1629456088..13028b3f55 100644 --- a/raphtory/Cargo.toml +++ b/raphtory/Cargo.toml @@ -187,5 +187,11 @@ proto = [ test-utils = [ "dep:proptest", - "dep:proptest-derive" + "dep:proptest-derive", + "storage/test-utils", + "panic-on-drop" +] + +panic-on-drop = [ + "storage/panic-on-drop" ] From 2012d8689340c2621c72e7967de2e5a4d45cff30 Mon Sep 17 00:00:00 2001 From: Lucas Jeub Date: Thu, 14 May 2026 16:29:41 +0200 Subject: [PATCH 12/24] make sure graph is dropped before replacing it --- raphtory-graphql/src/cache.rs | 26 ++++++++++------- raphtory-graphql/src/data.rs | 10 +++---- raphtory-graphql/src/paths.rs | 29 ++++++++++++------- .../src/python/server/running_server.rs | 20 ------------- raphtory-graphql/src/python/server/server.rs | 9 +++++- 5 files changed, 46 insertions(+), 48 deletions(-) diff --git a/raphtory-graphql/src/cache.rs b/raphtory-graphql/src/cache.rs index 1a587c480d..d607d860f2 100644 --- a/raphtory-graphql/src/cache.rs +++ b/raphtory-graphql/src/cache.rs @@ -114,33 +114,37 @@ impl GraphCache { } /// Insert a new item into the cache, replacing an existing item if it exists - pub async fn insert_with( + /// The old item is dropped before the closure to create the new graph is invoked + pub async fn insert_with( &self, key: &str, - with: impl Future>, + with: impl FnOnce() -> F, ) -> Result<(), InsertionError> where + F: Future>, InsertionError: From, { - let new_graph = with.await?; let cache_guard = self .cache .entry_async(key, |key, value| EntryAction::<()>::ReplaceWithGuard) .await; - match cache_guard { + let guard = match cache_guard { EntryResult::Replaced(guard, old_graph) => { drop(old_graph); - guard.insert(new_graph) + guard } - EntryResult::Vacant(guard) => guard.insert(new_graph), + EntryResult::Vacant(guard) => guard, _ => { unreachable!() } - } - .map_err(|_| InsertionError::Insertion { - graph: key.to_string(), - error: MutationErrorInner::CacheReplacementError, - })?; + }; + let new_graph = with().await?; + guard + .insert(new_graph) + .map_err(|_| InsertionError::Insertion { + graph: key.to_string(), + error: MutationErrorInner::CacheReplacementError, + })?; Ok(()) } diff --git a/raphtory-graphql/src/data.rs b/raphtory-graphql/src/data.rs index a75b13f66c..dea38257d6 100644 --- a/raphtory-graphql/src/data.rs +++ b/raphtory-graphql/src/data.rs @@ -237,15 +237,15 @@ impl Data { let key = writeable_folder.local_path().to_owned(); let config = self.graph_conf.clone(); self.cache - .insert_with( - &key, + .insert_with(&key, || { blocking_compute(move || { - writeable_folder.write_graph_data(graph.clone(), config)?; + let is_dirty = writeable_folder.write_graph_data(graph.clone(), config)?; let folder = writeable_folder.finish()?; let graph = GraphWithVectors::new(graph, None, folder.as_existing()?); + graph.set_dirty(is_dirty); Ok::<_, InsertionError>(graph) - }), - ) + }) + }) .await?; Ok(()) } diff --git a/raphtory-graphql/src/paths.rs b/raphtory-graphql/src/paths.rs index b3f8093663..6db06d9f66 100644 --- a/raphtory-graphql/src/paths.rs +++ b/raphtory-graphql/src/paths.rs @@ -1,4 +1,6 @@ -use crate::{data::DIRTY_PATH, model::blocking_io, rayon::blocking_compute}; +use crate::{ + data::DIRTY_PATH, graph::GraphWithVectors, model::blocking_io, rayon::blocking_compute, +}; use futures_util::io; use raphtory::{ db::api::{ @@ -6,7 +8,7 @@ use raphtory::{ view::{internal::InternalStorageOps, MaterializedGraph}, }, errors::{GraphError, InvalidPathReason}, - prelude::GraphViewOps, + prelude::{AdditionOps, GraphViewOps}, serialise::{ metadata::GraphMetadata, GraphFolder, GraphPaths, RelativePath, StableDecode, WriteableGraphFolder, ROOT_META_PATH, @@ -345,31 +347,35 @@ impl ValidWriteableGraphFolder { Self::new(path, relative_path) } + /// write graph data to folder (returns a flag to indicate if the graph should be considered dirty) fn write_graph_data_inner( &self, graph: MaterializedGraph, config: Config, - ) -> Result<(), InternalPathValidationError> { - if Extension::disk_storage_enabled() { + ) -> Result { + let is_dirty = if Extension::disk_storage_enabled() { let graph_path = self.graph_folder().graph_path()?; if graph .disk_storage_path() .is_some_and(|path| path == &graph_path) { self.global_path.write_metadata(&graph)?; + true } else { graph.materialize_at_with_config(self.graph_folder(), config)?; + true } } else { self.global_path.data_path()?.replace_graph(graph)?; - } - Ok(()) + false + }; + Ok(is_dirty) } pub fn write_graph_data( &self, graph: MaterializedGraph, config: Config, - ) -> Result<(), PathValidationError> { + ) -> Result { self.write_graph_data_inner(graph, config) .with_path(self.local_path()) } @@ -390,16 +396,17 @@ impl ValidWriteableGraphFolder { config: Config, ) -> Result<(), PathValidationError> { self.with_internal_errors(|| { - if Extension::disk_storage_enabled() { + let is_dirty = if Extension::disk_storage_enabled() { MaterializedGraph::decode_from_zip_at( ZipArchive::new(bytes)?, self.graph_folder(), config, - )?; + )? + .flush()?; } else { self.global_path.data_path()?.unzip_to_folder(bytes)?; - } - Ok::<(), GraphError>(()) + }; + Ok::<_, GraphError>(is_dirty) }) } diff --git a/raphtory-graphql/src/python/server/running_server.rs b/raphtory-graphql/src/python/server/running_server.rs index 62a93a758a..27ebb56bb2 100644 --- a/raphtory-graphql/src/python/server/running_server.rs +++ b/raphtory-graphql/src/python/server/running_server.rs @@ -51,26 +51,6 @@ impl PyRunningGraphServer { } } - pub(crate) fn wait_for_server_online(&self, url: &String, timeout_ms: u64) -> PyResult<()> { - let num_intervals = timeout_ms / WAIT_CHECK_INTERVAL_MILLIS; - for _ in 0..num_intervals { - let join_handle = &self.server_handler.as_ref().unwrap().join_handle; - if join_handle.is_finished() { - // this error will never be presented to the user, the result coming from the server task will instead - return Err(PyException::new_err("Server task finished too early")); - } - if is_online(url) { - return Ok(()); - } else { - sleep(Duration::from_millis(WAIT_CHECK_INTERVAL_MILLIS)) - } - } - Err(PyException::new_err(format!( - "Failed to start server in {} milliseconds", - timeout_ms - ))) - } - pub(crate) fn stop_server(&mut self, py: Python) -> PyResult<()> { Self::apply_if_alive(self, |handler| { match handler.sender.send(BridgeCommand::StopServer) { diff --git a/raphtory-graphql/src/python/server/server.rs b/raphtory-graphql/src/python/server/server.rs index 4e25f87cdc..a8c23c50d5 100644 --- a/raphtory-graphql/src/python/server/server.rs +++ b/raphtory-graphql/src/python/server/server.rs @@ -352,7 +352,14 @@ impl PyGraphServer { .recv_timeout(Duration::from_millis(timeout_ms)) .map_err(|err| { let _ = sender.try_send(BridgeCommand::StopServer); // best effort cleanup - PyRuntimeError::new_err("Failed to start server") + match err { + RecvTimeoutError::Timeout => PyRuntimeError::new_err(format!( + "Failed to start server in {timeout_ms} milliseconds" + )), + RecvTimeoutError::Disconnected => { + PyRuntimeError::new_err("Failed to start server") + } + } })?; let server = PyRunningGraphServer::new(join_handle, sender, port)?; Ok(server) From 25784edab181a4d4d65105a2717aa7abca43e05f Mon Sep 17 00:00:00 2001 From: Lucas Jeub Date: Thu, 14 May 2026 16:30:02 +0200 Subject: [PATCH 13/24] update tests more tests so they work with arbitrary ports --- .../edit_graph/test_copy_graph.py | 32 +++++++++---------- .../edit_graph/test_delete_graph.py | 16 +++++----- .../edit_graph/test_send_graph.py | 24 +++++++------- 3 files changed, 36 insertions(+), 36 deletions(-) diff --git a/python/tests/test_base_install/test_graphql/edit_graph/test_copy_graph.py b/python/tests/test_base_install/test_graphql/edit_graph/test_copy_graph.py index 734e08cce9..574292f787 100644 --- a/python/tests/test_base_install/test_graphql/edit_graph/test_copy_graph.py +++ b/python/tests/test_base_install/test_graphql/edit_graph/test_copy_graph.py @@ -9,8 +9,8 @@ def test_copy_graph_fails_if_graph_not_found(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() query = """mutation { copyGraph( @@ -34,8 +34,8 @@ def test_copy_graph_fails_if_graph_with_same_name_already_exists(): g.save_to_file(os.path.join(work_dir, "ben", "g5")) g.save_to_file(os.path.join(work_dir, "g6")) - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() query = """mutation { copyGraph( @@ -59,8 +59,8 @@ def test_copy_graph_fails_if_graph_with_same_name_already_exists_at_same_namespa g.save_to_file(os.path.join(work_dir, "ben", "g5")) g.save_to_file(os.path.join(work_dir, "ben", "g6")) - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() query = """mutation { copyGraph( @@ -85,8 +85,8 @@ def test_copy_graph_fails_if_graph_with_same_name_already_exists_at_diff_namespa g.save_to_file(os.path.join(work_dir, "ben", "g5")) g.save_to_file(os.path.join(work_dir, "shivam", "g6")) - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() query = """mutation { copyGraph( @@ -109,8 +109,8 @@ def test_copy_graph_succeeds(): os.makedirs(os.path.join(work_dir, "shivam"), exist_ok=True) g.save_to_file(os.path.join(work_dir, "shivam", "g3")) - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() # Assert if copy graph succeeds and old graph is retained query = """mutation { @@ -151,8 +151,8 @@ def test_copy_graph_using_client_api_succeeds(): os.makedirs(os.path.join(work_dir, "shivam"), exist_ok=True) g.save_to_file(os.path.join(work_dir, "shivam", "g3")) - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() # Assert if copy graph succeeds and old graph is retained client.copy_graph("shivam/g3", "ben/g4") @@ -188,8 +188,8 @@ def test_copy_graph_succeeds_at_same_namespace_as_graph(): g.save_to_file(os.path.join(work_dir, "shivam", "g3")) - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() # Assert if rename graph succeeds and old graph is deleted query = """mutation { @@ -232,8 +232,8 @@ def test_copy_graph_succeeds_at_diff_namespace_as_graph(): g.save_to_file(os.path.join(work_dir, "ben", "g3")) - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() # Assert if rename graph succeeds and old graph is deleted query = """mutation { diff --git a/python/tests/test_base_install/test_graphql/edit_graph/test_delete_graph.py b/python/tests/test_base_install/test_graphql/edit_graph/test_delete_graph.py index 7b74574344..9b6667b955 100644 --- a/python/tests/test_base_install/test_graphql/edit_graph/test_delete_graph.py +++ b/python/tests/test_base_install/test_graphql/edit_graph/test_delete_graph.py @@ -8,8 +8,8 @@ def test_delete_graph_fails_if_graph_not_found(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() query = """mutation { deleteGraph( @@ -23,8 +23,8 @@ def test_delete_graph_fails_if_graph_not_found(): def test_delete_graph_succeeds_if_graph_found(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() g = Graph() g.add_edge(1, "ben", "hamza") @@ -47,8 +47,8 @@ def test_delete_graph_succeeds_if_graph_found(): def test_delete_graph_using_client_api_succeeds_if_graph_found(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() g = Graph() g.add_edge(1, "ben", "hamza") @@ -66,8 +66,8 @@ def test_delete_graph_using_client_api_succeeds_if_graph_found(): def test_delete_graph_succeeds_if_graph_found_at_namespace(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() g = Graph() g.add_edge(1, "ben", "hamza") diff --git a/python/tests/test_base_install/test_graphql/edit_graph/test_send_graph.py b/python/tests/test_base_install/test_graphql/edit_graph/test_send_graph.py index 41a469f31f..329d5892fd 100644 --- a/python/tests/test_base_install/test_graphql/edit_graph/test_send_graph.py +++ b/python/tests/test_base_install/test_graphql/edit_graph/test_send_graph.py @@ -13,8 +13,8 @@ def test_send_graph_succeeds_if_no_graph_found_with_same_name(): g.add_edge(3, "ben", "haaroon") tmp_work_dir = tempfile.mkdtemp() - with GraphServer(tmp_work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() client.send_graph(path="g", graph=g) @@ -27,8 +27,8 @@ def test_send_graph_fails_if_graph_already_exists(): g.add_edge(3, "ben", "haaroon") g.save_to_file(os.path.join(tmp_work_dir, "g")) - with GraphServer(tmp_work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() with pytest.raises(Exception) as excinfo: client.send_graph(path="g", graph=g) assert "Graph 'g' already exists" in str(excinfo.value) @@ -42,8 +42,8 @@ def test_send_graph_succeeds_if_graph_already_exists_with_overwrite_enabled(): g.add_edge(2, "haaroon", "hamza") g.add_edge(3, "ben", "haaroon") - with GraphServer(tmp_work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() client.send_graph(path="g", graph=g) @@ -76,8 +76,8 @@ def test_send_graph_succeeds_if_no_graph_found_with_same_name_at_namespace(): g.add_edge(3, "ben", "haaroon") tmp_work_dir = tempfile.mkdtemp() - with GraphServer(tmp_work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() client.send_graph(path="shivam/g", graph=g) @@ -91,8 +91,8 @@ def test_send_graph_fails_if_graph_already_exists_at_namespace(): os.makedirs(os.path.join(tmp_work_dir, "shivam"), exist_ok=True) g.save_to_file(os.path.join(tmp_work_dir, "shivam", "g")) - with GraphServer(tmp_work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() with pytest.raises(Exception) as excinfo: client.send_graph(path="shivam/g", graph=g) assert "Graph 'shivam/g' already exists" in str(excinfo.value) @@ -108,8 +108,8 @@ def test_send_graph_succeeds_if_graph_already_exists_at_namespace_with_overwrite os.makedirs(os.path.join(tmp_work_dir, "shivam"), exist_ok=True) g.save_to_file(os.path.join(tmp_work_dir, "shivam", "g")) - with GraphServer(tmp_work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() g = Graph() g.add_edge(1, "ben", "hamza") From 172b89a9702254f31c18a04a0b65bf18700645e4 Mon Sep 17 00:00:00 2001 From: Lucas Jeub Date: Thu, 14 May 2026 17:49:16 +0200 Subject: [PATCH 14/24] make the python tests work even if there is a raphtory server running --- docs/user-guide/graphql/2_run-server.md | 27 +- docs/user-guide/graphql/3_writing-queries.md | 262 +++++++++++------- python/tests/test_auth.py | 108 ++++---- .../test_graphql/edit_graph/test_graphql.py | 74 ++--- .../edit_graph/test_move_graph.py | 32 +-- .../test_graphql/edit_graph/test_new_graph.py | 16 +- .../edit_graph/test_upload_graph.py | 28 +- .../test_graphql/misc/test_components.py | 4 +- .../test_graphql/misc/test_index_off.py | 8 +- .../test_graphql/misc/test_latest.py | 4 +- .../test_graphql/misc/test_map_props.py | 7 +- .../test_graphql/misc/test_snapshot.py | 4 +- .../test_graphql/misc/test_tracing.py | 8 +- .../test_graph_file_time_stats.py | 10 +- .../test_graphql/test_metadata_dispatch.py | 36 ++- .../test_graphql/test_misc.py | 4 +- .../test_graphql/test_namespace.py | 20 +- .../test_graphql/test_read_only_load.py | 32 +-- .../test_graphql/test_schema.py | 4 +- .../test_graphql/test_server_flags.py | 75 ++--- .../update_graph/test_batch_updates.py | 8 +- .../update_graph/test_edge_updates.py | 16 +- .../update_graph/test_graph_updates.py | 24 +- .../update_graph/test_node_updates.py | 16 +- .../tests/test_search/test_gql_index_spec.py | 5 +- .../test_vectors/test_graphql_vectors.py | 16 +- 26 files changed, 460 insertions(+), 388 deletions(-) diff --git a/docs/user-guide/graphql/2_run-server.md b/docs/user-guide/graphql/2_run-server.md index dca048d36d..33134c6998 100644 --- a/docs/user-guide/graphql/2_run-server.md +++ b/docs/user-guide/graphql/2_run-server.md @@ -8,11 +8,14 @@ Before reading this topic, please ensure you are familiar with: ## Saving your Raphtory graph into a directory -You will need some test data to complete the following examples. This can be your own data or one of the examples in the Raphtory documentation. +You will need some test data to complete the following examples. This can be your own data or one of the examples in the +Raphtory documentation. -Once your data is loaded into a Raphtory graph, the graph needs to be saved into your working directory. This can be done with the following code, where `g` is your graph: +Once your data is loaded into a Raphtory graph, the graph needs to be saved into your working directory. This can be +done with the following code, where `g` is your graph: /// tab | :fontawesome-brands-python: Python + ```{.python notest} import os working_dir = "graphs/" @@ -21,6 +24,7 @@ if not os.path.exists(working_dir): os.makedirs(working_dir) g.save_to_file(working_dir + "your_graph") ``` + /// ## Starting a server @@ -39,24 +43,31 @@ This option is the simplist and provides the most configuration options. ### Start a server in Python -If you have a [`GraphServer`][raphtory.graphql.GraphServer] object you can use either the [`.run()`][raphtory.graphql.GraphServer.run] or [`.start()`][raphtory.graphql.GraphServer.start] functions to start a GraphQL sever and Raphtory UI. +If you have a [`GraphServer`][raphtory.graphql.GraphServer] object you can use either the [ +`.run()`][raphtory.graphql.GraphServer.run] or [`.start()`][raphtory.graphql.GraphServer.start] functions to start a +GraphQL sever and Raphtory UI. -Below is an example of how to start the server and send a Raphtory graph to the server, where `new_graph` is your Raphtory graph object. +Below is an example of how to start the server and send a Raphtory graph to the server, where `new_graph` is your +Raphtory graph object. /// tab | :fontawesome-brands-python: Python + ```{.python notest} tmp_work_dir = tempfile.mkdtemp() -with GraphServer(tmp_work_dir, tracing=True).start(): - client = RaphtoryClient("http://localhost:1736") +with GraphServer(tmp_work_dir, tracing=True).start() as server: + client = server.get_client() client.send_graph(path="g", graph=new_graph) query = """{graph(path: "g") {nodes {list {name}}}}""" client.query(query) ``` + /// You can set the port in `RaphtoryClient()` to the port the GraphQL server should run on. -The `path` parameter is always the graph in your server that you would like to read or update. So in this example, we want to send `new_graph` to graph `g` on the server to update it. +The `path` parameter is always the graph in your server that you would like to read or update. So in this example, we +want to send `new_graph` to graph `g` on the server to update it. -The `graph` parameter is set to the Raphtory graph that you would like to send. An additional `overwrite` parameter can be stated if we want this new graph to overwrite the old graph. +The `graph` parameter is set to the Raphtory graph that you would like to send. An additional `overwrite` parameter can +be stated if we want this new graph to overwrite the old graph. diff --git a/docs/user-guide/graphql/3_writing-queries.md b/docs/user-guide/graphql/3_writing-queries.md index 26c18282e4..f8488bd057 100644 --- a/docs/user-guide/graphql/3_writing-queries.md +++ b/docs/user-guide/graphql/3_writing-queries.md @@ -2,13 +2,18 @@ The GraphQL API largely follows the same patterns as the Python API but has a few key differences. -In GraphQL, there are two different types of requests: a query to search through your data or a mutation of your data. Only the top-level fields in mutation operations are allowed to cause side effects. To accommodate this, in the Raphtory API you can make queries to graphs or metagraphs but must make changes using a mutable graph, node or edge. +In GraphQL, there are two different types of requests: a query to search through your data or a mutation of your data. +Only the top-level fields in mutation operations are allowed to cause side effects. To accommodate this, in the Raphtory +API you can make queries to graphs or metagraphs but must make changes using a mutable graph, node or edge. -This division means that the distinction between Graphs and GraphViews is less important in GraphQL and all non-mutable Graph endpoints are GraphViews while MutableGraphs are used for mutation operations. This is also true for Nodes and Edges and their respective views. Graphs can be further distinguished as either `PERSISTENT` or `EVENT` types. +This division means that the distinction between Graphs and GraphViews is less important in GraphQL and all non-mutable +Graph endpoints are GraphViews while MutableGraphs are used for mutation operations. This is also true for Nodes and +Edges and their respective views. Graphs can be further distinguished as either `PERSISTENT` or `EVENT` types. ## Graphical playground -When you start a GraphQL server, you can find your GraphQL UI in the browser at `localhost:1736/playground` or an alternative port if you specified one. +When you start a GraphQL server, you can find your GraphQL UI in the browser at `localhost:1736/playground` or an +alternative port if you specified one. An annotated schema is available from the documentation tab in the left hand menu of the playground. @@ -21,6 +26,7 @@ Here are some example queries to get you started: ### List of all the nodes /// tab | ![GraphQL](https://img.icons8.com/ios-filled/15/graphql.png) GraphQL + ``` query { graph(path: "your_graph") { @@ -32,6 +38,7 @@ query { } } ``` + /// ### List of all the edges, with specific node properties @@ -39,6 +46,7 @@ query { To find nodes with `age`: /// tab | ![GraphQL](https://img.icons8.com/ios-filled/15/graphql.png) GraphQL + ``` query { graph(path: "your_graph") { @@ -65,6 +73,7 @@ query { } } ``` + /// This will return something like: @@ -107,6 +116,7 @@ All the queries that can be done in Python can also be done in GraphQL. Here is an example: /// tab | ![GraphQL](https://img.icons8.com/ios-filled/15/graphql.png) GraphQL + ``` graphql query { graph(path: "your_graph") { @@ -120,20 +130,23 @@ query { } } ``` + /// /// tab | :fontawesome-brands-python: Python + ```{.python notest} g.node("Ben").properties.get("age") ``` + /// ### Examine the metadata of a node Metadata does not change over the lifetime of an object. You can request it with a query like the following: - /// tab | ![GraphQL](https://img.icons8.com/ios-filled/15/graphql.png) GraphQL + ``` { graph(path: "traffic_graph") { @@ -151,67 +164,69 @@ Metadata does not change over the lifetime of an object. You can request it with } } ``` + /// Which will return something like: !!! Output - ```json - { - "data": { - "graph": { - "nodes": { - "list": [ - { - "name": "ServerA", - "metadata": { - "values": [ - { - "key": "datasource", - "value": "network_traffic_edges.csv" - }, - { - "key": "server_name", - "value": "Alpha" - }, - { - "key": "hardware_type", - "value": "Blade Server" - } - ] - } - }, - { - "name": "ServerB", - "metadata": { - "values": [ - { - "key": "datasource", - "value": "network_traffic_edges.csv" - }, - { - "key": "server_name", - "value": "Beta" - }, - { - "key": "hardware_type", - "value": "Rack Server" - } - ] +```json +{ +"data": { + "graph": { + "nodes": { + "list": [ + { + "name": "ServerA", + "metadata": { + "values": [ + { + "key": "datasource", + "value": "network_traffic_edges.csv" + }, + { + "key": "server_name", + "value": "Alpha" + }, + { + "key": "hardware_type", + "value": "Blade Server" } + ] } + }, + { + "name": "ServerB", + "metadata": { + "values": [ + { + "key": "datasource", + "value": "network_traffic_edges.csv" + }, + { + "key": "server_name", + "value": "Beta" + }, + { + "key": "hardware_type", + "value": "Rack Server" + } ] + } } - } + ] } } - ``` +} +} +``` ### Examine the properties of a node Properties can change over time so it is often useful to make a query for a specific time or window. /// tab | ![GraphQL](https://img.icons8.com/ios-filled/15/graphql.png) GraphQL + ``` { graph(path: "traffic_graph") { @@ -231,114 +246,125 @@ Properties can change over time so it is often useful to make a query for a spec } } ``` + /// Which will return something like: !!! Output - ```json - { - "data": { - "graph": { - "at": { - "nodes": { - "list": [ +```json +{ +"data": { + "graph": { + "at": { + "nodes": { + "list": [ + { + "name": "ServerA", + "properties": { + "values": [] + } + }, + { + "name": "ServerB", + "properties": { + "values": [ { - "name": "ServerA", - "properties": { - "values": [] - } + "key": "OS_version", + "value": "Red Hat 8.1" }, { - "name": "ServerB", - "properties": { - "values": [ - { - "key": "OS_version", - "value": "Red Hat 8.1" - }, - { - "key": "primary_function", - "value": "Web Server" - }, - { - "key": "uptime_days", - "value": 45 - } - ] - } + "key": "primary_function", + "value": "Web Server" }, { - "name": "ServerC", - "properties": { - "values": [] - } + "key": "uptime_days", + "value": 45 } - ] + ] } - } + }, + { + "name": "ServerC", + "properties": { + "values": [] + } + } + ] } } } - ``` +} +} +``` ### Querying GraphQL in Python -You can also send GraphQL queries in Python directl using the [`.query()`][raphtory.graphql.RaphtoryClient.query] function on a `RaphtoryClient`. The following example shows you how to do this: +You can also send GraphQL queries in Python directl using the [`.query()`][raphtory.graphql.RaphtoryClient.query] +function on a `RaphtoryClient`. The following example shows you how to do this: /// tab | :fontawesome-brands-python: Python + ```{.python notest} from raphtory.graphql import GraphServer -with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") +with GraphServer(work_dir).start() as server: + client = server.get_client() query = """{graph(path: "graph") { created lastOpened lastUpdated }}""" result = client.query(query) ``` + /// -Pass your graph object string into the `client.query()` method to execute the GraphQL query and retrieve the result in a python dictionary object. +Pass your graph object string into the `client.query()` method to execute the GraphQL query and retrieve the result in a +python dictionary object. !!! Output - ```output - {'graph': {'created': 1729075008085, 'lastOpened': 1729075036222, 'lastUpdated': 1729075008085}} - ``` +```output +{'graph': {'created': 1729075008085, 'lastOpened': 1729075036222, 'lastUpdated': 1729075008085}} +``` ## Mutation requests You can also mutate your graph. This can be done both in the GraphQL IDE and in Python. -From GraphQL these operations are available from the [Mutation root](../../../reference/graphql/graphql_API/#mutation-mutroot) which operates on mutable objects by specified by a path. +From GraphQL these operations are available from +the [Mutation root](../../../reference/graphql/graphql_API/#mutation-mutroot) which operates on mutable objects by +specified by a path. !!! note - Some methods to mutate the graph are exclusive to Python. +Some methods to mutate the graph are exclusive to Python. ### Sending a graph You can send a graph to the server and overwrite an existing graph if needed. /// tab | :fontawesome-brands-python: Python + ```{.python notest} from raphtory.graphql import GraphServer tmp_work_dir = tempfile.mkdtemp() -with GraphServer(tmp_work_dir).start(): - client = RaphtoryClient("http://localhost:1736") +with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() g = Graph() g.add_edge(1, "bob", "emma") g.add_edge(2, "sally", "tony") client.send_graph(path="g", graph=g, overwrite=True) ``` + /// To check your query: /// tab | :fontawesome-brands-python: Python + ```{.python notest} query = """{graph(path: "g") {nodes {list {name}}}}""" client.query(query) ``` + /// This should return: @@ -365,36 +391,43 @@ This should return: You can retrieve graphs from a "path" on the server which returns a Python Raphtory graph object. /// tab | :fontawesome-brands-python: Python + ```{.python notest} g = client.receive_graph("path/to/graph") g.edge("sally", "tony") ``` + /// ### Creating a new graph This is an example of how to create a new graph in the server. -The first parameter is the path of the graph to be created and the second parameter is the type of graph that should be created, this will either be _EVENT_ or _PERSISTENT_. +The first parameter is the path of the graph to be created and the second parameter is the type of graph that should be +created, this will either be _EVENT_ or _PERSISTENT_. An explanation of the different types of graph can be found [here](../../user-guide/persistent-graph/1_intro.md) /// tab | ![GraphQL](https://img.icons8.com/ios-filled/15/graphql.png) GraphQL + ```graphql mutation { newGraph(path: "new_graph", graphType: PERSISTENT) } ``` + /// /// tab | :fontawesome-brands-python: Python + ```{.python notest} from raphtory.graphql import GraphServer work_dir = tempfile.mkdtemp() -with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") +with GraphServer(work_dir).start() as server: + client = server.get_client() client.new_graph("path/to/new_graph", "EVENT") ``` + /// The returning result to confirm that a new graph has been created: @@ -414,22 +447,26 @@ The returning result to confirm that a new graph has been created: It is possible to move a graph to a new path on the server. /// tab | ![GraphQL](https://img.icons8.com/ios-filled/15/graphql.png) GraphQL + ```graphql mutation { moveGraph(path: "graph", newPath: "new_path") } ``` + /// /// tab | :fontawesome-brands-python: Python + ```{.python notest} from raphtory.graphql import GraphServer work_dir = tempfile.mkdtemp() -with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") +with GraphServer(work_dir).start() as server: + client = server.get_client() client.move_graph("path/to/graph", "path/to/new_path) ``` + /// The returning GraphQL result to confirm that the graph has been moved: @@ -449,22 +486,26 @@ The returning GraphQL result to confirm that the graph has been moved: It is possible to copy a graph to a new path on the server. /// tab | ![GraphQL](https://img.icons8.com/ios-filled/15/graphql.png) GraphQL + ```graphql mutation { copyGraph(path: "graph", newPath: "new_path") } ``` + /// /// tab | :fontawesome-brands-python: Python + ```{.python notest} from raphtory.graphql import GraphServer work_dir = tempfile.mkdtemp() -with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") +with GraphServer(work_dir).start() as server: + client = server.get_client() client.copy_graph("path/to/graph", "path/to/new_path) ``` + /// The returning GraphQL result to confirm that the graph has been copied: @@ -484,22 +525,26 @@ The returning GraphQL result to confirm that the graph has been copied: It is possible to delete a graph on the server. /// tab | ![GraphQL](https://img.icons8.com/ios-filled/15/graphql.png) GraphQL + ```graphql mutation { deleteGraph(path: "graph") } ``` + /// /// tab | :fontawesome-brands-python: Python + ```{.python notest} from raphtory.graphql import GraphServer work_dir = tempfile.mkdtemp() -with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") +with GraphServer(work_dir).start() as server: + client = server.get_client() client.delete_graph("graph") ``` + /// The returning GraphQL result to confirm that the graph has been deleted: @@ -519,22 +564,27 @@ The returning GraphQL result to confirm that the graph has been deleted: It is possible to update the graph using the `remote_graph()` method. /// tab | :fontawesome-brands-python: Python + ```{.python notest} from raphtory.graphql import GraphServer work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() client.new_graph("path/to/event_graph", "EVENT") rg = client.remote_graph("path/to/event_graph") rg.add_edge(1, "sally", "tony", layer="friendship") ``` + /// -Once you have updated the graph, for example by adding an edge, you can receive a graph by using `receive_graph()` and specifying the path of the graph you would like to receive. +Once you have updated the graph, for example by adding an edge, you can receive a graph by using `receive_graph()` and +specifying the path of the graph you would like to receive. /// tab | :fontawesome-brands-python: Python + ```{.python notest} g = client.receive_graph("path/to/event_graph") ``` + /// diff --git a/python/tests/test_auth.py b/python/tests/test_auth.py index 9c6fe3d52a..a1dca760d3 100644 --- a/python/tests/test_auth.py +++ b/python/tests/test_auth.py @@ -17,8 +17,6 @@ MC4CAQAwBQYDK2VwBCIEIFzEcSO/duEjjX4qKxDVy4uLqfmiEIA6bEw1qiPyzTQg -----END PRIVATE KEY-----""" -RAPHTORY = "http://localhost:1736" - READ_JWT = jwt.encode({"access": "ro"}, PRIVATE_KEY, algorithm="EdDSA") READ_HEADERS = { "Authorization": f"Bearer {READ_JWT}", @@ -69,6 +67,10 @@ TEST_QUERIES = [QUERY_NAMESPACES, QUERY_GRAPH, QUERY_ROOT] +def raphtory_url(port: int) -> str: + return f"http://localhost:{port}" + + def assert_successful_response(response: requests.Response): assert "errors" not in response.json() assert type(response.json()["data"]) == dict @@ -76,22 +78,23 @@ def assert_successful_response(response: requests.Response): # TODO: implement this so we can use the with sintax -def add_test_graph(): +def add_test_graph(port): requests.post( - RAPHTORY, headers=WRITE_HEADERS, data=json.dumps({"query": NEW_TEST_GRAPH}) + raphtory_url(port), headers=WRITE_HEADERS, data=json.dumps({"query": NEW_TEST_GRAPH}) ) def test_expired_token(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, auth_public_key=PUB_KEY).start(): + with GraphServer(work_dir, auth_public_key=PUB_KEY).start() as server: + port = server.port() exp = time() - 100 token = jwt.encode({"access": "ro", "exp": exp}, PRIVATE_KEY, algorithm="EdDSA") headers = { "Authorization": f"Bearer {token}", } response = requests.post( - RAPHTORY, headers=headers, data=json.dumps({"query": QUERY_ROOT}) + raphtory_url(port), headers=headers, data=json.dumps({"query": QUERY_ROOT}) ) assert response.status_code == 401 @@ -100,7 +103,7 @@ def test_expired_token(): "Authorization": f"Bearer {token}", } response = requests.post( - RAPHTORY, headers=headers, data=json.dumps({"query": QUERY_ROOT}) + raphtory_url(port), headers=headers, data=json.dumps({"query": QUERY_ROOT}) ) assert response.status_code == 401 @@ -108,17 +111,18 @@ def test_expired_token(): @pytest.mark.parametrize("query", TEST_QUERIES) def test_default_read_access(query): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, auth_public_key=PUB_KEY).start(): - add_test_graph() + with GraphServer(work_dir, auth_public_key=PUB_KEY).start() as server: + port = server.port() + add_test_graph(port) data = json.dumps({"query": query}) - response = requests.post(RAPHTORY, data=data) + response = requests.post(raphtory_url(port), data=data) assert response.status_code == 401 - response = requests.post(RAPHTORY, headers=READ_HEADERS, data=data) + response = requests.post(raphtory_url(port), headers=READ_HEADERS, data=data) assert_successful_response(response) - response = requests.post(RAPHTORY, headers=WRITE_HEADERS, data=data) + response = requests.post(raphtory_url(port), headers=WRITE_HEADERS, data=data) assert_successful_response(response) @@ -126,18 +130,19 @@ def test_default_read_access(query): def test_disabled_read_access(query): work_dir = tempfile.mkdtemp() with GraphServer( - work_dir, auth_public_key=PUB_KEY, require_auth_for_reads=False - ).start(): - add_test_graph() + work_dir, auth_public_key=PUB_KEY, require_auth_for_reads=False + ).start() as server: + port = server.port() + add_test_graph(port) data = json.dumps({"query": query}) - response = requests.post(RAPHTORY, data=data) + response = requests.post(raphtory_url(port), data=data) assert_successful_response(response) - response = requests.post(RAPHTORY, headers=READ_HEADERS, data=data) + response = requests.post(raphtory_url(port), headers=READ_HEADERS, data=data) assert_successful_response(response) - response = requests.post(RAPHTORY, headers=WRITE_HEADERS, data=data) + response = requests.post(raphtory_url(port), headers=WRITE_HEADERS, data=data) assert_successful_response(response) @@ -178,21 +183,22 @@ def test_disabled_read_access(query): @pytest.mark.parametrize("query", [ADD_NODE, ADD_EDGE, ADD_TEMP_PROP, ADD_CONST_PROP]) def test_update_graph(query): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, auth_public_key=PUB_KEY).start(): - add_test_graph() + with GraphServer(work_dir, auth_public_key=PUB_KEY).start() as server: + port = server.port() + add_test_graph(port) data = json.dumps({"query": query}) - response = requests.post(RAPHTORY, data=data) + response = requests.post(raphtory_url(port), data=data) assert response.status_code == 401 - response = requests.post(RAPHTORY, headers=READ_HEADERS, data=data) + response = requests.post(raphtory_url(port), headers=READ_HEADERS, data=data) assert response.json()["data"] is None assert ( - response.json()["errors"][0]["message"] - == "The requested endpoint requires write access" + response.json()["errors"][0]["message"] + == "The requested endpoint requires write access" ) - response = requests.post(RAPHTORY, headers=WRITE_HEADERS, data=data) + response = requests.post(raphtory_url(port), headers=WRITE_HEADERS, data=data) assert_successful_response(response) @@ -208,28 +214,30 @@ def test_update_graph(query): ) def test_mutations(query): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, auth_public_key=PUB_KEY).start(): - add_test_graph() + with GraphServer(work_dir, auth_public_key=PUB_KEY).start() as server: + port = server.port() + add_test_graph(port) data = json.dumps({"query": query}) - response = requests.post(RAPHTORY, data=data) + response = requests.post(raphtory_url(port), data=data) assert response.status_code == 401 - response = requests.post(RAPHTORY, headers=READ_HEADERS, data=data) + response = requests.post(raphtory_url(port), headers=READ_HEADERS, data=data) assert response.json()["data"] is None assert ( - response.json()["errors"][0]["message"] - == "The requested endpoint requires write access" + response.json()["errors"][0]["message"] + == "The requested endpoint requires write access" ) - response = requests.post(RAPHTORY, headers=WRITE_HEADERS, data=data) + response = requests.post(raphtory_url(port), headers=WRITE_HEADERS, data=data) assert_successful_response(response) def test_raphtory_client(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, auth_public_key=PUB_KEY).start(): - client = RaphtoryClient(url=RAPHTORY, token=WRITE_JWT) + with GraphServer(work_dir, auth_public_key=PUB_KEY).start() as server: + port = server.port() + client = RaphtoryClient(url=raphtory_url(port), token=WRITE_JWT) client.new_graph("test", "EVENT") g = client.remote_graph("test") g.add_node(0, "test") @@ -241,8 +249,9 @@ def test_raphtory_client(): def test_raphtory_client_write_denied_for_read_jwt(): """RaphtoryClient initialized with a read JWT is denied write operations.""" work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, auth_public_key=PUB_KEY).start(): - client = RaphtoryClient(url=RAPHTORY, token=READ_JWT) + with GraphServer(work_dir, auth_public_key=PUB_KEY).start() as server: + port = server.port() + client = RaphtoryClient(url=raphtory_url(port), token=READ_JWT) with pytest.raises(Exception, match="requires write access"): client.new_graph("test", "EVENT") @@ -253,10 +262,11 @@ def test_raphtory_client_write_denied_for_read_jwt(): def test_rsa_signed_jwt_rs256_accepted(): """Server configured with an RSA public key accepts RS256-signed JWTs.""" work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, auth_public_key=RSA_PUB_KEY).start(): + with GraphServer(work_dir, auth_public_key=RSA_PUB_KEY).start() as server: + port = server.port() token = jwt.encode({"access": "ro"}, RSA_PRIVATE_KEY, algorithm="RS256") response = requests.post( - RAPHTORY, + raphtory_url(port), headers={"Authorization": f"Bearer {token}"}, data=json.dumps({"query": QUERY_ROOT}), ) @@ -266,10 +276,11 @@ def test_rsa_signed_jwt_rs256_accepted(): def test_rsa_signed_jwt_rs512_accepted(): """RS512 JWT is also accepted for the same RSA key (different hash, same key material).""" work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, auth_public_key=RSA_PUB_KEY).start(): + with GraphServer(work_dir, auth_public_key=RSA_PUB_KEY).start() as server: + port = server.port() token = jwt.encode({"access": "ro"}, RSA_PRIVATE_KEY, algorithm="RS512") response = requests.post( - RAPHTORY, + raphtory_url(port), headers={"Authorization": f"Bearer {token}"}, data=json.dumps({"query": QUERY_ROOT}), ) @@ -279,10 +290,11 @@ def test_rsa_signed_jwt_rs512_accepted(): def test_eddsa_jwt_rejected_against_rsa_key(): """EdDSA JWT is rejected when the server is configured with an RSA public key.""" work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, auth_public_key=RSA_PUB_KEY).start(): + with GraphServer(work_dir, auth_public_key=RSA_PUB_KEY).start() as server: + port = server.port() token = jwt.encode({"access": "ro"}, PRIVATE_KEY, algorithm="EdDSA") response = requests.post( - RAPHTORY, + raphtory_url(port), headers={"Authorization": f"Bearer {token}"}, data=json.dumps({"query": QUERY_ROOT}), ) @@ -292,20 +304,22 @@ def test_eddsa_jwt_rejected_against_rsa_key(): def test_raphtory_client_read_jwt_can_receive_graph(): """RaphtoryClient initialized with a read JWT can download graphs.""" work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, auth_public_key=PUB_KEY).start(): - client = RaphtoryClient(url=RAPHTORY, token=WRITE_JWT) + with GraphServer(work_dir, auth_public_key=PUB_KEY).start() as server: + port = server.port() + client = RaphtoryClient(url=raphtory_url(port), token=WRITE_JWT) client.new_graph("test", "EVENT") client.remote_graph("test").add_node(0, "mynode") - client2 = RaphtoryClient(url=RAPHTORY, token=READ_JWT) + client2 = RaphtoryClient(url=raphtory_url(port), token=READ_JWT) g = client2.receive_graph("test") assert g.node("mynode") is not None def test_upload_graph(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, auth_public_key=PUB_KEY).start(): - client = RaphtoryClient(url=RAPHTORY, token=WRITE_JWT) + with GraphServer(work_dir, auth_public_key=PUB_KEY).start() as server: + port = server.port() + client = RaphtoryClient(url=raphtory_url(port), token=WRITE_JWT) g = Graph() g.add_node(0, "uploaded-node") tmp_dir = tempfile.mkdtemp() diff --git a/python/tests/test_base_install/test_graphql/edit_graph/test_graphql.py b/python/tests/test_base_install/test_graphql/edit_graph/test_graphql.py index 9c0f624be9..379558586d 100644 --- a/python/tests/test_base_install/test_graphql/edit_graph/test_graphql.py +++ b/python/tests/test_base_install/test_graphql/edit_graph/test_graphql.py @@ -45,8 +45,8 @@ def test_wrong_url(): with pytest.raises(Exception) as excinfo: client = RaphtoryClient("http://broken_url.com") assert ( - str(excinfo.value) - == "Could not connect to the given server - no response --error sending request for url (http://broken_url.com/)" + str(excinfo.value) + == "Could not connect to the given server - no response --error sending request for url (http://broken_url.com/)" ) @@ -66,8 +66,9 @@ def test_server_start_on_default_port(): g.add_edge(3, "ben", "haaroon") tmp_work_dir = tempfile.mkdtemp() - with GraphServer(tmp_work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(tmp_work_dir).start() as server: + port = server.port() + client = RaphtoryClient(f"http://localhost:{port}") client.send_graph(path="g", graph=g) query = """{graph(path: "g") {nodes {list {name}}}}""" @@ -119,8 +120,8 @@ def assert_graph_fetch(path): path = "g" tmp_work_dir = tempfile.mkdtemp() - with GraphServer(tmp_work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() # Default namespace, graph is saved in the work dir client.send_graph(path=path, graph=g, overwrite=True) @@ -155,40 +156,40 @@ def assert_graph_fetch(path): with pytest.raises(Exception) as excinfo: client.send_graph(path=path, graph=g, overwrite=True) assert ( - "Invalid path '../shivam/g': References to the parent dir are not allowed within the path" - in str(excinfo.value) + "Invalid path '../shivam/g': References to the parent dir are not allowed within the path" + in str(excinfo.value) ) path = "./shivam/g" with pytest.raises(Exception) as excinfo: client.send_graph(path=path, graph=g, overwrite=True) assert ( - "Invalid path './shivam/g': References to the current dir are not allowed within the path" - in str(excinfo.value) + "Invalid path './shivam/g': References to the current dir are not allowed within the path" + in str(excinfo.value) ) path = "shivam/../../../../investigation/g" with pytest.raises(Exception) as excinfo: client.send_graph(path=path, graph=g, overwrite=True) assert ( - "Invalid path 'shivam/../../../../investigation/g': References to the parent dir are not allowed within the path" - in str(excinfo.value) + "Invalid path 'shivam/../../../../investigation/g': References to the parent dir are not allowed within the path" + in str(excinfo.value) ) path = "//shivam/investigation/g" with pytest.raises(Exception) as excinfo: client.send_graph(path=path, graph=g, overwrite=True) assert ( - "Invalid path '//shivam/investigation/g': Double forward slashes are not allowed in path" - in str(excinfo.value) + "Invalid path '//shivam/investigation/g': Double forward slashes are not allowed in path" + in str(excinfo.value) ) path = "shivam/investigation//2024-12-12/g" with pytest.raises(Exception) as excinfo: client.send_graph(path=path, graph=g, overwrite=True) assert ( - "Invalid path 'shivam/investigation//2024-12-12/g': Double forward slashes are not allowed in path" - in str(excinfo.value) + "Invalid path 'shivam/investigation//2024-12-12/g': Double forward slashes are not allowed in path" + in str(excinfo.value) ) path = r"shivam/investigation\2024-12-12" @@ -206,8 +207,8 @@ def assert_graph_fetch(path): with pytest.raises(Exception) as excinfo: client.send_graph(path=path, graph=g, overwrite=True) assert ( - "Invalid path 'shivam/graphs/not_a_symlink_i_promise/escaped': A component of the given path was a symlink" - in str(excinfo.value) + "Invalid path 'shivam/graphs/not_a_symlink_i_promise/escaped': A component of the given path was a symlink" + in str(excinfo.value) ) @@ -474,8 +475,8 @@ def test_create_node(): g.add_edge(1, "ben", "shivam") tmp_work_dir = tempfile.mkdtemp() - with GraphServer(tmp_work_dir).start(port=1737): - client = RaphtoryClient("http://localhost:1737") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() client.send_graph(path="g", graph=g) query_nodes = """{graph(path: "g") {nodes {list {name}}}}""" @@ -505,8 +506,8 @@ def test_create_node_using_client(): g.add_edge(1, "ben", "shivam") tmp_work_dir = tempfile.mkdtemp() - with GraphServer(tmp_work_dir).start(port=1737): - client = RaphtoryClient("http://localhost:1737") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() client.send_graph(path="g", graph=g) query_nodes = """{graph(path: "g") {nodes {list {name}}}}""" @@ -533,8 +534,8 @@ def test_create_node_using_client_with_properties(): g.add_edge(1, "ben", "shivam") tmp_work_dir = tempfile.mkdtemp() - with GraphServer(tmp_work_dir).start(port=1737): - client = RaphtoryClient("http://localhost:1737") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() client.send_graph(path="g", graph=g) query_nodes = ( @@ -596,8 +597,8 @@ def test_create_node_using_client_with_properties_node_type(): g.add_edge(1, "ben", "shivam") tmp_work_dir = tempfile.mkdtemp() - with GraphServer(tmp_work_dir).start(port=1737): - client = RaphtoryClient("http://localhost:1737") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() client.send_graph(path="g", graph=g) query_nodes = """{graph(path: "g") {nodes {list {name, nodeType, properties { keys }}}}}""" @@ -664,8 +665,8 @@ def test_create_node_using_client_with_node_type(): g.add_edge(1, "ben", "shivam") tmp_work_dir = tempfile.mkdtemp() - with GraphServer(tmp_work_dir).start(port=1737): - client = RaphtoryClient("http://localhost:1737") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() client.send_graph(path="g", graph=g) query_nodes = """{graph(path: "g") {nodes {list {name, nodeType}}}}""" @@ -702,8 +703,8 @@ def test_edge_id(): g.add_edge(3, "po", "ben") tmp_work_dir = tempfile.mkdtemp() - with GraphServer(tmp_work_dir).start(port=1737): - client = RaphtoryClient("http://localhost:1737") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() client.send_graph(path="g", graph=g) query_nodes = """{graph(path: "g") {edges {list {id}}}}""" @@ -724,8 +725,8 @@ def test_graph_persistence_across_restarts(): tmp_work_dir = tempfile.mkdtemp() # First server session: create graph with 3 nodes and 2 edges - with GraphServer(tmp_work_dir).start(port=1738): - client = RaphtoryClient("http://localhost:1738") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() client.new_graph(path="persistent_graph", graph_type="EVENT") remote_graph = client.remote_graph(path="persistent_graph") # Create 3 nodes @@ -761,8 +762,8 @@ def test_graph_persistence_across_restarts(): } # Server is now shutdown, start it again - with GraphServer(tmp_work_dir).start(port=1738): - client = RaphtoryClient("http://localhost:1738") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() # Verify persistence: check that nodes and edges are still there query_nodes = """{graph(path: "persistent_graph") {nodes {sorted (sortBys: [{id: true}]){ list {name} }}}}""" @@ -841,8 +842,8 @@ def test_float_is_stable_on_roundtrip(): ] prop_key = "p" - with GraphServer(tmp_work_dir).start(port=1738): - client = RaphtoryClient("http://localhost:1738") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() client.new_graph(path="g", graph_type="EVENT") remote_graph = client.remote_graph(path="g") @@ -867,7 +868,6 @@ def test_float_is_stable_on_roundtrip(): retrieved_float = resp["graph"]["node"]["at"]["properties"]["get"]["value"] assert retrieved_float == num - # def test_disk_graph_name(): # import pandas as pd # from raphtory import DiskGraphStorage diff --git a/python/tests/test_base_install/test_graphql/edit_graph/test_move_graph.py b/python/tests/test_base_install/test_graphql/edit_graph/test_move_graph.py index f72762e3d8..72a98facf1 100644 --- a/python/tests/test_base_install/test_graphql/edit_graph/test_move_graph.py +++ b/python/tests/test_base_install/test_graphql/edit_graph/test_move_graph.py @@ -9,8 +9,8 @@ def test_move_graph_fails_if_graph_not_found(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() query = """mutation { moveGraph( @@ -34,8 +34,8 @@ def test_move_graph_fails_if_graph_with_same_name_already_exists(): g.save_to_file(os.path.join(work_dir, "ben", "g5")) g.save_to_file(os.path.join(work_dir, "g6")) - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() query = """mutation { moveGraph( @@ -59,8 +59,8 @@ def test_move_graph_fails_if_graph_with_same_name_already_exists_at_same_namespa g.save_to_file(os.path.join(work_dir, "ben", "g5")) g.save_to_file(os.path.join(work_dir, "ben", "g6")) - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() query = """mutation { moveGraph( @@ -85,8 +85,8 @@ def test_move_graph_fails_if_graph_with_same_name_already_exists_at_diff_namespa g.save_to_file(os.path.join(work_dir, "ben", "g5")) g.save_to_file(os.path.join(work_dir, "shivam", "g6")) - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() query = """mutation { moveGraph( @@ -109,8 +109,8 @@ def test_move_graph_succeeds(): os.makedirs(os.path.join(work_dir, "shivam"), exist_ok=True) g.save_to_file(os.path.join(work_dir, "shivam", "g3")) - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() # Assert if rename graph succeeds and old graph is deleted query = """mutation { @@ -148,8 +148,8 @@ def test_move_graph_using_client_api_succeeds(): os.makedirs(os.path.join(work_dir, "shivam"), exist_ok=True) g.save_to_file(os.path.join(work_dir, "shivam", "g3")) - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() # Assert if rename graph succeeds and old graph is deleted client.move_graph("shivam/g3", "ben/g4") @@ -182,8 +182,8 @@ def test_move_graph_succeeds_at_same_namespace_as_graph(): g.save_to_file(os.path.join(work_dir, "shivam", "g3")) - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() # Assert if rename graph succeeds and old graph is deleted query = """mutation { @@ -223,8 +223,8 @@ def test_move_graph_succeeds_at_diff_namespace_as_graph(): g.save_to_file(os.path.join(work_dir, "ben", "g3")) - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() # Assert if rename graph succeeds and old graph is deleted query = """mutation { diff --git a/python/tests/test_base_install/test_graphql/edit_graph/test_new_graph.py b/python/tests/test_base_install/test_graphql/edit_graph/test_new_graph.py index adba406a92..4481a52485 100644 --- a/python/tests/test_base_install/test_graphql/edit_graph/test_new_graph.py +++ b/python/tests/test_base_install/test_graphql/edit_graph/test_new_graph.py @@ -8,8 +8,8 @@ def test_new_graph_succeeds(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() query = """mutation { newGraph( @@ -25,8 +25,8 @@ def test_new_graph_succeeds(): def test_new_graph_fails_if_graph_found(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() g = Graph() g.add_edge(1, "ben", "hamza") @@ -50,8 +50,8 @@ def test_new_graph_fails_if_graph_found(): def test_client_new_graph_works(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() client.new_graph("path/to/event_graph", "EVENT") client.new_graph("path/to/persistent_graph", "PERSISTENT") @@ -63,8 +63,8 @@ def test_client_new_graph_works(): def test_client_new_graph_broken_type(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() with pytest.raises(Exception) as excinfo: client.new_graph("path/to/event_graph", "EVENdddT") assert "Invalid value for argument" in str(excinfo.value) diff --git a/python/tests/test_base_install/test_graphql/edit_graph/test_upload_graph.py b/python/tests/test_base_install/test_graphql/edit_graph/test_upload_graph.py index 5f92d5e37a..fd8ffc5019 100644 --- a/python/tests/test_base_install/test_graphql/edit_graph/test_upload_graph.py +++ b/python/tests/test_base_install/test_graphql/edit_graph/test_upload_graph.py @@ -17,8 +17,8 @@ def test_upload_graph_succeeds_if_no_graph_found_with_same_name(): g.save_to_zip(g_file_path) tmp_work_dir = tempfile.mkdtemp() - with GraphServer(tmp_work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() client.upload_graph(path="g", file_path=g_file_path, overwrite=False) query = """{graph(path: "g") {nodes {list {name}}}}""" @@ -41,8 +41,8 @@ def test_upload_graph_succeeds_if_no_graph_found_with_same_name_non_zip(): g.save_to_file(g_file_path) tmp_work_dir = tempfile.mkdtemp() - with GraphServer(tmp_work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() client.upload_graph(path="g", file_path=g_file_path, overwrite=False) query = """{graph(path: "g") {nodes {list {name}}}}""" @@ -66,8 +66,8 @@ def test_upload_graph_fails_if_graph_already_exists(): tmp_work_dir = tempfile.mkdtemp() g.save_to_file(os.path.join(tmp_work_dir, "g")) - with GraphServer(tmp_work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() with pytest.raises(Exception) as excinfo: client.upload_graph(path="g", file_path=g_file_path) assert "Graph 'g' already exists" in str(excinfo.value) @@ -83,8 +83,8 @@ def test_upload_graph_succeeds_if_graph_already_exists_with_overwrite_enabled(): g.save_to_file(g_file_path) tmp_work_dir = tempfile.mkdtemp() - with GraphServer(tmp_work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() g = Graph() g.add_edge(1, "ben", "hamza") @@ -123,8 +123,8 @@ def test_upload_graph_succeeds_if_no_graph_found_with_same_name_at_namespace(): g.save_to_zip(g_file_path) tmp_work_dir = tempfile.mkdtemp() - with GraphServer(tmp_work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() client.upload_graph(path="shivam/g", file_path=g_file_path, overwrite=False) query = """{graph(path: "shivam/g") {nodes {list {name}}}}""" @@ -151,8 +151,8 @@ def test_upload_graph_fails_if_graph_already_exists_at_namespace(): tmp_work_dir = tempfile.mkdtemp() os.makedirs(os.path.join(tmp_work_dir, "shivam"), exist_ok=True) g.save_to_file(os.path.join(tmp_work_dir, "shivam", "g")) - with GraphServer(tmp_work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() with pytest.raises(Exception) as excinfo: client.upload_graph(path="shivam/g", file_path=g_file_path, overwrite=False) assert "Graph 'shivam/g' already exists" in str(excinfo.value) @@ -167,8 +167,8 @@ def test_upload_graph_succeeds_if_graph_already_exists_at_namespace_with_overwri os.makedirs(os.path.join(tmp_work_dir, "shivam"), exist_ok=True) g.save_to_file(os.path.join(tmp_work_dir, "shivam", "g")) - with GraphServer(tmp_work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() g = Graph() g.add_edge(1, "ben", "hamza") diff --git a/python/tests/test_base_install/test_graphql/misc/test_components.py b/python/tests/test_base_install/test_graphql/misc/test_components.py index 9197d17069..dfa67dc776 100644 --- a/python/tests/test_base_install/test_graphql/misc/test_components.py +++ b/python/tests/test_base_install/test_graphql/misc/test_components.py @@ -89,8 +89,8 @@ def test_in_out_components(): g.add_edge(6, 7, 3) g.save_to_file(work_dir + "/graph") - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() query_res = client.query(query) prepare_for_comparison(query_res["graph"]) prepare_for_comparison(result["graph"]) diff --git a/python/tests/test_base_install/test_graphql/misc/test_index_off.py b/python/tests/test_base_install/test_graphql/misc/test_index_off.py index 62786fae60..2d3d482929 100644 --- a/python/tests/test_base_install/test_graphql/misc/test_index_off.py +++ b/python/tests/test_base_install/test_graphql/misc/test_index_off.py @@ -19,11 +19,11 @@ def test_latest_and_active(): work_dir = tempfile.mkdtemp() g = Graph() g.save_to_file(work_dir + "/graph") - with GraphServer(work_dir).turn_off_index().start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).turn_off_index().start() as server: + client = server.get_client() with pytest.raises(Exception) as excinfo: client.query(query) assert ( - "An operation tried to make use of the graph index but indexing has been turned off for the server" - in str(excinfo.value) + "An operation tried to make use of the graph index but indexing has been turned off for the server" + in str(excinfo.value) ) diff --git a/python/tests/test_base_install/test_graphql/misc/test_latest.py b/python/tests/test_base_install/test_graphql/misc/test_latest.py index ee58148aed..8dfd3ba0dc 100644 --- a/python/tests/test_base_install/test_graphql/misc/test_latest.py +++ b/python/tests/test_base_install/test_graphql/misc/test_latest.py @@ -232,6 +232,6 @@ def test_latest_and_active(): g.add_node(2, 2, {"int_prop": 125}) g.save_to_file(work_dir + "/graph") - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() assert sort_by_gql_name_or_id(client.query(query)) == result diff --git a/python/tests/test_base_install/test_graphql/misc/test_map_props.py b/python/tests/test_base_install/test_graphql/misc/test_map_props.py index f5ca23d73a..5546b75bc8 100644 --- a/python/tests/test_base_install/test_graphql/misc/test_map_props.py +++ b/python/tests/test_base_install/test_graphql/misc/test_map_props.py @@ -14,9 +14,9 @@ def test_map_props(): work_dir = tempfile.mkdtemp() server = GraphServer(work_dir) - with server.start(): + with server.start() as server: temp_dir = tempfile.mkdtemp() - client = RaphtoryClient("http://localhost:1736") + client = server.get_client() g = Graph() g.update_metadata({"test": TEST_PROPS}) node = g.add_node(0, "test") @@ -28,7 +28,8 @@ def test_map_props(): work_dir = tempfile.mkdtemp() server = GraphServer(work_dir) - with server.start(): + with server.start() as server: + client = server.get_client() client.new_graph("test", "EVENT") rg = client.remote_graph("test") rg.update_metadata({"test": TEST_PROPS}) diff --git a/python/tests/test_base_install/test_graphql/misc/test_snapshot.py b/python/tests/test_base_install/test_graphql/misc/test_snapshot.py index de8c8f8501..1bb4dc8ba6 100644 --- a/python/tests/test_base_install/test_graphql/misc/test_snapshot.py +++ b/python/tests/test_base_install/test_graphql/misc/test_snapshot.py @@ -5,8 +5,8 @@ def test_snapshot(): work_dir = tempfile.mkdtemp() server = GraphServer(work_dir) - with server.start(): - client = RaphtoryClient("http://localhost:1736") + with server.start() as server: + client = server.get_client() def query(graph: str, window: str): return client.query(f"""{{ diff --git a/python/tests/test_base_install/test_graphql/misc/test_tracing.py b/python/tests/test_base_install/test_graphql/misc/test_tracing.py index f870d20e68..b7f0ebeca0 100644 --- a/python/tests/test_base_install/test_graphql/misc/test_tracing.py +++ b/python/tests/test_base_install/test_graphql/misc/test_tracing.py @@ -12,8 +12,8 @@ def test_server_start_on_default_port(): g.add_edge(3, "ben", "haaroon") tmp_work_dir = tempfile.mkdtemp() - with GraphServer(tmp_work_dir, tracing=True).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(tmp_work_dir, tracing=True).start() as server: + client = server.get_client() client.send_graph(path="g", graph=g) query = """{graph(path: "g") {nodes {list {name}}}}""" @@ -24,8 +24,8 @@ def test_server_start_on_default_port(): } } } - with GraphServer(tmp_work_dir, tracing=True).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(tmp_work_dir, tracing=True).start() as server: + client = server.get_client() client.send_graph(path="g2", graph=g) query = """{graph(path: "g2") {nodes {list {name}}}}""" diff --git a/python/tests/test_base_install/test_graphql/test_graph_file_time_stats.py b/python/tests/test_base_install/test_graphql/test_graph_file_time_stats.py index 63e6de6dba..9cef6a1d38 100644 --- a/python/tests/test_base_install/test_graphql/test_graph_file_time_stats.py +++ b/python/tests/test_base_install/test_graphql/test_graph_file_time_stats.py @@ -16,8 +16,8 @@ def test_graph_file_time_stats(): graph_file_path = os.path.join(work_dir, "shivam", "g3") g.save_to_file(graph_file_path) - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() query = """{graph(path: "shivam/g3") { created lastOpened lastUpdated }}""" result = client.query(query) @@ -35,11 +35,11 @@ def test_graph_file_time_stats(): last_updated_time_fs = meta_file_stats.st_mtime * 1000 assert ( - abs(gql_created_time - created_time_fs) < 1000 + abs(gql_created_time - created_time_fs) < 1000 ), f"Mismatch in created time: FS({created_time_fs}) vs GQL({gql_created_time})" assert ( - abs(gql_last_opened_time - last_opened_time_fs) < 1000 + abs(gql_last_opened_time - last_opened_time_fs) < 1000 ), f"Mismatch in last opened time: FS({last_opened_time_fs}) vs GQL({gql_last_opened_time})" assert ( - abs(gql_last_updated_time - last_updated_time_fs) < 1000 + abs(gql_last_updated_time - last_updated_time_fs) < 1000 ), f"Mismatch in last updated time: FS({last_updated_time_fs}) vs GQL({gql_last_updated_time})" diff --git a/python/tests/test_base_install/test_graphql/test_metadata_dispatch.py b/python/tests/test_base_install/test_graphql/test_metadata_dispatch.py index 01c60bfe11..e5c255bac0 100644 --- a/python/tests/test_base_install/test_graphql/test_metadata_dispatch.py +++ b/python/tests/test_base_install/test_graphql/test_metadata_dispatch.py @@ -33,8 +33,6 @@ reason="disk-backed graph tests require the storage feature", ) -SERVER_URL = "http://localhost:1736" - def _persist_disk_graph(graph_dir): """Build a disk-backed graph at `graph_dir`, populate it, flush it, @@ -106,14 +104,14 @@ def test_metadata_returned_for_both_disk_and_parquet_graphs(): # test still passing (metadata round-trips for either format), and # the parquet dispatch path would silently stop being exercised. assert ( - _read_is_diskgraph(disk_graph_dir) is True + _read_is_diskgraph(disk_graph_dir) is True ), "disk_graph was not saved as a disk graph" assert ( - _read_is_diskgraph(parquet_graph_dir) is False + _read_is_diskgraph(parquet_graph_dir) is False ), "parquet_graph was not saved as parquet" - with GraphServer(work_dir).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir).start() as server: + client = server.get_client() # ---- Path 2 (disk on-disk read) and Path 3 (parquet on-disk read). # Neither graph has been loaded into the server's cache yet, so @@ -141,7 +139,7 @@ def test_metadata_returned_for_both_disk_and_parquet_graphs(): meta_cached = _list_metadata_by_path(client) assert ( - meta_cached == meta + meta_cached == meta ), "cached-path metadata should match the on-disk-path metadata" @@ -160,11 +158,11 @@ def test_metadata_update_in_single_segment_returns_latest(): g.flush() del g - with GraphServer(work_dir).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir).start() as server: + client = server.get_client() meta = _list_metadata_by_path(client) assert ( - meta["g"]["version"] == "v2" + meta["g"]["version"] == "v2" ), f"expected latest in-segment value 'v2', got {meta['g'].get('version')!r}" @@ -187,11 +185,11 @@ def test_metadata_update_across_flushes_returns_newest_segment(): g.flush() del g - with GraphServer(work_dir).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir).start() as server: + client = server.get_client() meta = _list_metadata_by_path(client) assert ( - meta["g"]["version"] == "v2" + meta["g"]["version"] == "v2" ), f"expected newest-segment value 'v2', got {meta['g'].get('version')!r}" @@ -211,11 +209,11 @@ def test_metadata_many_updates_across_flushes_returns_last(): g.flush() del g - with GraphServer(work_dir).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir).start() as server: + client = server.get_client() meta = _list_metadata_by_path(client) assert ( - meta["g"]["version"] == "v499" + meta["g"]["version"] == "v499" ), f"expected last-write value 'v499', got {meta['g'].get('version')!r}" @@ -278,10 +276,10 @@ def test_metadata_mixed_keys_across_flushes(): g.flush() del g - with GraphServer(work_dir).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir).start() as server: + client = server.get_client() meta = _list_metadata_by_path(client) assert meta["g"]["untouched"] == "stable" assert ( - meta["g"]["bumped"] == "new" + meta["g"]["bumped"] == "new" ), f"expected updated value 'new', got {meta['g'].get('bumped')!r}" diff --git a/python/tests/test_base_install/test_graphql/test_misc.py b/python/tests/test_base_install/test_graphql/test_misc.py index a8a1992845..f37d1cd45e 100644 --- a/python/tests/test_base_install/test_graphql/test_misc.py +++ b/python/tests/test_base_install/test_graphql/test_misc.py @@ -8,6 +8,6 @@ def test_version_query(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() assert client.query("{version}")["version"] == raphtory.version() diff --git a/python/tests/test_base_install/test_graphql/test_namespace.py b/python/tests/test_base_install/test_graphql/test_namespace.py index 5ab219149a..30ed9a3bc8 100644 --- a/python/tests/test_base_install/test_graphql/test_namespace.py +++ b/python/tests/test_base_install/test_graphql/test_namespace.py @@ -29,8 +29,8 @@ def sort_dict(d): def test_namespaces_and_metagraph(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() make_folder_structure(client) # tests list and page on namespaces and metagraphs @@ -188,8 +188,8 @@ def test_namespaces_and_metagraph(): def test_counting(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() make_folder_structure(client) query = """ @@ -227,8 +227,8 @@ def test_counting(): def test_escaping_parent(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() make_folder_structure(client) query = """{ @@ -265,8 +265,8 @@ def test_escaping_parent(): def test_wrong_paths(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() make_folder_structure(client) query = """{ @@ -347,8 +347,8 @@ def test_wrong_paths(): def test_namespaces(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() make_folder_structure(client) query = """ diff --git a/python/tests/test_base_install/test_graphql/test_read_only_load.py b/python/tests/test_base_install/test_graphql/test_read_only_load.py index b4a266f12d..beb72d4351 100644 --- a/python/tests/test_base_install/test_graphql/test_read_only_load.py +++ b/python/tests/test_base_install/test_graphql/test_read_only_load.py @@ -31,8 +31,6 @@ reason="disk-backed graph tests require the storage feature", ) -SERVER_URL = "http://localhost:1736" - def _persist_graph(graph_dir): """Build a disk-backed graph at `graph_dir`, populate it, flush it, and @@ -65,8 +63,8 @@ def test_read_only_load_while_server_owns_directory(): read-only handle should both succeed and report the same data. """ work_dir, graph_dir = _make_work_dir_with_graph() - with GraphServer(work_dir).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir).start() as server: + client = server.get_client() # Trigger the server to load the graph so it holds the writer. client.query('{ graph(path: "g") { created } }') @@ -104,8 +102,8 @@ def test_graphql_and_read_only_handle_interleaved(): read-only handle to the same directory. Both pathways should keep serving consistent results across multiple round-trips.""" work_dir, graph_dir = _make_work_dir_with_graph() - with GraphServer(work_dir).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir).start() as server: + client = server.get_client() # Prime the server's load. client.query('{ graph(path: "g") { created } }') ro = Graph.load(graph_dir, read_only=True) @@ -130,8 +128,8 @@ def test_read_only_load_blocks_all_mutation_paths(): - graph-level metadata (never touches the id resolver at all) """ work_dir, graph_dir = _make_work_dir_with_graph() - with GraphServer(work_dir).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir).start() as server: + client = server.get_client() client.query('{ graph(path: "g") { created } }') ro = Graph.load(graph_dir, read_only=True) @@ -148,7 +146,7 @@ def test_read_only_load_blocks_all_mutation_paths(): mutate() msg = str(exc.value).lower() assert ( - "locked" in msg or "immutable" in msg + "locked" in msg or "immutable" in msg ), f"{name} did not raise a locked/immutable error: {exc.value}" @@ -159,8 +157,8 @@ def test_writer_load_against_live_server_directory_fails(): `read_only=True` is not accidentally a no-op flag — it really is the only way to coexist with a live writer.""" work_dir, graph_dir = _make_work_dir_with_graph() - with GraphServer(work_dir).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir).start() as server: + client = server.get_client() # Force the server to take the writer lock by loading the graph. client.query('{ graph(path: "g") { created } }') @@ -172,8 +170,8 @@ def test_multiple_read_only_handles_can_coexist_with_server(): """Two simultaneous read-only handles + the server writer = three total attachments to the same graph directory.""" work_dir, graph_dir = _make_work_dir_with_graph() - with GraphServer(work_dir).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir).start() as server: + client = server.get_client() client.query('{ graph(path: "g") { created } }') ro1 = Graph.load(graph_dir, read_only=True) @@ -212,8 +210,8 @@ def test_flush_via_update_graph_makes_writes_visible_to_read_only_handle(): when it opens) wouldn't see writes the server has made via `updateGraph` since the last flush.""" work_dir, graph_dir = _make_work_dir_with_graph() - with GraphServer(work_dir).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir).start() as server: + client = server.get_client() # Server loads as writer. client.query('{ graph(path: "g") { created } }') @@ -234,8 +232,8 @@ def test_flush_via_update_graph_makes_writes_visible_to_read_only_handle(): def test_read_only_load_from_a_separate_python_process(): work_dir, graph_dir = _make_work_dir_with_graph() - with GraphServer(work_dir).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir).start() as server: + client = server.get_client() client.query('{ graph(path: "g") { created } }') result = subprocess.run( diff --git a/python/tests/test_base_install/test_graphql/test_schema.py b/python/tests/test_base_install/test_graphql/test_schema.py index 2ec0412efd..6c67fd58b3 100644 --- a/python/tests/test_base_install/test_graphql/test_schema.py +++ b/python/tests/test_base_install/test_graphql/test_schema.py @@ -58,8 +58,8 @@ def test_node_edge_properties_schema(): graph_file_path = os.path.join(work_dir, "graph") g.save_to_file(graph_file_path) - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() query = """{ graph(path: "graph") { diff --git a/python/tests/test_base_install/test_graphql/test_server_flags.py b/python/tests/test_base_install/test_graphql/test_server_flags.py index 1c80de56f1..07ee01d486 100644 --- a/python/tests/test_base_install/test_graphql/test_server_flags.py +++ b/python/tests/test_base_install/test_graphql/test_server_flags.py @@ -10,11 +10,11 @@ SERVER_URL = "http://localhost:1736" -def batch_query(body): +def batch_query(port, body): """POST a raw JSON body (needed for batch requests — the client only sends single queries).""" data = json.dumps(body).encode("utf-8") req = urllib.request.Request( - SERVER_URL + "/", + f"http://localhost:{port}/", data=data, headers={"Content-Type": "application/json"}, method="POST", @@ -40,16 +40,16 @@ def make_graph(client, path="g"): def test_introspection_enabled_by_default(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir).start() as server: + client = server.get_client() result = client.query("{ __schema { queryType { name } } }") assert result["__schema"]["queryType"]["name"] def test_disable_introspection(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, disable_introspection=True).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir, disable_introspection=True).start() as server: + client = server.get_client() client.query("{ version }") with pytest.raises(Exception) as excinfo: @@ -60,8 +60,8 @@ def test_disable_introspection(): def test_max_query_depth(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, max_query_depth=3).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir, max_query_depth=3).start() as server: + client = server.get_client() make_graph(client) client.query('{ graph(path: "g") { created } }') @@ -75,8 +75,8 @@ def test_max_query_depth(): def test_max_query_complexity(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, max_query_complexity=3).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir, max_query_complexity=3).start() as server: + client = server.get_client() make_graph(client) client.query("{ version }") @@ -253,24 +253,24 @@ def test_max_query_complexity(): def test_disable_lists_all_resolvers(): """Every `list` endpoint across every paginated type rejects with the same error.""" work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, disable_lists=True).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir, disable_lists=True).start() as server: + client = server.get_client() make_graph(client) for name, query in LIST_QUERIES: with pytest.raises(Exception) as excinfo: client.query(query) assert ( - "Bulk list endpoints are disabled on this server. Use `page` instead." - in str(excinfo.value) + "Bulk list endpoints are disabled on this server. Use `page` instead." + in str(excinfo.value) ), f"{name} did not reject with the expected error: {excinfo.value}" def test_disable_lists_page_still_works(): """Even with `disable_lists=True`, `page` queries still succeed.""" work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, disable_lists=True).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir, disable_lists=True).start() as server: + client = server.get_client() make_graph(client) result = client.query( '{ graph(path: "g") { nodes { page(limit: 10) { name } } } }' @@ -281,8 +281,8 @@ def test_disable_lists_page_still_works(): def test_max_page_size_all_resolvers(): """Every `page` endpoint across every paginated type enforces max_page_size.""" work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, max_page_size=2).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir, max_page_size=2).start() as server: + client = server.get_client() make_graph(client) for name, query in PAGE_QUERIES: @@ -296,8 +296,8 @@ def test_max_page_size_all_resolvers(): def test_max_page_size_under_cap_works(): """Pages at or below max_page_size still succeed.""" work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, max_page_size=2).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir, max_page_size=2).start() as server: + client = server.get_client() make_graph(client) result = client.query( '{ graph(path: "g") { nodes { page(limit: 2) { name } } } }' @@ -307,30 +307,31 @@ def test_max_page_size_under_cap_works(): def test_disable_batching(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, disable_batching=True).start(): - RaphtoryClient(SERVER_URL).query("{ version }") + with GraphServer(work_dir, disable_batching=True).start() as server: + server.get_client().query("{ version }") - status, body = batch_query([{"query": "{ version }"}, {"query": "{ version }"}]) + status, body = batch_query(server.port(), [{"query": "{ version }"}, {"query": "{ version }"}]) assert status == 400 assert "Query batching is disabled on this server" in str(body) def test_max_batch_size(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, max_batch_size=2).start(): - status, body = batch_query([{"query": "{ version }"}] * 2) + with GraphServer(work_dir, max_batch_size=2).start() as server: + port = server.port() + status, body = batch_query(port, [{"query": "{ version }"}] * 2) assert status == 200 assert isinstance(body, list) and len(body) == 2 - status, body = batch_query([{"query": "{ version }"}] * 3) + status, body = batch_query(port, [{"query": "{ version }"}] * 3) assert status == 400 assert "Batch size 3 exceeds the maximum allowed 2" in str(body) def test_max_recursive_depth(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, max_recursive_depth=2).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir, max_recursive_depth=2).start() as server: + client = server.get_client() make_graph(client) # depth 2: { graph { created } } — root selection set is depth 0, graph{...} pushes to 1 @@ -345,8 +346,8 @@ def test_max_recursive_depth(): def test_max_directives_per_field(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, max_directives_per_field=1).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir, max_directives_per_field=1).start() as server: + client = server.get_client() # 1 directive — allowed client.query("{ version @skip(if: false) }") @@ -355,8 +356,8 @@ def test_max_directives_per_field(): with pytest.raises(Exception) as excinfo: client.query("{ version @skip(if: false) @include(if: true) }") assert ( - "number of directives on the field `version` cannot be greater than `1`" - in str(excinfo.value) + "number of directives on the field `version` cannot be greater than `1`" + in str(excinfo.value) ) @@ -367,11 +368,11 @@ def test_max_directives_per_field(): def test_concurrency_flags_smoke(): work_dir = tempfile.mkdtemp() with GraphServer( - work_dir, - heavy_query_limit=4, - exclusive_writes=True, - ).start(): - client = RaphtoryClient(SERVER_URL) + work_dir, + heavy_query_limit=4, + exclusive_writes=True, + ).start() as server: + client = server.get_client() make_graph(client) # Read path: works under exclusive_writes's read lock. assert client.query('{ graph(path: "g") { nodes { count } } }') diff --git a/python/tests/test_base_install/test_graphql/update_graph/test_batch_updates.py b/python/tests/test_base_install/test_graphql/update_graph/test_batch_updates.py index 35988b24a3..13c2c425b0 100644 --- a/python/tests/test_base_install/test_graphql/update_graph/test_batch_updates.py +++ b/python/tests/test_base_install/test_graphql/update_graph/test_batch_updates.py @@ -70,8 +70,8 @@ def create_updates(timestamps: List[int]): def test_add_nodes(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() client.new_graph("path/to/event_graph", "EVENT") rg: RemoteGraph = client.remote_graph("path/to/event_graph") node_updates = [] @@ -124,8 +124,8 @@ def test_add_nodes(): def test_add_edges(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() client.new_graph("path/to/event_graph", "EVENT") rg: RemoteGraph = client.remote_graph("path/to/event_graph") edge_updates = [] diff --git a/python/tests/test_base_install/test_graphql/update_graph/test_edge_updates.py b/python/tests/test_base_install/test_graphql/update_graph/test_edge_updates.py index f7c0181542..ed9cef7c49 100644 --- a/python/tests/test_base_install/test_graphql/update_graph/test_edge_updates.py +++ b/python/tests/test_base_install/test_graphql/update_graph/test_edge_updates.py @@ -54,8 +54,8 @@ def make_props2(): def test_add_updates(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() client.new_graph("path/to/event_graph", "EVENT") rg = client.remote_graph("path/to/event_graph") props = make_props() @@ -78,8 +78,8 @@ def test_add_updates(): def test_add_metadata(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() client.new_graph("path/to/event_graph", "EVENT") rg = client.remote_graph("path/to/event_graph") props = make_props() @@ -100,8 +100,8 @@ def test_add_metadata(): def test_update_metadata(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() client.new_graph("path/to/event_graph", "EVENT") rg = client.remote_graph("path/to/event_graph") props = make_props() @@ -122,8 +122,8 @@ def test_update_metadata(): def test_delete(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() client.new_graph("path/to/event_graph", "EVENT") rg = client.remote_graph("path/to/event_graph") diff --git a/python/tests/test_base_install/test_graphql/update_graph/test_graph_updates.py b/python/tests/test_base_install/test_graphql/update_graph/test_graph_updates.py index 4ca5df2c6e..026fe7c64e 100644 --- a/python/tests/test_base_install/test_graphql/update_graph/test_graph_updates.py +++ b/python/tests/test_base_install/test_graphql/update_graph/test_graph_updates.py @@ -40,8 +40,8 @@ def make_props(): def test_add_metadata(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() client.new_graph("path/to/event_graph", "EVENT") rg = client.remote_graph("path/to/event_graph") props = make_props() @@ -56,8 +56,8 @@ def test_add_metadata(): def test_update_metadata(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() client.new_graph("path/to/event_graph", "EVENT") rg = client.remote_graph("path/to/event_graph") props = make_props() @@ -73,8 +73,8 @@ def test_update_metadata(): def test_add_properties(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() client.new_graph("path/to/event_graph", "EVENT") rg = client.remote_graph("path/to/event_graph") props = make_props() @@ -104,8 +104,8 @@ def test_add_properties(): def test_add_node(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() client.new_graph("path/to/event_graph", "EVENT") rg = client.remote_graph("path/to/event_graph") props = make_props() @@ -123,8 +123,8 @@ def test_add_node(): def test_add_edge(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() client.new_graph("path/to/event_graph", "EVENT") rg = client.remote_graph("path/to/event_graph") props = make_props() @@ -142,8 +142,8 @@ def test_add_edge(): def test_delete_edge(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() client.new_graph("path/to/event_graph", "EVENT") rg = client.remote_graph("path/to/event_graph") diff --git a/python/tests/test_base_install/test_graphql/update_graph/test_node_updates.py b/python/tests/test_base_install/test_graphql/update_graph/test_node_updates.py index c394dd657d..4690d4eb92 100644 --- a/python/tests/test_base_install/test_graphql/update_graph/test_node_updates.py +++ b/python/tests/test_base_install/test_graphql/update_graph/test_node_updates.py @@ -36,8 +36,8 @@ def make_props(): def test_set_node_type(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() client.new_graph("path/to/event_graph", "EVENT") rg = client.remote_graph("path/to/event_graph") node = rg.add_node(1, "ben") @@ -51,8 +51,8 @@ def test_set_node_type(): def test_add_updates(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() client.new_graph("path/to/event_graph", "EVENT") rg = client.remote_graph("path/to/event_graph") props = make_props() @@ -67,8 +67,8 @@ def test_add_updates(): def test_add_metadata(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() client.new_graph("path/to/event_graph", "EVENT") rg = client.remote_graph("path/to/event_graph") props = make_props() @@ -84,8 +84,8 @@ def test_add_metadata(): def test_update_metadata(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() client.new_graph("path/to/event_graph", "EVENT") rg = client.remote_graph("path/to/event_graph") props = make_props() diff --git a/python/tests/test_search/test_gql_index_spec.py b/python/tests/test_search/test_gql_index_spec.py index 04a639757f..65811627b9 100644 --- a/python/tests/test_search/test_gql_index_spec.py +++ b/python/tests/test_search/test_gql_index_spec.py @@ -108,10 +108,9 @@ def test_create_index_with_default_spec(graph): @pytest.mark.parametrize("graph", [EVENT_GRAPH, PERSISTENT_GRAPH]) def test_create_index_using_client(graph): - tmp_work_dir = tempfile.mkdtemp() - with GraphServer(tmp_work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() client.send_graph(path="g", graph=graph) query = """{graph(path: "g") {nodes {list {name}}}}""" diff --git a/python/tests/test_vectors/test_graphql_vectors.py b/python/tests/test_vectors/test_graphql_vectors.py index 6a22c0b35c..cf46800859 100644 --- a/python/tests/test_vectors/test_graphql_vectors.py +++ b/python/tests/test_vectors/test_graphql_vectors.py @@ -60,8 +60,8 @@ def test_new_graph(): work_dir = tempfile.TemporaryDirectory() server = GraphServer(work_dir.name) with embeddings.start(7340): - with server.start(): - client = RaphtoryClient("http://localhost:1736") + with server.start() as server: + client = server.get_client() client.new_graph("abb", "EVENT") rg = client.remote_graph("abb") setup_graph(rg) @@ -79,8 +79,8 @@ def test_upload_graph(): temp_dir = tempfile.TemporaryDirectory() server = GraphServer(work_dir.name) with embeddings.start(7340): - with server.start(): - client = RaphtoryClient("http://localhost:1736") + with server.start() as server: + client = server.get_client() g = Graph() setup_graph(g) g_path = temp_dir.name + "/abb" @@ -104,8 +104,8 @@ def test_vectorised_graph_window_accepts_time_input_shapes(): work_dir = tempfile.TemporaryDirectory() server = GraphServer(work_dir.name) with embeddings.start(7340): - with server.start(): - client = RaphtoryClient("http://localhost:1736") + with server.start() as server: + client = server.get_client() client.new_graph("abb", "EVENT") rg = client.remote_graph("abb") setup_graph(rg) @@ -163,6 +163,6 @@ def test_include_graph(): nodes="{{ name }}", edges=False, ) - with server.start(): - client = RaphtoryClient("http://localhost:1736") + with server.start() as server: + client = server.get_client() assert_correct_documents(client) From ab0773b2a1f5ee4ae4e387f84d84f3a34e2b00c6 Mon Sep 17 00:00:00 2001 From: Lucas Jeub Date: Mon, 18 May 2026 12:24:03 +0200 Subject: [PATCH 15/24] Need to actually use the newly materialized graph and not the old graph when inserting with disk storage enabled --- raphtory-graphql/src/data.rs | 5 ++--- raphtory-graphql/src/paths.rs | 15 ++++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/raphtory-graphql/src/data.rs b/raphtory-graphql/src/data.rs index dea38257d6..c7e5b4f245 100644 --- a/raphtory-graphql/src/data.rs +++ b/raphtory-graphql/src/data.rs @@ -239,9 +239,9 @@ impl Data { self.cache .insert_with(&key, || { blocking_compute(move || { - let is_dirty = writeable_folder.write_graph_data(graph.clone(), config)?; + let (is_dirty, new_graph) = writeable_folder.write_graph_data(graph, config)?; let folder = writeable_folder.finish()?; - let graph = GraphWithVectors::new(graph, None, folder.as_existing()?); + let graph = GraphWithVectors::new(new_graph, None, folder.as_existing()?); graph.set_dirty(is_dirty); Ok::<_, InsertionError>(graph) }) @@ -293,7 +293,6 @@ impl Data { self.delete_graph_inner(graph_folder) .await .map_err(|err| DeletionError::from_inner(path, err))?; - self.cache.remove(path).await; Ok(()) } diff --git a/raphtory-graphql/src/paths.rs b/raphtory-graphql/src/paths.rs index 6db06d9f66..686b073fc7 100644 --- a/raphtory-graphql/src/paths.rs +++ b/raphtory-graphql/src/paths.rs @@ -352,7 +352,7 @@ impl ValidWriteableGraphFolder { &self, graph: MaterializedGraph, config: Config, - ) -> Result { + ) -> Result<(bool, MaterializedGraph), InternalPathValidationError> { let is_dirty = if Extension::disk_storage_enabled() { let graph_path = self.graph_folder().graph_path()?; if graph @@ -360,14 +360,15 @@ impl ValidWriteableGraphFolder { .is_some_and(|path| path == &graph_path) { self.global_path.write_metadata(&graph)?; - true + (true, graph) } else { - graph.materialize_at_with_config(self.graph_folder(), config)?; - true + dbg!(self.graph_folder().graph_path()?); + let new_graph = graph.materialize_at_with_config(self.graph_folder(), config)?; + (true, new_graph) } } else { - self.global_path.data_path()?.replace_graph(graph)?; - false + self.global_path.data_path()?.replace_graph(graph.clone())?; + (false, graph) }; Ok(is_dirty) } @@ -375,7 +376,7 @@ impl ValidWriteableGraphFolder { &self, graph: MaterializedGraph, config: Config, - ) -> Result { + ) -> Result<(bool, MaterializedGraph), PathValidationError> { self.write_graph_data_inner(graph, config) .with_path(self.local_path()) } From 97258ecd0391439c6875ac8a7e7e4705c8e86ab4 Mon Sep 17 00:00:00 2001 From: Lucas Jeub Date: Mon, 18 May 2026 12:24:48 +0200 Subject: [PATCH 16/24] moving a graph to the same name should be a no-op, not delete the graph --- raphtory-graphql/src/model/mod.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/raphtory-graphql/src/model/mod.rs b/raphtory-graphql/src/model/mod.rs index b25419818a..521f7ff355 100644 --- a/raphtory-graphql/src/model/mod.rs +++ b/raphtory-graphql/src/model/mod.rs @@ -412,8 +412,11 @@ impl Mut { let src_ns = parent_namespace(path); require_namespace_write(ctx, &data.auth_policy, src_ns, path, "move")?; // copy_graph handles dst namespace WRITE check (and src READ, which WRITE implies) - Self::copy_graph(ctx, path, new_path, overwrite).await?; - data.delete_graph(path).await?; + if path != new_path { + // moving with the same path should be a no-op, not delete the graph + Self::copy_graph(ctx, path, new_path, overwrite).await?; + data.delete_graph(path).await?; + } Ok(true) } From 1f0392cfc3ff9ef5b0d7d0174768fb1db1b98bac Mon Sep 17 00:00:00 2001 From: Lucas Jeub Date: Mon, 18 May 2026 12:27:26 +0200 Subject: [PATCH 17/24] remove dbg --- raphtory-graphql/src/paths.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/raphtory-graphql/src/paths.rs b/raphtory-graphql/src/paths.rs index 686b073fc7..26771265e2 100644 --- a/raphtory-graphql/src/paths.rs +++ b/raphtory-graphql/src/paths.rs @@ -362,7 +362,6 @@ impl ValidWriteableGraphFolder { self.global_path.write_metadata(&graph)?; (true, graph) } else { - dbg!(self.graph_folder().graph_path()?); let new_graph = graph.materialize_at_with_config(self.graph_folder(), config)?; (true, new_graph) } From 9b1f470cf7919a6b19842c3437f156a4862e0d6b Mon Sep 17 00:00:00 2001 From: Lucas Jeub Date: Mon, 18 May 2026 14:32:39 +0200 Subject: [PATCH 18/24] make sure the timeout test isn't flaky if the server happens to start quickly --- .../test_graphql/edit_graph/test_graphql.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/python/tests/test_base_install/test_graphql/edit_graph/test_graphql.py b/python/tests/test_base_install/test_graphql/edit_graph/test_graphql.py index 379558586d..e879daf999 100644 --- a/python/tests/test_base_install/test_graphql/edit_graph/test_graphql.py +++ b/python/tests/test_base_install/test_graphql/edit_graph/test_graphql.py @@ -1,6 +1,8 @@ import json import os import tempfile +import time + import pytest from utils import sort_by_gql_name_or_id from raphtory import Graph, graph_loader @@ -31,14 +33,13 @@ def test_encode_graph(): def test_failed_server_start_in_time(): tmp_work_dir = tempfile.mkdtemp() - server = None try: - with pytest.raises(Exception) as excinfo: - server = GraphServer(tmp_work_dir).start(timeout_ms=1) - assert str(excinfo.value) == "Failed to start server in 1 milliseconds" - finally: - if server: - server.stop() + start = time.perf_counter() + with GraphServer(tmp_work_dir).start(timeout_ms=1) as server: + assert server.get_client().is_server_online() + assert (time.perf_counter() - start) < 1 # generous timeout check (1s versus 1ms) + except Exception as excinfo: + assert str(excinfo) == "Failed to start server in 1 milliseconds" def test_wrong_url(): From 0a4b23207b47596273dc35603888a07abc9e0add Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 18 May 2026 16:40:41 +0000 Subject: [PATCH 19/24] chore: apply tidy-public auto-fixes --- python/python/raphtory/graphql/__init__.pyi | 15 ++++++--- python/tests/test_auth.py | 14 ++++---- .../test_graphql/edit_graph/test_graphql.py | 33 ++++++++++--------- .../test_graphql/misc/test_index_off.py | 4 +-- .../test_graph_file_time_stats.py | 6 ++-- .../test_graphql/test_metadata_dispatch.py | 14 ++++---- .../test_graphql/test_namespace.py | 16 ++++----- .../test_graphql/test_read_only_load.py | 2 +- .../test_graphql/test_server_flags.py | 18 +++++----- 9 files changed, 68 insertions(+), 54 deletions(-) diff --git a/python/python/raphtory/graphql/__init__.pyi b/python/python/raphtory/graphql/__init__.pyi index 39e07ceca4..38b0a230fb 100644 --- a/python/python/raphtory/graphql/__init__.pyi +++ b/python/python/raphtory/graphql/__init__.pyi @@ -112,24 +112,28 @@ class GraphServer(object): ) -> GraphServer: """Create and return a new object. See help(type) for accurate signature.""" - def run(self, port: int = 1736, timeout_ms: int = 180000) -> None: + def run(self, port: Optional[int] = None, timeout_ms: int = 180000) -> None: """ Run the server until completion. Arguments: - port (int): The port to use. Defaults to 1736. + port (int, optional): The port to use. If not specified, tries 1736 by default and if that is not available starts on an arbitrary port. + If specified and the port is in use, the server will fail to start. timeout_ms (int): Timeout for waiting for the server to start. Defaults to 180000. Returns: None: """ - def start(self, port: int = 1736, timeout_ms: int = 5000) -> RunningGraphServer: + def start( + self, port: Optional[int] = None, timeout_ms: int = 5000 + ) -> RunningGraphServer: """ Start the server and return a handle to it. Arguments: - port (int): the port to use. Defaults to 1736. + port (int, optional): the port to use. If not specified, tries 1736 by default and if that is not available starts on an arbitrary port. + If specified and the port is in use, the server will fail to start. timeout_ms (int): wait for server to be online. Defaults to 5000. The server is stopped if not online within timeout_ms but manages to come online as soon as timeout_ms finishes! @@ -197,6 +201,9 @@ class RunningGraphServer(object): RaphtoryClient: the client. """ + def port(self): + """Get the port the server is listening on""" + def stop(self) -> None: """ Stop the server and wait for it to finish. diff --git a/python/tests/test_auth.py b/python/tests/test_auth.py index a1dca760d3..0678c45353 100644 --- a/python/tests/test_auth.py +++ b/python/tests/test_auth.py @@ -80,7 +80,9 @@ def assert_successful_response(response: requests.Response): # TODO: implement this so we can use the with sintax def add_test_graph(port): requests.post( - raphtory_url(port), headers=WRITE_HEADERS, data=json.dumps({"query": NEW_TEST_GRAPH}) + raphtory_url(port), + headers=WRITE_HEADERS, + data=json.dumps({"query": NEW_TEST_GRAPH}), ) @@ -130,7 +132,7 @@ def test_default_read_access(query): def test_disabled_read_access(query): work_dir = tempfile.mkdtemp() with GraphServer( - work_dir, auth_public_key=PUB_KEY, require_auth_for_reads=False + work_dir, auth_public_key=PUB_KEY, require_auth_for_reads=False ).start() as server: port = server.port() add_test_graph(port) @@ -194,8 +196,8 @@ def test_update_graph(query): response = requests.post(raphtory_url(port), headers=READ_HEADERS, data=data) assert response.json()["data"] is None assert ( - response.json()["errors"][0]["message"] - == "The requested endpoint requires write access" + response.json()["errors"][0]["message"] + == "The requested endpoint requires write access" ) response = requests.post(raphtory_url(port), headers=WRITE_HEADERS, data=data) @@ -225,8 +227,8 @@ def test_mutations(query): response = requests.post(raphtory_url(port), headers=READ_HEADERS, data=data) assert response.json()["data"] is None assert ( - response.json()["errors"][0]["message"] - == "The requested endpoint requires write access" + response.json()["errors"][0]["message"] + == "The requested endpoint requires write access" ) response = requests.post(raphtory_url(port), headers=WRITE_HEADERS, data=data) diff --git a/python/tests/test_base_install/test_graphql/edit_graph/test_graphql.py b/python/tests/test_base_install/test_graphql/edit_graph/test_graphql.py index e879daf999..5fbd1882a5 100644 --- a/python/tests/test_base_install/test_graphql/edit_graph/test_graphql.py +++ b/python/tests/test_base_install/test_graphql/edit_graph/test_graphql.py @@ -37,7 +37,9 @@ def test_failed_server_start_in_time(): start = time.perf_counter() with GraphServer(tmp_work_dir).start(timeout_ms=1) as server: assert server.get_client().is_server_online() - assert (time.perf_counter() - start) < 1 # generous timeout check (1s versus 1ms) + assert ( + time.perf_counter() - start + ) < 1 # generous timeout check (1s versus 1ms) except Exception as excinfo: assert str(excinfo) == "Failed to start server in 1 milliseconds" @@ -46,8 +48,8 @@ def test_wrong_url(): with pytest.raises(Exception) as excinfo: client = RaphtoryClient("http://broken_url.com") assert ( - str(excinfo.value) - == "Could not connect to the given server - no response --error sending request for url (http://broken_url.com/)" + str(excinfo.value) + == "Could not connect to the given server - no response --error sending request for url (http://broken_url.com/)" ) @@ -157,40 +159,40 @@ def assert_graph_fetch(path): with pytest.raises(Exception) as excinfo: client.send_graph(path=path, graph=g, overwrite=True) assert ( - "Invalid path '../shivam/g': References to the parent dir are not allowed within the path" - in str(excinfo.value) + "Invalid path '../shivam/g': References to the parent dir are not allowed within the path" + in str(excinfo.value) ) path = "./shivam/g" with pytest.raises(Exception) as excinfo: client.send_graph(path=path, graph=g, overwrite=True) assert ( - "Invalid path './shivam/g': References to the current dir are not allowed within the path" - in str(excinfo.value) + "Invalid path './shivam/g': References to the current dir are not allowed within the path" + in str(excinfo.value) ) path = "shivam/../../../../investigation/g" with pytest.raises(Exception) as excinfo: client.send_graph(path=path, graph=g, overwrite=True) assert ( - "Invalid path 'shivam/../../../../investigation/g': References to the parent dir are not allowed within the path" - in str(excinfo.value) + "Invalid path 'shivam/../../../../investigation/g': References to the parent dir are not allowed within the path" + in str(excinfo.value) ) path = "//shivam/investigation/g" with pytest.raises(Exception) as excinfo: client.send_graph(path=path, graph=g, overwrite=True) assert ( - "Invalid path '//shivam/investigation/g': Double forward slashes are not allowed in path" - in str(excinfo.value) + "Invalid path '//shivam/investigation/g': Double forward slashes are not allowed in path" + in str(excinfo.value) ) path = "shivam/investigation//2024-12-12/g" with pytest.raises(Exception) as excinfo: client.send_graph(path=path, graph=g, overwrite=True) assert ( - "Invalid path 'shivam/investigation//2024-12-12/g': Double forward slashes are not allowed in path" - in str(excinfo.value) + "Invalid path 'shivam/investigation//2024-12-12/g': Double forward slashes are not allowed in path" + in str(excinfo.value) ) path = r"shivam/investigation\2024-12-12" @@ -208,8 +210,8 @@ def assert_graph_fetch(path): with pytest.raises(Exception) as excinfo: client.send_graph(path=path, graph=g, overwrite=True) assert ( - "Invalid path 'shivam/graphs/not_a_symlink_i_promise/escaped': A component of the given path was a symlink" - in str(excinfo.value) + "Invalid path 'shivam/graphs/not_a_symlink_i_promise/escaped': A component of the given path was a symlink" + in str(excinfo.value) ) @@ -869,6 +871,7 @@ def test_float_is_stable_on_roundtrip(): retrieved_float = resp["graph"]["node"]["at"]["properties"]["get"]["value"] assert retrieved_float == num + # def test_disk_graph_name(): # import pandas as pd # from raphtory import DiskGraphStorage diff --git a/python/tests/test_base_install/test_graphql/misc/test_index_off.py b/python/tests/test_base_install/test_graphql/misc/test_index_off.py index 2d3d482929..4fc4848995 100644 --- a/python/tests/test_base_install/test_graphql/misc/test_index_off.py +++ b/python/tests/test_base_install/test_graphql/misc/test_index_off.py @@ -24,6 +24,6 @@ def test_latest_and_active(): with pytest.raises(Exception) as excinfo: client.query(query) assert ( - "An operation tried to make use of the graph index but indexing has been turned off for the server" - in str(excinfo.value) + "An operation tried to make use of the graph index but indexing has been turned off for the server" + in str(excinfo.value) ) diff --git a/python/tests/test_base_install/test_graphql/test_graph_file_time_stats.py b/python/tests/test_base_install/test_graphql/test_graph_file_time_stats.py index 9cef6a1d38..04f31253d5 100644 --- a/python/tests/test_base_install/test_graphql/test_graph_file_time_stats.py +++ b/python/tests/test_base_install/test_graphql/test_graph_file_time_stats.py @@ -35,11 +35,11 @@ def test_graph_file_time_stats(): last_updated_time_fs = meta_file_stats.st_mtime * 1000 assert ( - abs(gql_created_time - created_time_fs) < 1000 + abs(gql_created_time - created_time_fs) < 1000 ), f"Mismatch in created time: FS({created_time_fs}) vs GQL({gql_created_time})" assert ( - abs(gql_last_opened_time - last_opened_time_fs) < 1000 + abs(gql_last_opened_time - last_opened_time_fs) < 1000 ), f"Mismatch in last opened time: FS({last_opened_time_fs}) vs GQL({gql_last_opened_time})" assert ( - abs(gql_last_updated_time - last_updated_time_fs) < 1000 + abs(gql_last_updated_time - last_updated_time_fs) < 1000 ), f"Mismatch in last updated time: FS({last_updated_time_fs}) vs GQL({gql_last_updated_time})" diff --git a/python/tests/test_base_install/test_graphql/test_metadata_dispatch.py b/python/tests/test_base_install/test_graphql/test_metadata_dispatch.py index e5c255bac0..0802ae4c31 100644 --- a/python/tests/test_base_install/test_graphql/test_metadata_dispatch.py +++ b/python/tests/test_base_install/test_graphql/test_metadata_dispatch.py @@ -104,10 +104,10 @@ def test_metadata_returned_for_both_disk_and_parquet_graphs(): # test still passing (metadata round-trips for either format), and # the parquet dispatch path would silently stop being exercised. assert ( - _read_is_diskgraph(disk_graph_dir) is True + _read_is_diskgraph(disk_graph_dir) is True ), "disk_graph was not saved as a disk graph" assert ( - _read_is_diskgraph(parquet_graph_dir) is False + _read_is_diskgraph(parquet_graph_dir) is False ), "parquet_graph was not saved as parquet" with GraphServer(work_dir).start() as server: @@ -139,7 +139,7 @@ def test_metadata_returned_for_both_disk_and_parquet_graphs(): meta_cached = _list_metadata_by_path(client) assert ( - meta_cached == meta + meta_cached == meta ), "cached-path metadata should match the on-disk-path metadata" @@ -162,7 +162,7 @@ def test_metadata_update_in_single_segment_returns_latest(): client = server.get_client() meta = _list_metadata_by_path(client) assert ( - meta["g"]["version"] == "v2" + meta["g"]["version"] == "v2" ), f"expected latest in-segment value 'v2', got {meta['g'].get('version')!r}" @@ -189,7 +189,7 @@ def test_metadata_update_across_flushes_returns_newest_segment(): client = server.get_client() meta = _list_metadata_by_path(client) assert ( - meta["g"]["version"] == "v2" + meta["g"]["version"] == "v2" ), f"expected newest-segment value 'v2', got {meta['g'].get('version')!r}" @@ -213,7 +213,7 @@ def test_metadata_many_updates_across_flushes_returns_last(): client = server.get_client() meta = _list_metadata_by_path(client) assert ( - meta["g"]["version"] == "v499" + meta["g"]["version"] == "v499" ), f"expected last-write value 'v499', got {meta['g'].get('version')!r}" @@ -281,5 +281,5 @@ def test_metadata_mixed_keys_across_flushes(): meta = _list_metadata_by_path(client) assert meta["g"]["untouched"] == "stable" assert ( - meta["g"]["bumped"] == "new" + meta["g"]["bumped"] == "new" ), f"expected updated value 'new', got {meta['g'].get('bumped')!r}" diff --git a/python/tests/test_base_install/test_graphql/test_namespace.py b/python/tests/test_base_install/test_graphql/test_namespace.py index 30ed9a3bc8..c3ed99538c 100644 --- a/python/tests/test_base_install/test_graphql/test_namespace.py +++ b/python/tests/test_base_install/test_graphql/test_namespace.py @@ -294,8 +294,8 @@ def test_wrong_paths(): with pytest.raises(Exception) as excinfo: client.query(query) assert ( - "Only relative paths are allowed to be used within the working_dir: /test" - in str(excinfo.value) + "Only relative paths are allowed to be used within the working_dir: /test" + in str(excinfo.value) ) query = """{ @@ -309,8 +309,8 @@ def test_wrong_paths(): with pytest.raises(Exception) as excinfo: client.query(query) assert ( - "References to the parent dir are not allowed within the path: test/../../" - in str(excinfo.value) + "References to the parent dir are not allowed within the path: test/../../" + in str(excinfo.value) ) query = """{ @@ -324,8 +324,8 @@ def test_wrong_paths(): with pytest.raises(Exception) as excinfo: client.query(query) assert ( - "References to the current dir are not allowed within the path: ./test" - in str(excinfo.value) + "References to the current dir are not allowed within the path: ./test" + in str(excinfo.value) ) query = """{ @@ -339,8 +339,8 @@ def test_wrong_paths(): with pytest.raises(Exception) as excinfo: client.query(query) assert ( - "The path to the graph contains a subpath to an existing graph: test/second/internal/graph1" - in str(excinfo.value) + "The path to the graph contains a subpath to an existing graph: test/second/internal/graph1" + in str(excinfo.value) ) diff --git a/python/tests/test_base_install/test_graphql/test_read_only_load.py b/python/tests/test_base_install/test_graphql/test_read_only_load.py index beb72d4351..af456943a5 100644 --- a/python/tests/test_base_install/test_graphql/test_read_only_load.py +++ b/python/tests/test_base_install/test_graphql/test_read_only_load.py @@ -146,7 +146,7 @@ def test_read_only_load_blocks_all_mutation_paths(): mutate() msg = str(exc.value).lower() assert ( - "locked" in msg or "immutable" in msg + "locked" in msg or "immutable" in msg ), f"{name} did not raise a locked/immutable error: {exc.value}" diff --git a/python/tests/test_base_install/test_graphql/test_server_flags.py b/python/tests/test_base_install/test_graphql/test_server_flags.py index 07ee01d486..409899b6bb 100644 --- a/python/tests/test_base_install/test_graphql/test_server_flags.py +++ b/python/tests/test_base_install/test_graphql/test_server_flags.py @@ -261,8 +261,8 @@ def test_disable_lists_all_resolvers(): with pytest.raises(Exception) as excinfo: client.query(query) assert ( - "Bulk list endpoints are disabled on this server. Use `page` instead." - in str(excinfo.value) + "Bulk list endpoints are disabled on this server. Use `page` instead." + in str(excinfo.value) ), f"{name} did not reject with the expected error: {excinfo.value}" @@ -310,7 +310,9 @@ def test_disable_batching(): with GraphServer(work_dir, disable_batching=True).start() as server: server.get_client().query("{ version }") - status, body = batch_query(server.port(), [{"query": "{ version }"}, {"query": "{ version }"}]) + status, body = batch_query( + server.port(), [{"query": "{ version }"}, {"query": "{ version }"}] + ) assert status == 400 assert "Query batching is disabled on this server" in str(body) @@ -356,8 +358,8 @@ def test_max_directives_per_field(): with pytest.raises(Exception) as excinfo: client.query("{ version @skip(if: false) @include(if: true) }") assert ( - "number of directives on the field `version` cannot be greater than `1`" - in str(excinfo.value) + "number of directives on the field `version` cannot be greater than `1`" + in str(excinfo.value) ) @@ -368,9 +370,9 @@ def test_max_directives_per_field(): def test_concurrency_flags_smoke(): work_dir = tempfile.mkdtemp() with GraphServer( - work_dir, - heavy_query_limit=4, - exclusive_writes=True, + work_dir, + heavy_query_limit=4, + exclusive_writes=True, ).start() as server: client = server.get_client() make_graph(client) From cd5eddf3166c1f6d72803cf72dd2ca3f820ac345 Mon Sep 17 00:00:00 2001 From: Lucas Jeub Date: Tue, 19 May 2026 08:54:05 +0200 Subject: [PATCH 20/24] delete empty tests file --- python/tests/test_permissions.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 python/tests/test_permissions.py diff --git a/python/tests/test_permissions.py b/python/tests/test_permissions.py deleted file mode 100644 index e69de29bb2..0000000000 From 176ab375906313a3fb14fa495ccdae3654b79985 Mon Sep 17 00:00:00 2001 From: Lucas Jeub Date: Tue, 19 May 2026 08:54:32 +0200 Subject: [PATCH 21/24] make sure we don't return the unfiltered graph when only filtered access is available --- raphtory-graphql/src/data.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/raphtory-graphql/src/data.rs b/raphtory-graphql/src/data.rs index c7e5b4f245..8b4d1056e7 100644 --- a/raphtory-graphql/src/data.rs +++ b/raphtory-graphql/src/data.rs @@ -1,6 +1,6 @@ use crate::{ auth::ContextValidation, - auth_policy::{AuthorizationPolicy, GraphPermission, NamespacePermission}, + auth_policy::{AuthorizationPolicy, GraphPermission, NamespacePermission, PermissionLevel}, cache::GraphCache, config::app_config::AppConfig, graph::GraphWithVectors, @@ -388,6 +388,10 @@ pub(crate) enum PermissionError { /// Caller has read-only access but the operation requires write. #[error("Access denied: WRITE permission required for graph '{graph}'")] GraphWriteRequired { graph: String }, + + /// Caller has filtered read-only access but the opration requires unfiltered read + #[error("Access denied: unfiltered READ permissions required for graph '{graph}'")] + GraphUnfilteredReadRequired { graph: String }, /// Caller lacks write permission on the destination namespace. #[error( "Access denied: WRITE required on namespace '{namespace}' to {operation} graph '{graph}'" @@ -630,7 +634,12 @@ impl Data { ctx: &Context<'_>, path: &str, ) -> async_graphql::Result { - require_at_least_read(ctx, &self.auth_policy, path)?; + let res = require_at_least_read(ctx, &self.auth_policy, path)?; + if res.level() < PermissionLevel::Read { + Err(PermissionError::GraphUnfilteredReadRequired { + graph: path.to_string(), + })?; + } let graph = self.get_graph(path).await?; Ok(graph) } From 2bba27868ac74b9ac8378a01122240e997b68291 Mon Sep 17 00:00:00 2001 From: Lucas Jeub Date: Tue, 19 May 2026 09:41:52 +0200 Subject: [PATCH 22/24] get list of nodes and edges by querying the graph on the server to avoid any potential ordering issues in the test --- raphtory-graphql/src/lib.rs | 57 +++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/raphtory-graphql/src/lib.rs b/raphtory-graphql/src/lib.rs index 4ab71b0e9f..b342a59dce 100644 --- a/raphtory-graphql/src/lib.rs +++ b/raphtory-graphql/src/lib.rs @@ -1305,6 +1305,63 @@ mod graphql_test { save_graphs_to_work_dir(&data, &graphs).await.unwrap(); let schema = App::create_schema().data(data).finish().unwrap(); + let all = r#"{ + graph(path: "graph1") { + nodes { + list { + name + } + } + edges { + list { + id + } + } + } + }"#; + + let res = schema.execute(Request::new(all)).await; + let data = res.data.into_json().unwrap(); + + let all_nodes: Vec<_> = data + .get("graph") + .unwrap() + .get("nodes") + .unwrap() + .get("list") + .unwrap() + .as_array() + .unwrap() + .iter() + .filter_map(|v| v.get("name").unwrap().as_str()) + .collect(); + + let all_edges: Vec<(_, _)> = data + .get("graph") + .unwrap() + .get("edges") + .unwrap() + .get("list") + .unwrap() + .as_array() + .unwrap() + .iter() + .filter_map(|v| v.get("id").unwrap().as_array()) + .filter_map(|ids| ids.iter().filter_map(|v| v.as_u64()).collect_tuple()) + .collect(); + + // make sure we have the correct edges + assert_eq!( + all_edges.iter().cloned().sorted().collect_vec(), + [(1, 2), (2, 4), (3, 2), (3, 6), (4, 5), (4, 6), (5, 6),] + ); + + // make sure we have the correct nodes + assert_eq!( + all_nodes.iter().copied().sorted().collect_vec(), + ["1", "2", "3", "4", "5", "6"] + ); + let req = r#" { graph(path: "graph1") { From d45e9842868ae224f457805dbd57491061b280cb Mon Sep 17 00:00:00 2001 From: Lucas Jeub Date: Tue, 19 May 2026 12:45:44 +0200 Subject: [PATCH 23/24] explicitly control the drop order in test to make sure graph and data are dropped before the directory is deleted --- raphtory-graphql/src/model/graph/mutable_graph.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/raphtory-graphql/src/model/graph/mutable_graph.rs b/raphtory-graphql/src/model/graph/mutable_graph.rs index f5f429bc65..c33146e59c 100644 --- a/raphtory-graphql/src/model/graph/mutable_graph.rs +++ b/raphtory-graphql/src/model/graph/mutable_graph.rs @@ -918,7 +918,7 @@ mod tests { #[tokio::test] async fn test_add_nodes_empty_list() { - let (mutable_graph, _data, _tmp_dir, embedding_server) = create_mutable_graph(1745).await; + let (mutable_graph, data, tmp_dir, embedding_server) = create_mutable_graph(1745).await; let nodes = vec![]; let result = mutable_graph.add_nodes(nodes).await; @@ -926,6 +926,11 @@ mod tests { assert!(result.is_ok()); assert!(result.unwrap()); embedding_server.stop().await; + + // control the drop order + drop(mutable_graph); + drop(data); + drop(tmp_dir); } #[tokio::test] From a97712acd77d847c1d9ae5bae5b2fb480cd72ef2 Mon Sep 17 00:00:00 2001 From: Lucas Jeub Date: Tue, 19 May 2026 14:47:11 +0200 Subject: [PATCH 24/24] fix drop order problem in tests that would cause the directory to be deleted before the graph is dropped and enable panic on drop for graphql tests by default --- raphtory-graphql/Cargo.toml | 2 +- .../src/model/graph/mutable_graph.rs | 48 ++++++++++++------- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/raphtory-graphql/Cargo.toml b/raphtory-graphql/Cargo.toml index 390424e323..a6e625e239 100644 --- a/raphtory-graphql/Cargo.toml +++ b/raphtory-graphql/Cargo.toml @@ -71,7 +71,7 @@ rust-embed = { workspace = true } parking_lot = { workspace = true } tempfile = { workspace = true } pretty_assertions = { workspace = true } -raphtory = { workspace = true, features = ["test-utils"] } +raphtory = { workspace = true, features = ["test-utils", "panic-on-drop"] } arrow-array = { workspace = true } [features] diff --git a/raphtory-graphql/src/model/graph/mutable_graph.rs b/raphtory-graphql/src/model/graph/mutable_graph.rs index c33146e59c..91a3a111ac 100644 --- a/raphtory-graphql/src/model/graph/mutable_graph.rs +++ b/raphtory-graphql/src/model/graph/mutable_graph.rs @@ -865,7 +865,7 @@ mod tests { template::DocumentTemplate, }, }; - use tempfile::tempdir; + use tempfile::{tempdir, TempDir}; fn fake_embedding(_: &str) -> Vec { vec![1.0] @@ -876,9 +876,16 @@ mod tests { graph.into() } - async fn create_mutable_graph( - port: u16, - ) -> (GqlMutableGraph, Data, tempfile::TempDir, EmbeddingServer) { + /// Struct returned by `create_mutable_graph` to make sure the directory is dropped last. + /// Otherwise, the drop of the graph might fail as the directory no longer exists. + pub struct GraphTestContext { + mutable_graph: GqlMutableGraph, + embedding_server: EmbeddingServer, + data: Data, + tmp_dir: TempDir, + } + + async fn create_mutable_graph(port: u16) -> GraphTestContext { let graph = create_test_graph(); let tmp_dir = tempdir().unwrap(); @@ -913,12 +920,21 @@ mod tests { let graph_with_vectors = data.get_graph_for_test(graph_name).await.unwrap(); let mutable_graph = GqlMutableGraph::from(graph_with_vectors); - (mutable_graph, data, tmp_dir, embedding_server) + GraphTestContext { + mutable_graph, + data, + tmp_dir, + embedding_server, + } } #[tokio::test] async fn test_add_nodes_empty_list() { - let (mutable_graph, data, tmp_dir, embedding_server) = create_mutable_graph(1745).await; + let GraphTestContext { + mutable_graph, + embedding_server, + .. + } = create_mutable_graph(0).await; let nodes = vec![]; let result = mutable_graph.add_nodes(nodes).await; @@ -926,17 +942,13 @@ mod tests { assert!(result.is_ok()); assert!(result.unwrap()); embedding_server.stop().await; - - // control the drop order - drop(mutable_graph); - drop(data); - drop(tmp_dir); } #[tokio::test] #[ignore = "TODO: #2384"] async fn test_add_nodes_simple() { - let (mutable_graph, _data, _tmp_dir, es) = create_mutable_graph(1746).await; + let context = create_mutable_graph(1746).await; + let mutable_graph = &context.mutable_graph; let nodes = vec![ NodeAddition { @@ -978,13 +990,14 @@ mod tests { assert!(result.is_ok()); assert!(result.unwrap().get_documents().await.unwrap().len() == 2); - es.stop().await; + context.embedding_server.stop().await; } #[tokio::test] #[ignore = "TODO: #2384"] async fn test_add_nodes_with_properties() { - let (mutable_graph, _data, _tmp_dir, es) = create_mutable_graph(1747).await; + let context = create_mutable_graph(1747).await; + let mutable_graph = &context.mutable_graph; let nodes = vec![ NodeAddition { @@ -1054,13 +1067,14 @@ mod tests { assert!(result.is_ok()); assert!(result.unwrap().get_documents().await.unwrap().len() == 3); - es.stop().await; + context.embedding_server.stop().await; } #[tokio::test] #[ignore = "TODO: #2384"] async fn test_add_edges_simple() { - let (mutable_graph, _data, _tmp_dir, es) = create_mutable_graph(1748).await; + let context = create_mutable_graph(1748).await; + let mutable_graph = &context.mutable_graph; // First add some nodes. let nodes = vec![ @@ -1135,6 +1149,6 @@ mod tests { assert!(result.is_ok()); assert!(result.unwrap().get_documents().await.unwrap().len() == 2); - es.stop().await; + context.embedding_server.stop().await; } }