diff --git a/Cargo.lock b/Cargo.lock index 1a5ce66f44..98784d3686 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2651,8 +2651,9 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] name = "dynamic-graphql" -version = "0.10.1" -source = "git+https://github.com/miratepuffin/dynamic-graphql?branch=add-arg-descriptions#69a07c5fe3c16b4baf76f676c96cde5865cae1de" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d58055cecef1736f42bee8819059054e18239cda3d5de56b0c8836c2a2322b" dependencies = [ "async-graphql", "dynamic-graphql-derive", @@ -2661,8 +2662,9 @@ dependencies = [ [[package]] name = "dynamic-graphql-derive" -version = "0.10.1" -source = "git+https://github.com/miratepuffin/dynamic-graphql?branch=add-arg-descriptions#69a07c5fe3c16b4baf76f676c96cde5865cae1de" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff1e2aa3d6affef7ce88263e83c0110b11f57409cf25aa81689bcc5f023a602e" dependencies = [ "Inflector", "darling 0.20.11", @@ -6121,6 +6123,18 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quick_cache" +version = "0.6.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a70b1b8b47e31d0498ecbc3c5470bb931399a8bfed1fd79d1717a61ce7f96e3" +dependencies = [ + "ahash", + "equivalent", + "hashbrown 0.16.1", + "parking_lot", +] + [[package]] name = "quinn" version = "0.11.9" @@ -6536,6 +6550,7 @@ dependencies = [ "clap", "config", "crossbeam-channel", + "dashmap", "dynamic-graphql", "futures-util", "itertools 0.13.0", @@ -6551,6 +6566,7 @@ dependencies = [ "poem", "pretty_assertions", "pyo3", + "quick_cache", "raphtory", "raphtory-api", "raphtory-storage", diff --git a/Cargo.toml b/Cargo.toml index 4baf275165..c953d1e2c0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,10 +70,11 @@ raphtory-storage = { version = "0.18.0", path = "raphtory-storage", default-feat raphtory-itertools = { version = "0.18.0", path = "raphtory-itertools" } clam-core = { version = "0.18.0", path = "clam-core" } optd-core = { version = "0.18.0", path = "optd/optd/core" } + async-graphql = { version = "7.2.1", features = ["dynamic-schema"] } bincode = { version = "2", features = ["serde"] } async-graphql-poem = "7.2.1" -dynamic-graphql = { git = "https://github.com/miratepuffin/dynamic-graphql", branch = "add-arg-descriptions" } +dynamic-graphql = "0.10.2" derive_more = "2.1.1" tikv-jemallocator = "0.6.1" reqwest = { version = "0.12.28", default-features = false, features = [ @@ -167,7 +168,6 @@ pest = "2.8.6" pest_derive = "2.8.6" minijinja = "2.18.0" minijinja-contrib = { version = "2.18.0", features = ["datetime"] } - lancedb = { version = "0.27.2", features = [] } heed = "0.22.0" sqlparser = "0.59.0" @@ -185,6 +185,7 @@ arrow-data = { version = "57.3.0" } arrow-select = { version = "57.3.0" } serde_arrow = { version = "0.13.7", features = ["arrow-57"] } moka = { version = "0.12.15", features = ["future"] } +quick_cache = "0.6.21" indexmap = { version = "2.13.0", features = ["rayon"] } fake = { version = "3.1.0", features = ["chrono"] } strsim = { version = "0.11.1" } diff --git a/db4-storage/Cargo.toml b/db4-storage/Cargo.toml index fc03b2f476..3cf61c6661 100644 --- a/db4-storage/Cargo.toml +++ b/db4-storage/Cargo.toml @@ -46,5 +46,6 @@ rayon.workspace = true test-log.workspace = true [features] -test-utils = ["dep:proptest", "dep:chrono"] -default = ["test-utils"] +test-utils = ["dep:proptest", "dep:chrono", "panic-on-drop"] +panic-on-drop = [] + diff --git a/db4-storage/src/pages/graph_prop_store.rs b/db4-storage/src/pages/graph_prop_store.rs index c91f436dd5..05c2a42cdf 100644 --- a/db4-storage/src/pages/graph_prop_store.rs +++ b/db4-storage/src/pages/graph_prop_store.rs @@ -9,6 +9,7 @@ use crate::{ }; use raphtory_api::core::entities::properties::meta::Meta; use std::{ + marker::PhantomData, path::{Path, PathBuf}, sync::Arc, }; @@ -23,10 +24,7 @@ pub struct GraphPropStorageInner { /// Stores graph prop metadata (prop name -> prop id mappings). meta: Arc, - - path: Option, - - ext: EXT, + _ext: PhantomData, } impl, EXT: PersistenceStrategy> @@ -37,9 +35,8 @@ impl, EXT: PersistenceStrategy> Self { page, - path: path.map(|p| p.to_path_buf()), meta, - ext, + _ext: PhantomData, } } @@ -52,9 +49,8 @@ impl, EXT: PersistenceStrategy> path.as_ref(), ext.clone(), )?), - path: Some(path.as_ref().to_path_buf()), meta: graph_props_meta, - ext, + _ext: PhantomData, }) } diff --git a/db4-storage/src/pages/mod.rs b/db4-storage/src/pages/mod.rs index 7fa0ab7114..1752db4526 100644 --- a/db4-storage/src/pages/mod.rs +++ b/db4-storage/src/pages/mod.rs @@ -42,8 +42,21 @@ pub mod session; #[cfg(any(test, feature = "test-utils"))] pub mod test_utils; -// graph // (node/edges) // segment // layer_ids (0, 1, 2, ...) // actual graphy bits +#[cfg(any(test, feature = "panic-on-drop"))] +macro_rules! drop_error { + ($($arg:tt)*) => {{ + panic!($($arg)*) + }}; +} + +#[cfg(not(any(test, feature = "panic-on-drop")))] +macro_rules! drop_error { + ($($arg:tt)*) => {{ + eprintln!($($arg)*) + }}; +} +// graph // (node/edges) // segment // layer_ids (0, 1, 2, ...) // actual graphy bits #[derive(Debug)] pub struct GraphStore< NS: NodeSegmentOps, @@ -372,7 +385,7 @@ impl< let checkpoint_lsn = match wal.log_shutdown_checkpoint() { Ok(lsn) => lsn, Err(err) => { - eprintln!("Failed to log shutdown checkpoint in drop: {err}"); + drop_error!("Failed to log shutdown checkpoint in drop: {err}"); return; } }; @@ -381,7 +394,7 @@ impl< let flush_lsn = wal.position(); if let Err(err) = wal.flush(flush_lsn) { - eprintln!("Failed to flush checkpoint record in drop: {err}"); + drop_error!("Failed to flush checkpoint record in drop: {err}"); return; } @@ -390,12 +403,12 @@ impl< control_file.set_db_state(DBState::Shutdown); if let Err(err) = control_file.save() { - eprintln!("Failed to save control file in drop: {err}"); + drop_error!("Failed to save control file in drop: {err}"); return; } } Err(err) => { - eprintln!("Failed to flush storage in drop: {err}") + drop_error!("Failed to flush storage in drop: {err}"); } } } @@ -458,4 +471,10 @@ mod test { assert_eq!(actual, expected); } + + #[test] + #[should_panic] + fn test_drop_error() { + drop_error!("failed"); + } } diff --git a/docs/user-guide/graphql/2_run-server.md b/docs/user-guide/graphql/2_run-server.md index dca048d36d..33134c6998 100644 --- a/docs/user-guide/graphql/2_run-server.md +++ b/docs/user-guide/graphql/2_run-server.md @@ -8,11 +8,14 @@ Before reading this topic, please ensure you are familiar with: ## Saving your Raphtory graph into a directory -You will need some test data to complete the following examples. This can be your own data or one of the examples in the Raphtory documentation. +You will need some test data to complete the following examples. This can be your own data or one of the examples in the +Raphtory documentation. -Once your data is loaded into a Raphtory graph, the graph needs to be saved into your working directory. This can be done with the following code, where `g` is your graph: +Once your data is loaded into a Raphtory graph, the graph needs to be saved into your working directory. This can be +done with the following code, where `g` is your graph: /// tab | :fontawesome-brands-python: Python + ```{.python notest} import os working_dir = "graphs/" @@ -21,6 +24,7 @@ if not os.path.exists(working_dir): os.makedirs(working_dir) g.save_to_file(working_dir + "your_graph") ``` + /// ## Starting a server @@ -39,24 +43,31 @@ This option is the simplist and provides the most configuration options. ### Start a server in Python -If you have a [`GraphServer`][raphtory.graphql.GraphServer] object you can use either the [`.run()`][raphtory.graphql.GraphServer.run] or [`.start()`][raphtory.graphql.GraphServer.start] functions to start a GraphQL sever and Raphtory UI. +If you have a [`GraphServer`][raphtory.graphql.GraphServer] object you can use either the [ +`.run()`][raphtory.graphql.GraphServer.run] or [`.start()`][raphtory.graphql.GraphServer.start] functions to start a +GraphQL sever and Raphtory UI. -Below is an example of how to start the server and send a Raphtory graph to the server, where `new_graph` is your Raphtory graph object. +Below is an example of how to start the server and send a Raphtory graph to the server, where `new_graph` is your +Raphtory graph object. /// tab | :fontawesome-brands-python: Python + ```{.python notest} tmp_work_dir = tempfile.mkdtemp() -with GraphServer(tmp_work_dir, tracing=True).start(): - client = RaphtoryClient("http://localhost:1736") +with GraphServer(tmp_work_dir, tracing=True).start() as server: + client = server.get_client() client.send_graph(path="g", graph=new_graph) query = """{graph(path: "g") {nodes {list {name}}}}""" client.query(query) ``` + /// You can set the port in `RaphtoryClient()` to the port the GraphQL server should run on. -The `path` parameter is always the graph in your server that you would like to read or update. So in this example, we want to send `new_graph` to graph `g` on the server to update it. +The `path` parameter is always the graph in your server that you would like to read or update. So in this example, we +want to send `new_graph` to graph `g` on the server to update it. -The `graph` parameter is set to the Raphtory graph that you would like to send. An additional `overwrite` parameter can be stated if we want this new graph to overwrite the old graph. +The `graph` parameter is set to the Raphtory graph that you would like to send. An additional `overwrite` parameter can +be stated if we want this new graph to overwrite the old graph. diff --git a/docs/user-guide/graphql/3_writing-queries.md b/docs/user-guide/graphql/3_writing-queries.md index 26c18282e4..f8488bd057 100644 --- a/docs/user-guide/graphql/3_writing-queries.md +++ b/docs/user-guide/graphql/3_writing-queries.md @@ -2,13 +2,18 @@ The GraphQL API largely follows the same patterns as the Python API but has a few key differences. -In GraphQL, there are two different types of requests: a query to search through your data or a mutation of your data. Only the top-level fields in mutation operations are allowed to cause side effects. To accommodate this, in the Raphtory API you can make queries to graphs or metagraphs but must make changes using a mutable graph, node or edge. +In GraphQL, there are two different types of requests: a query to search through your data or a mutation of your data. +Only the top-level fields in mutation operations are allowed to cause side effects. To accommodate this, in the Raphtory +API you can make queries to graphs or metagraphs but must make changes using a mutable graph, node or edge. -This division means that the distinction between Graphs and GraphViews is less important in GraphQL and all non-mutable Graph endpoints are GraphViews while MutableGraphs are used for mutation operations. This is also true for Nodes and Edges and their respective views. Graphs can be further distinguished as either `PERSISTENT` or `EVENT` types. +This division means that the distinction between Graphs and GraphViews is less important in GraphQL and all non-mutable +Graph endpoints are GraphViews while MutableGraphs are used for mutation operations. This is also true for Nodes and +Edges and their respective views. Graphs can be further distinguished as either `PERSISTENT` or `EVENT` types. ## Graphical playground -When you start a GraphQL server, you can find your GraphQL UI in the browser at `localhost:1736/playground` or an alternative port if you specified one. +When you start a GraphQL server, you can find your GraphQL UI in the browser at `localhost:1736/playground` or an +alternative port if you specified one. An annotated schema is available from the documentation tab in the left hand menu of the playground. @@ -21,6 +26,7 @@ Here are some example queries to get you started: ### List of all the nodes /// tab | ![GraphQL](https://img.icons8.com/ios-filled/15/graphql.png) GraphQL + ``` query { graph(path: "your_graph") { @@ -32,6 +38,7 @@ query { } } ``` + /// ### List of all the edges, with specific node properties @@ -39,6 +46,7 @@ query { To find nodes with `age`: /// tab | ![GraphQL](https://img.icons8.com/ios-filled/15/graphql.png) GraphQL + ``` query { graph(path: "your_graph") { @@ -65,6 +73,7 @@ query { } } ``` + /// This will return something like: @@ -107,6 +116,7 @@ All the queries that can be done in Python can also be done in GraphQL. Here is an example: /// tab | ![GraphQL](https://img.icons8.com/ios-filled/15/graphql.png) GraphQL + ``` graphql query { graph(path: "your_graph") { @@ -120,20 +130,23 @@ query { } } ``` + /// /// tab | :fontawesome-brands-python: Python + ```{.python notest} g.node("Ben").properties.get("age") ``` + /// ### Examine the metadata of a node Metadata does not change over the lifetime of an object. You can request it with a query like the following: - /// tab | ![GraphQL](https://img.icons8.com/ios-filled/15/graphql.png) GraphQL + ``` { graph(path: "traffic_graph") { @@ -151,67 +164,69 @@ Metadata does not change over the lifetime of an object. You can request it with } } ``` + /// Which will return something like: !!! Output - ```json - { - "data": { - "graph": { - "nodes": { - "list": [ - { - "name": "ServerA", - "metadata": { - "values": [ - { - "key": "datasource", - "value": "network_traffic_edges.csv" - }, - { - "key": "server_name", - "value": "Alpha" - }, - { - "key": "hardware_type", - "value": "Blade Server" - } - ] - } - }, - { - "name": "ServerB", - "metadata": { - "values": [ - { - "key": "datasource", - "value": "network_traffic_edges.csv" - }, - { - "key": "server_name", - "value": "Beta" - }, - { - "key": "hardware_type", - "value": "Rack Server" - } - ] +```json +{ +"data": { + "graph": { + "nodes": { + "list": [ + { + "name": "ServerA", + "metadata": { + "values": [ + { + "key": "datasource", + "value": "network_traffic_edges.csv" + }, + { + "key": "server_name", + "value": "Alpha" + }, + { + "key": "hardware_type", + "value": "Blade Server" } + ] } + }, + { + "name": "ServerB", + "metadata": { + "values": [ + { + "key": "datasource", + "value": "network_traffic_edges.csv" + }, + { + "key": "server_name", + "value": "Beta" + }, + { + "key": "hardware_type", + "value": "Rack Server" + } ] + } } - } + ] } } - ``` +} +} +``` ### Examine the properties of a node Properties can change over time so it is often useful to make a query for a specific time or window. /// tab | ![GraphQL](https://img.icons8.com/ios-filled/15/graphql.png) GraphQL + ``` { graph(path: "traffic_graph") { @@ -231,114 +246,125 @@ Properties can change over time so it is often useful to make a query for a spec } } ``` + /// Which will return something like: !!! Output - ```json - { - "data": { - "graph": { - "at": { - "nodes": { - "list": [ +```json +{ +"data": { + "graph": { + "at": { + "nodes": { + "list": [ + { + "name": "ServerA", + "properties": { + "values": [] + } + }, + { + "name": "ServerB", + "properties": { + "values": [ { - "name": "ServerA", - "properties": { - "values": [] - } + "key": "OS_version", + "value": "Red Hat 8.1" }, { - "name": "ServerB", - "properties": { - "values": [ - { - "key": "OS_version", - "value": "Red Hat 8.1" - }, - { - "key": "primary_function", - "value": "Web Server" - }, - { - "key": "uptime_days", - "value": 45 - } - ] - } + "key": "primary_function", + "value": "Web Server" }, { - "name": "ServerC", - "properties": { - "values": [] - } + "key": "uptime_days", + "value": 45 } - ] + ] } - } + }, + { + "name": "ServerC", + "properties": { + "values": [] + } + } + ] } } } - ``` +} +} +``` ### Querying GraphQL in Python -You can also send GraphQL queries in Python directl using the [`.query()`][raphtory.graphql.RaphtoryClient.query] function on a `RaphtoryClient`. The following example shows you how to do this: +You can also send GraphQL queries in Python directl using the [`.query()`][raphtory.graphql.RaphtoryClient.query] +function on a `RaphtoryClient`. The following example shows you how to do this: /// tab | :fontawesome-brands-python: Python + ```{.python notest} from raphtory.graphql import GraphServer -with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") +with GraphServer(work_dir).start() as server: + client = server.get_client() query = """{graph(path: "graph") { created lastOpened lastUpdated }}""" result = client.query(query) ``` + /// -Pass your graph object string into the `client.query()` method to execute the GraphQL query and retrieve the result in a python dictionary object. +Pass your graph object string into the `client.query()` method to execute the GraphQL query and retrieve the result in a +python dictionary object. !!! Output - ```output - {'graph': {'created': 1729075008085, 'lastOpened': 1729075036222, 'lastUpdated': 1729075008085}} - ``` +```output +{'graph': {'created': 1729075008085, 'lastOpened': 1729075036222, 'lastUpdated': 1729075008085}} +``` ## Mutation requests You can also mutate your graph. This can be done both in the GraphQL IDE and in Python. -From GraphQL these operations are available from the [Mutation root](../../../reference/graphql/graphql_API/#mutation-mutroot) which operates on mutable objects by specified by a path. +From GraphQL these operations are available from +the [Mutation root](../../../reference/graphql/graphql_API/#mutation-mutroot) which operates on mutable objects by +specified by a path. !!! note - Some methods to mutate the graph are exclusive to Python. +Some methods to mutate the graph are exclusive to Python. ### Sending a graph You can send a graph to the server and overwrite an existing graph if needed. /// tab | :fontawesome-brands-python: Python + ```{.python notest} from raphtory.graphql import GraphServer tmp_work_dir = tempfile.mkdtemp() -with GraphServer(tmp_work_dir).start(): - client = RaphtoryClient("http://localhost:1736") +with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() g = Graph() g.add_edge(1, "bob", "emma") g.add_edge(2, "sally", "tony") client.send_graph(path="g", graph=g, overwrite=True) ``` + /// To check your query: /// tab | :fontawesome-brands-python: Python + ```{.python notest} query = """{graph(path: "g") {nodes {list {name}}}}""" client.query(query) ``` + /// This should return: @@ -365,36 +391,43 @@ This should return: You can retrieve graphs from a "path" on the server which returns a Python Raphtory graph object. /// tab | :fontawesome-brands-python: Python + ```{.python notest} g = client.receive_graph("path/to/graph") g.edge("sally", "tony") ``` + /// ### Creating a new graph This is an example of how to create a new graph in the server. -The first parameter is the path of the graph to be created and the second parameter is the type of graph that should be created, this will either be _EVENT_ or _PERSISTENT_. +The first parameter is the path of the graph to be created and the second parameter is the type of graph that should be +created, this will either be _EVENT_ or _PERSISTENT_. An explanation of the different types of graph can be found [here](../../user-guide/persistent-graph/1_intro.md) /// tab | ![GraphQL](https://img.icons8.com/ios-filled/15/graphql.png) GraphQL + ```graphql mutation { newGraph(path: "new_graph", graphType: PERSISTENT) } ``` + /// /// tab | :fontawesome-brands-python: Python + ```{.python notest} from raphtory.graphql import GraphServer work_dir = tempfile.mkdtemp() -with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") +with GraphServer(work_dir).start() as server: + client = server.get_client() client.new_graph("path/to/new_graph", "EVENT") ``` + /// The returning result to confirm that a new graph has been created: @@ -414,22 +447,26 @@ The returning result to confirm that a new graph has been created: It is possible to move a graph to a new path on the server. /// tab | ![GraphQL](https://img.icons8.com/ios-filled/15/graphql.png) GraphQL + ```graphql mutation { moveGraph(path: "graph", newPath: "new_path") } ``` + /// /// tab | :fontawesome-brands-python: Python + ```{.python notest} from raphtory.graphql import GraphServer work_dir = tempfile.mkdtemp() -with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") +with GraphServer(work_dir).start() as server: + client = server.get_client() client.move_graph("path/to/graph", "path/to/new_path) ``` + /// The returning GraphQL result to confirm that the graph has been moved: @@ -449,22 +486,26 @@ The returning GraphQL result to confirm that the graph has been moved: It is possible to copy a graph to a new path on the server. /// tab | ![GraphQL](https://img.icons8.com/ios-filled/15/graphql.png) GraphQL + ```graphql mutation { copyGraph(path: "graph", newPath: "new_path") } ``` + /// /// tab | :fontawesome-brands-python: Python + ```{.python notest} from raphtory.graphql import GraphServer work_dir = tempfile.mkdtemp() -with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") +with GraphServer(work_dir).start() as server: + client = server.get_client() client.copy_graph("path/to/graph", "path/to/new_path) ``` + /// The returning GraphQL result to confirm that the graph has been copied: @@ -484,22 +525,26 @@ The returning GraphQL result to confirm that the graph has been copied: It is possible to delete a graph on the server. /// tab | ![GraphQL](https://img.icons8.com/ios-filled/15/graphql.png) GraphQL + ```graphql mutation { deleteGraph(path: "graph") } ``` + /// /// tab | :fontawesome-brands-python: Python + ```{.python notest} from raphtory.graphql import GraphServer work_dir = tempfile.mkdtemp() -with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") +with GraphServer(work_dir).start() as server: + client = server.get_client() client.delete_graph("graph") ``` + /// The returning GraphQL result to confirm that the graph has been deleted: @@ -519,22 +564,27 @@ The returning GraphQL result to confirm that the graph has been deleted: It is possible to update the graph using the `remote_graph()` method. /// tab | :fontawesome-brands-python: Python + ```{.python notest} from raphtory.graphql import GraphServer work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() client.new_graph("path/to/event_graph", "EVENT") rg = client.remote_graph("path/to/event_graph") rg.add_edge(1, "sally", "tony", layer="friendship") ``` + /// -Once you have updated the graph, for example by adding an edge, you can receive a graph by using `receive_graph()` and specifying the path of the graph you would like to receive. +Once you have updated the graph, for example by adding an edge, you can receive a graph by using `receive_graph()` and +specifying the path of the graph you would like to receive. /// tab | :fontawesome-brands-python: Python + ```{.python notest} g = client.receive_graph("path/to/event_graph") ``` + /// diff --git a/python/Cargo.toml b/python/Cargo.toml index 3fcef1949a..9a1a64579a 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -34,6 +34,7 @@ clam-core = { workspace = true, features = ["python"] } extension-module = ["pyo3/extension-module"] search = ["raphtory/search", "raphtory-graphql/search"] proto = ["raphtory/proto"] +test = ["raphtory/panic-on-drop", "extension-module"] [build-dependencies] diff --git a/python/python/raphtory/graphql/__init__.pyi b/python/python/raphtory/graphql/__init__.pyi index b5c21e2146..38b0a230fb 100644 --- a/python/python/raphtory/graphql/__init__.pyi +++ b/python/python/raphtory/graphql/__init__.pyi @@ -87,7 +87,6 @@ class GraphServer(object): cls, work_dir: str | PathLike, cache_capacity: Optional[int] = None, - cache_tti_seconds: Optional[int] = None, log_level: Optional[str] = None, tracing: Optional[bool] = None, tracing_level: Optional[str] = None, @@ -113,24 +112,28 @@ class GraphServer(object): ) -> GraphServer: """Create and return a new object. See help(type) for accurate signature.""" - def run(self, port: int = 1736, timeout_ms: int = 180000) -> None: + def run(self, port: Optional[int] = None, timeout_ms: int = 180000) -> None: """ Run the server until completion. Arguments: - port (int): The port to use. Defaults to 1736. + port (int, optional): The port to use. If not specified, tries 1736 by default and if that is not available starts on an arbitrary port. + If specified and the port is in use, the server will fail to start. timeout_ms (int): Timeout for waiting for the server to start. Defaults to 180000. Returns: None: """ - def start(self, port: int = 1736, timeout_ms: int = 5000) -> RunningGraphServer: + def start( + self, port: Optional[int] = None, timeout_ms: int = 5000 + ) -> RunningGraphServer: """ Start the server and return a handle to it. Arguments: - port (int): the port to use. Defaults to 1736. + port (int, optional): the port to use. If not specified, tries 1736 by default and if that is not available starts on an arbitrary port. + If specified and the port is in use, the server will fail to start. timeout_ms (int): wait for server to be online. Defaults to 5000. The server is stopped if not online within timeout_ms but manages to come online as soon as timeout_ms finishes! @@ -198,6 +201,9 @@ class RunningGraphServer(object): RaphtoryClient: the client. """ + def port(self): + """Get the port the server is listening on""" + def stop(self) -> None: """ Stop the server and wait for it to finish. diff --git a/python/tests/test_auth.py b/python/tests/test_auth.py index 9c6fe3d52a..0678c45353 100644 --- a/python/tests/test_auth.py +++ b/python/tests/test_auth.py @@ -17,8 +17,6 @@ MC4CAQAwBQYDK2VwBCIEIFzEcSO/duEjjX4qKxDVy4uLqfmiEIA6bEw1qiPyzTQg -----END PRIVATE KEY-----""" -RAPHTORY = "http://localhost:1736" - READ_JWT = jwt.encode({"access": "ro"}, PRIVATE_KEY, algorithm="EdDSA") READ_HEADERS = { "Authorization": f"Bearer {READ_JWT}", @@ -69,6 +67,10 @@ TEST_QUERIES = [QUERY_NAMESPACES, QUERY_GRAPH, QUERY_ROOT] +def raphtory_url(port: int) -> str: + return f"http://localhost:{port}" + + def assert_successful_response(response: requests.Response): assert "errors" not in response.json() assert type(response.json()["data"]) == dict @@ -76,22 +78,25 @@ def assert_successful_response(response: requests.Response): # TODO: implement this so we can use the with sintax -def add_test_graph(): +def add_test_graph(port): requests.post( - RAPHTORY, headers=WRITE_HEADERS, data=json.dumps({"query": NEW_TEST_GRAPH}) + raphtory_url(port), + headers=WRITE_HEADERS, + data=json.dumps({"query": NEW_TEST_GRAPH}), ) def test_expired_token(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, auth_public_key=PUB_KEY).start(): + with GraphServer(work_dir, auth_public_key=PUB_KEY).start() as server: + port = server.port() exp = time() - 100 token = jwt.encode({"access": "ro", "exp": exp}, PRIVATE_KEY, algorithm="EdDSA") headers = { "Authorization": f"Bearer {token}", } response = requests.post( - RAPHTORY, headers=headers, data=json.dumps({"query": QUERY_ROOT}) + raphtory_url(port), headers=headers, data=json.dumps({"query": QUERY_ROOT}) ) assert response.status_code == 401 @@ -100,7 +105,7 @@ def test_expired_token(): "Authorization": f"Bearer {token}", } response = requests.post( - RAPHTORY, headers=headers, data=json.dumps({"query": QUERY_ROOT}) + raphtory_url(port), headers=headers, data=json.dumps({"query": QUERY_ROOT}) ) assert response.status_code == 401 @@ -108,17 +113,18 @@ def test_expired_token(): @pytest.mark.parametrize("query", TEST_QUERIES) def test_default_read_access(query): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, auth_public_key=PUB_KEY).start(): - add_test_graph() + with GraphServer(work_dir, auth_public_key=PUB_KEY).start() as server: + port = server.port() + add_test_graph(port) data = json.dumps({"query": query}) - response = requests.post(RAPHTORY, data=data) + response = requests.post(raphtory_url(port), data=data) assert response.status_code == 401 - response = requests.post(RAPHTORY, headers=READ_HEADERS, data=data) + response = requests.post(raphtory_url(port), headers=READ_HEADERS, data=data) assert_successful_response(response) - response = requests.post(RAPHTORY, headers=WRITE_HEADERS, data=data) + response = requests.post(raphtory_url(port), headers=WRITE_HEADERS, data=data) assert_successful_response(response) @@ -127,17 +133,18 @@ def test_disabled_read_access(query): work_dir = tempfile.mkdtemp() with GraphServer( work_dir, auth_public_key=PUB_KEY, require_auth_for_reads=False - ).start(): - add_test_graph() + ).start() as server: + port = server.port() + add_test_graph(port) data = json.dumps({"query": query}) - response = requests.post(RAPHTORY, data=data) + response = requests.post(raphtory_url(port), data=data) assert_successful_response(response) - response = requests.post(RAPHTORY, headers=READ_HEADERS, data=data) + response = requests.post(raphtory_url(port), headers=READ_HEADERS, data=data) assert_successful_response(response) - response = requests.post(RAPHTORY, headers=WRITE_HEADERS, data=data) + response = requests.post(raphtory_url(port), headers=WRITE_HEADERS, data=data) assert_successful_response(response) @@ -178,21 +185,22 @@ def test_disabled_read_access(query): @pytest.mark.parametrize("query", [ADD_NODE, ADD_EDGE, ADD_TEMP_PROP, ADD_CONST_PROP]) def test_update_graph(query): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, auth_public_key=PUB_KEY).start(): - add_test_graph() + with GraphServer(work_dir, auth_public_key=PUB_KEY).start() as server: + port = server.port() + add_test_graph(port) data = json.dumps({"query": query}) - response = requests.post(RAPHTORY, data=data) + response = requests.post(raphtory_url(port), data=data) assert response.status_code == 401 - response = requests.post(RAPHTORY, headers=READ_HEADERS, data=data) + response = requests.post(raphtory_url(port), headers=READ_HEADERS, data=data) assert response.json()["data"] is None assert ( response.json()["errors"][0]["message"] == "The requested endpoint requires write access" ) - response = requests.post(RAPHTORY, headers=WRITE_HEADERS, data=data) + response = requests.post(raphtory_url(port), headers=WRITE_HEADERS, data=data) assert_successful_response(response) @@ -208,28 +216,30 @@ def test_update_graph(query): ) def test_mutations(query): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, auth_public_key=PUB_KEY).start(): - add_test_graph() + with GraphServer(work_dir, auth_public_key=PUB_KEY).start() as server: + port = server.port() + add_test_graph(port) data = json.dumps({"query": query}) - response = requests.post(RAPHTORY, data=data) + response = requests.post(raphtory_url(port), data=data) assert response.status_code == 401 - response = requests.post(RAPHTORY, headers=READ_HEADERS, data=data) + response = requests.post(raphtory_url(port), headers=READ_HEADERS, data=data) assert response.json()["data"] is None assert ( response.json()["errors"][0]["message"] == "The requested endpoint requires write access" ) - response = requests.post(RAPHTORY, headers=WRITE_HEADERS, data=data) + response = requests.post(raphtory_url(port), headers=WRITE_HEADERS, data=data) assert_successful_response(response) def test_raphtory_client(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, auth_public_key=PUB_KEY).start(): - client = RaphtoryClient(url=RAPHTORY, token=WRITE_JWT) + with GraphServer(work_dir, auth_public_key=PUB_KEY).start() as server: + port = server.port() + client = RaphtoryClient(url=raphtory_url(port), token=WRITE_JWT) client.new_graph("test", "EVENT") g = client.remote_graph("test") g.add_node(0, "test") @@ -241,8 +251,9 @@ def test_raphtory_client(): def test_raphtory_client_write_denied_for_read_jwt(): """RaphtoryClient initialized with a read JWT is denied write operations.""" work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, auth_public_key=PUB_KEY).start(): - client = RaphtoryClient(url=RAPHTORY, token=READ_JWT) + with GraphServer(work_dir, auth_public_key=PUB_KEY).start() as server: + port = server.port() + client = RaphtoryClient(url=raphtory_url(port), token=READ_JWT) with pytest.raises(Exception, match="requires write access"): client.new_graph("test", "EVENT") @@ -253,10 +264,11 @@ def test_raphtory_client_write_denied_for_read_jwt(): def test_rsa_signed_jwt_rs256_accepted(): """Server configured with an RSA public key accepts RS256-signed JWTs.""" work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, auth_public_key=RSA_PUB_KEY).start(): + with GraphServer(work_dir, auth_public_key=RSA_PUB_KEY).start() as server: + port = server.port() token = jwt.encode({"access": "ro"}, RSA_PRIVATE_KEY, algorithm="RS256") response = requests.post( - RAPHTORY, + raphtory_url(port), headers={"Authorization": f"Bearer {token}"}, data=json.dumps({"query": QUERY_ROOT}), ) @@ -266,10 +278,11 @@ def test_rsa_signed_jwt_rs256_accepted(): def test_rsa_signed_jwt_rs512_accepted(): """RS512 JWT is also accepted for the same RSA key (different hash, same key material).""" work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, auth_public_key=RSA_PUB_KEY).start(): + with GraphServer(work_dir, auth_public_key=RSA_PUB_KEY).start() as server: + port = server.port() token = jwt.encode({"access": "ro"}, RSA_PRIVATE_KEY, algorithm="RS512") response = requests.post( - RAPHTORY, + raphtory_url(port), headers={"Authorization": f"Bearer {token}"}, data=json.dumps({"query": QUERY_ROOT}), ) @@ -279,10 +292,11 @@ def test_rsa_signed_jwt_rs512_accepted(): def test_eddsa_jwt_rejected_against_rsa_key(): """EdDSA JWT is rejected when the server is configured with an RSA public key.""" work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, auth_public_key=RSA_PUB_KEY).start(): + with GraphServer(work_dir, auth_public_key=RSA_PUB_KEY).start() as server: + port = server.port() token = jwt.encode({"access": "ro"}, PRIVATE_KEY, algorithm="EdDSA") response = requests.post( - RAPHTORY, + raphtory_url(port), headers={"Authorization": f"Bearer {token}"}, data=json.dumps({"query": QUERY_ROOT}), ) @@ -292,20 +306,22 @@ def test_eddsa_jwt_rejected_against_rsa_key(): def test_raphtory_client_read_jwt_can_receive_graph(): """RaphtoryClient initialized with a read JWT can download graphs.""" work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, auth_public_key=PUB_KEY).start(): - client = RaphtoryClient(url=RAPHTORY, token=WRITE_JWT) + with GraphServer(work_dir, auth_public_key=PUB_KEY).start() as server: + port = server.port() + client = RaphtoryClient(url=raphtory_url(port), token=WRITE_JWT) client.new_graph("test", "EVENT") client.remote_graph("test").add_node(0, "mynode") - client2 = RaphtoryClient(url=RAPHTORY, token=READ_JWT) + client2 = RaphtoryClient(url=raphtory_url(port), token=READ_JWT) g = client2.receive_graph("test") assert g.node("mynode") is not None def test_upload_graph(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, auth_public_key=PUB_KEY).start(): - client = RaphtoryClient(url=RAPHTORY, token=WRITE_JWT) + with GraphServer(work_dir, auth_public_key=PUB_KEY).start() as server: + port = server.port() + client = RaphtoryClient(url=raphtory_url(port), token=WRITE_JWT) g = Graph() g.add_node(0, "uploaded-node") tmp_dir = tempfile.mkdtemp() diff --git a/python/tests/test_base_install/test_graphql/edit_graph/test_copy_graph.py b/python/tests/test_base_install/test_graphql/edit_graph/test_copy_graph.py index 734e08cce9..574292f787 100644 --- a/python/tests/test_base_install/test_graphql/edit_graph/test_copy_graph.py +++ b/python/tests/test_base_install/test_graphql/edit_graph/test_copy_graph.py @@ -9,8 +9,8 @@ def test_copy_graph_fails_if_graph_not_found(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() query = """mutation { copyGraph( @@ -34,8 +34,8 @@ def test_copy_graph_fails_if_graph_with_same_name_already_exists(): g.save_to_file(os.path.join(work_dir, "ben", "g5")) g.save_to_file(os.path.join(work_dir, "g6")) - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() query = """mutation { copyGraph( @@ -59,8 +59,8 @@ def test_copy_graph_fails_if_graph_with_same_name_already_exists_at_same_namespa g.save_to_file(os.path.join(work_dir, "ben", "g5")) g.save_to_file(os.path.join(work_dir, "ben", "g6")) - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() query = """mutation { copyGraph( @@ -85,8 +85,8 @@ def test_copy_graph_fails_if_graph_with_same_name_already_exists_at_diff_namespa g.save_to_file(os.path.join(work_dir, "ben", "g5")) g.save_to_file(os.path.join(work_dir, "shivam", "g6")) - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() query = """mutation { copyGraph( @@ -109,8 +109,8 @@ def test_copy_graph_succeeds(): os.makedirs(os.path.join(work_dir, "shivam"), exist_ok=True) g.save_to_file(os.path.join(work_dir, "shivam", "g3")) - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() # Assert if copy graph succeeds and old graph is retained query = """mutation { @@ -151,8 +151,8 @@ def test_copy_graph_using_client_api_succeeds(): os.makedirs(os.path.join(work_dir, "shivam"), exist_ok=True) g.save_to_file(os.path.join(work_dir, "shivam", "g3")) - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() # Assert if copy graph succeeds and old graph is retained client.copy_graph("shivam/g3", "ben/g4") @@ -188,8 +188,8 @@ def test_copy_graph_succeeds_at_same_namespace_as_graph(): g.save_to_file(os.path.join(work_dir, "shivam", "g3")) - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() # Assert if rename graph succeeds and old graph is deleted query = """mutation { @@ -232,8 +232,8 @@ def test_copy_graph_succeeds_at_diff_namespace_as_graph(): g.save_to_file(os.path.join(work_dir, "ben", "g3")) - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() # Assert if rename graph succeeds and old graph is deleted query = """mutation { diff --git a/python/tests/test_base_install/test_graphql/edit_graph/test_delete_graph.py b/python/tests/test_base_install/test_graphql/edit_graph/test_delete_graph.py index 7b74574344..9b6667b955 100644 --- a/python/tests/test_base_install/test_graphql/edit_graph/test_delete_graph.py +++ b/python/tests/test_base_install/test_graphql/edit_graph/test_delete_graph.py @@ -8,8 +8,8 @@ def test_delete_graph_fails_if_graph_not_found(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() query = """mutation { deleteGraph( @@ -23,8 +23,8 @@ def test_delete_graph_fails_if_graph_not_found(): def test_delete_graph_succeeds_if_graph_found(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() g = Graph() g.add_edge(1, "ben", "hamza") @@ -47,8 +47,8 @@ def test_delete_graph_succeeds_if_graph_found(): def test_delete_graph_using_client_api_succeeds_if_graph_found(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() g = Graph() g.add_edge(1, "ben", "hamza") @@ -66,8 +66,8 @@ def test_delete_graph_using_client_api_succeeds_if_graph_found(): def test_delete_graph_succeeds_if_graph_found_at_namespace(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() g = Graph() g.add_edge(1, "ben", "hamza") diff --git a/python/tests/test_base_install/test_graphql/edit_graph/test_graphql.py b/python/tests/test_base_install/test_graphql/edit_graph/test_graphql.py index 9c0f624be9..5fbd1882a5 100644 --- a/python/tests/test_base_install/test_graphql/edit_graph/test_graphql.py +++ b/python/tests/test_base_install/test_graphql/edit_graph/test_graphql.py @@ -1,6 +1,8 @@ import json import os import tempfile +import time + import pytest from utils import sort_by_gql_name_or_id from raphtory import Graph, graph_loader @@ -31,14 +33,15 @@ def test_encode_graph(): def test_failed_server_start_in_time(): tmp_work_dir = tempfile.mkdtemp() - server = None try: - with pytest.raises(Exception) as excinfo: - server = GraphServer(tmp_work_dir).start(timeout_ms=1) - assert str(excinfo.value) == "Failed to start server in 1 milliseconds" - finally: - if server: - server.stop() + start = time.perf_counter() + with GraphServer(tmp_work_dir).start(timeout_ms=1) as server: + assert server.get_client().is_server_online() + assert ( + time.perf_counter() - start + ) < 1 # generous timeout check (1s versus 1ms) + except Exception as excinfo: + assert str(excinfo) == "Failed to start server in 1 milliseconds" def test_wrong_url(): @@ -66,8 +69,9 @@ def test_server_start_on_default_port(): g.add_edge(3, "ben", "haaroon") tmp_work_dir = tempfile.mkdtemp() - with GraphServer(tmp_work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(tmp_work_dir).start() as server: + port = server.port() + client = RaphtoryClient(f"http://localhost:{port}") client.send_graph(path="g", graph=g) query = """{graph(path: "g") {nodes {list {name}}}}""" @@ -119,8 +123,8 @@ def assert_graph_fetch(path): path = "g" tmp_work_dir = tempfile.mkdtemp() - with GraphServer(tmp_work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() # Default namespace, graph is saved in the work dir client.send_graph(path=path, graph=g, overwrite=True) @@ -474,8 +478,8 @@ def test_create_node(): g.add_edge(1, "ben", "shivam") tmp_work_dir = tempfile.mkdtemp() - with GraphServer(tmp_work_dir).start(port=1737): - client = RaphtoryClient("http://localhost:1737") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() client.send_graph(path="g", graph=g) query_nodes = """{graph(path: "g") {nodes {list {name}}}}""" @@ -505,8 +509,8 @@ def test_create_node_using_client(): g.add_edge(1, "ben", "shivam") tmp_work_dir = tempfile.mkdtemp() - with GraphServer(tmp_work_dir).start(port=1737): - client = RaphtoryClient("http://localhost:1737") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() client.send_graph(path="g", graph=g) query_nodes = """{graph(path: "g") {nodes {list {name}}}}""" @@ -533,8 +537,8 @@ def test_create_node_using_client_with_properties(): g.add_edge(1, "ben", "shivam") tmp_work_dir = tempfile.mkdtemp() - with GraphServer(tmp_work_dir).start(port=1737): - client = RaphtoryClient("http://localhost:1737") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() client.send_graph(path="g", graph=g) query_nodes = ( @@ -596,8 +600,8 @@ def test_create_node_using_client_with_properties_node_type(): g.add_edge(1, "ben", "shivam") tmp_work_dir = tempfile.mkdtemp() - with GraphServer(tmp_work_dir).start(port=1737): - client = RaphtoryClient("http://localhost:1737") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() client.send_graph(path="g", graph=g) query_nodes = """{graph(path: "g") {nodes {list {name, nodeType, properties { keys }}}}}""" @@ -664,8 +668,8 @@ def test_create_node_using_client_with_node_type(): g.add_edge(1, "ben", "shivam") tmp_work_dir = tempfile.mkdtemp() - with GraphServer(tmp_work_dir).start(port=1737): - client = RaphtoryClient("http://localhost:1737") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() client.send_graph(path="g", graph=g) query_nodes = """{graph(path: "g") {nodes {list {name, nodeType}}}}""" @@ -702,8 +706,8 @@ def test_edge_id(): g.add_edge(3, "po", "ben") tmp_work_dir = tempfile.mkdtemp() - with GraphServer(tmp_work_dir).start(port=1737): - client = RaphtoryClient("http://localhost:1737") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() client.send_graph(path="g", graph=g) query_nodes = """{graph(path: "g") {edges {list {id}}}}""" @@ -724,8 +728,8 @@ def test_graph_persistence_across_restarts(): tmp_work_dir = tempfile.mkdtemp() # First server session: create graph with 3 nodes and 2 edges - with GraphServer(tmp_work_dir).start(port=1738): - client = RaphtoryClient("http://localhost:1738") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() client.new_graph(path="persistent_graph", graph_type="EVENT") remote_graph = client.remote_graph(path="persistent_graph") # Create 3 nodes @@ -761,8 +765,8 @@ def test_graph_persistence_across_restarts(): } # Server is now shutdown, start it again - with GraphServer(tmp_work_dir).start(port=1738): - client = RaphtoryClient("http://localhost:1738") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() # Verify persistence: check that nodes and edges are still there query_nodes = """{graph(path: "persistent_graph") {nodes {sorted (sortBys: [{id: true}]){ list {name} }}}}""" @@ -841,8 +845,8 @@ def test_float_is_stable_on_roundtrip(): ] prop_key = "p" - with GraphServer(tmp_work_dir).start(port=1738): - client = RaphtoryClient("http://localhost:1738") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() client.new_graph(path="g", graph_type="EVENT") remote_graph = client.remote_graph(path="g") diff --git a/python/tests/test_base_install/test_graphql/edit_graph/test_move_graph.py b/python/tests/test_base_install/test_graphql/edit_graph/test_move_graph.py index f72762e3d8..72a98facf1 100644 --- a/python/tests/test_base_install/test_graphql/edit_graph/test_move_graph.py +++ b/python/tests/test_base_install/test_graphql/edit_graph/test_move_graph.py @@ -9,8 +9,8 @@ def test_move_graph_fails_if_graph_not_found(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() query = """mutation { moveGraph( @@ -34,8 +34,8 @@ def test_move_graph_fails_if_graph_with_same_name_already_exists(): g.save_to_file(os.path.join(work_dir, "ben", "g5")) g.save_to_file(os.path.join(work_dir, "g6")) - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() query = """mutation { moveGraph( @@ -59,8 +59,8 @@ def test_move_graph_fails_if_graph_with_same_name_already_exists_at_same_namespa g.save_to_file(os.path.join(work_dir, "ben", "g5")) g.save_to_file(os.path.join(work_dir, "ben", "g6")) - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() query = """mutation { moveGraph( @@ -85,8 +85,8 @@ def test_move_graph_fails_if_graph_with_same_name_already_exists_at_diff_namespa g.save_to_file(os.path.join(work_dir, "ben", "g5")) g.save_to_file(os.path.join(work_dir, "shivam", "g6")) - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() query = """mutation { moveGraph( @@ -109,8 +109,8 @@ def test_move_graph_succeeds(): os.makedirs(os.path.join(work_dir, "shivam"), exist_ok=True) g.save_to_file(os.path.join(work_dir, "shivam", "g3")) - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() # Assert if rename graph succeeds and old graph is deleted query = """mutation { @@ -148,8 +148,8 @@ def test_move_graph_using_client_api_succeeds(): os.makedirs(os.path.join(work_dir, "shivam"), exist_ok=True) g.save_to_file(os.path.join(work_dir, "shivam", "g3")) - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() # Assert if rename graph succeeds and old graph is deleted client.move_graph("shivam/g3", "ben/g4") @@ -182,8 +182,8 @@ def test_move_graph_succeeds_at_same_namespace_as_graph(): g.save_to_file(os.path.join(work_dir, "shivam", "g3")) - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() # Assert if rename graph succeeds and old graph is deleted query = """mutation { @@ -223,8 +223,8 @@ def test_move_graph_succeeds_at_diff_namespace_as_graph(): g.save_to_file(os.path.join(work_dir, "ben", "g3")) - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() # Assert if rename graph succeeds and old graph is deleted query = """mutation { diff --git a/python/tests/test_base_install/test_graphql/edit_graph/test_new_graph.py b/python/tests/test_base_install/test_graphql/edit_graph/test_new_graph.py index adba406a92..4481a52485 100644 --- a/python/tests/test_base_install/test_graphql/edit_graph/test_new_graph.py +++ b/python/tests/test_base_install/test_graphql/edit_graph/test_new_graph.py @@ -8,8 +8,8 @@ def test_new_graph_succeeds(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() query = """mutation { newGraph( @@ -25,8 +25,8 @@ def test_new_graph_succeeds(): def test_new_graph_fails_if_graph_found(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() g = Graph() g.add_edge(1, "ben", "hamza") @@ -50,8 +50,8 @@ def test_new_graph_fails_if_graph_found(): def test_client_new_graph_works(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() client.new_graph("path/to/event_graph", "EVENT") client.new_graph("path/to/persistent_graph", "PERSISTENT") @@ -63,8 +63,8 @@ def test_client_new_graph_works(): def test_client_new_graph_broken_type(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() with pytest.raises(Exception) as excinfo: client.new_graph("path/to/event_graph", "EVENdddT") assert "Invalid value for argument" in str(excinfo.value) diff --git a/python/tests/test_base_install/test_graphql/edit_graph/test_send_graph.py b/python/tests/test_base_install/test_graphql/edit_graph/test_send_graph.py index 41a469f31f..329d5892fd 100644 --- a/python/tests/test_base_install/test_graphql/edit_graph/test_send_graph.py +++ b/python/tests/test_base_install/test_graphql/edit_graph/test_send_graph.py @@ -13,8 +13,8 @@ def test_send_graph_succeeds_if_no_graph_found_with_same_name(): g.add_edge(3, "ben", "haaroon") tmp_work_dir = tempfile.mkdtemp() - with GraphServer(tmp_work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() client.send_graph(path="g", graph=g) @@ -27,8 +27,8 @@ def test_send_graph_fails_if_graph_already_exists(): g.add_edge(3, "ben", "haaroon") g.save_to_file(os.path.join(tmp_work_dir, "g")) - with GraphServer(tmp_work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() with pytest.raises(Exception) as excinfo: client.send_graph(path="g", graph=g) assert "Graph 'g' already exists" in str(excinfo.value) @@ -42,8 +42,8 @@ def test_send_graph_succeeds_if_graph_already_exists_with_overwrite_enabled(): g.add_edge(2, "haaroon", "hamza") g.add_edge(3, "ben", "haaroon") - with GraphServer(tmp_work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() client.send_graph(path="g", graph=g) @@ -76,8 +76,8 @@ def test_send_graph_succeeds_if_no_graph_found_with_same_name_at_namespace(): g.add_edge(3, "ben", "haaroon") tmp_work_dir = tempfile.mkdtemp() - with GraphServer(tmp_work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() client.send_graph(path="shivam/g", graph=g) @@ -91,8 +91,8 @@ def test_send_graph_fails_if_graph_already_exists_at_namespace(): os.makedirs(os.path.join(tmp_work_dir, "shivam"), exist_ok=True) g.save_to_file(os.path.join(tmp_work_dir, "shivam", "g")) - with GraphServer(tmp_work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() with pytest.raises(Exception) as excinfo: client.send_graph(path="shivam/g", graph=g) assert "Graph 'shivam/g' already exists" in str(excinfo.value) @@ -108,8 +108,8 @@ def test_send_graph_succeeds_if_graph_already_exists_at_namespace_with_overwrite os.makedirs(os.path.join(tmp_work_dir, "shivam"), exist_ok=True) g.save_to_file(os.path.join(tmp_work_dir, "shivam", "g")) - with GraphServer(tmp_work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() g = Graph() g.add_edge(1, "ben", "hamza") diff --git a/python/tests/test_base_install/test_graphql/edit_graph/test_upload_graph.py b/python/tests/test_base_install/test_graphql/edit_graph/test_upload_graph.py index 5f92d5e37a..fd8ffc5019 100644 --- a/python/tests/test_base_install/test_graphql/edit_graph/test_upload_graph.py +++ b/python/tests/test_base_install/test_graphql/edit_graph/test_upload_graph.py @@ -17,8 +17,8 @@ def test_upload_graph_succeeds_if_no_graph_found_with_same_name(): g.save_to_zip(g_file_path) tmp_work_dir = tempfile.mkdtemp() - with GraphServer(tmp_work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() client.upload_graph(path="g", file_path=g_file_path, overwrite=False) query = """{graph(path: "g") {nodes {list {name}}}}""" @@ -41,8 +41,8 @@ def test_upload_graph_succeeds_if_no_graph_found_with_same_name_non_zip(): g.save_to_file(g_file_path) tmp_work_dir = tempfile.mkdtemp() - with GraphServer(tmp_work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() client.upload_graph(path="g", file_path=g_file_path, overwrite=False) query = """{graph(path: "g") {nodes {list {name}}}}""" @@ -66,8 +66,8 @@ def test_upload_graph_fails_if_graph_already_exists(): tmp_work_dir = tempfile.mkdtemp() g.save_to_file(os.path.join(tmp_work_dir, "g")) - with GraphServer(tmp_work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() with pytest.raises(Exception) as excinfo: client.upload_graph(path="g", file_path=g_file_path) assert "Graph 'g' already exists" in str(excinfo.value) @@ -83,8 +83,8 @@ def test_upload_graph_succeeds_if_graph_already_exists_with_overwrite_enabled(): g.save_to_file(g_file_path) tmp_work_dir = tempfile.mkdtemp() - with GraphServer(tmp_work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() g = Graph() g.add_edge(1, "ben", "hamza") @@ -123,8 +123,8 @@ def test_upload_graph_succeeds_if_no_graph_found_with_same_name_at_namespace(): g.save_to_zip(g_file_path) tmp_work_dir = tempfile.mkdtemp() - with GraphServer(tmp_work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() client.upload_graph(path="shivam/g", file_path=g_file_path, overwrite=False) query = """{graph(path: "shivam/g") {nodes {list {name}}}}""" @@ -151,8 +151,8 @@ def test_upload_graph_fails_if_graph_already_exists_at_namespace(): tmp_work_dir = tempfile.mkdtemp() os.makedirs(os.path.join(tmp_work_dir, "shivam"), exist_ok=True) g.save_to_file(os.path.join(tmp_work_dir, "shivam", "g")) - with GraphServer(tmp_work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() with pytest.raises(Exception) as excinfo: client.upload_graph(path="shivam/g", file_path=g_file_path, overwrite=False) assert "Graph 'shivam/g' already exists" in str(excinfo.value) @@ -167,8 +167,8 @@ def test_upload_graph_succeeds_if_graph_already_exists_at_namespace_with_overwri os.makedirs(os.path.join(tmp_work_dir, "shivam"), exist_ok=True) g.save_to_file(os.path.join(tmp_work_dir, "shivam", "g")) - with GraphServer(tmp_work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() g = Graph() g.add_edge(1, "ben", "hamza") diff --git a/python/tests/test_base_install/test_graphql/misc/test_components.py b/python/tests/test_base_install/test_graphql/misc/test_components.py index 9197d17069..dfa67dc776 100644 --- a/python/tests/test_base_install/test_graphql/misc/test_components.py +++ b/python/tests/test_base_install/test_graphql/misc/test_components.py @@ -89,8 +89,8 @@ def test_in_out_components(): g.add_edge(6, 7, 3) g.save_to_file(work_dir + "/graph") - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() query_res = client.query(query) prepare_for_comparison(query_res["graph"]) prepare_for_comparison(result["graph"]) diff --git a/python/tests/test_base_install/test_graphql/misc/test_index_off.py b/python/tests/test_base_install/test_graphql/misc/test_index_off.py index 62786fae60..4fc4848995 100644 --- a/python/tests/test_base_install/test_graphql/misc/test_index_off.py +++ b/python/tests/test_base_install/test_graphql/misc/test_index_off.py @@ -19,8 +19,8 @@ def test_latest_and_active(): work_dir = tempfile.mkdtemp() g = Graph() g.save_to_file(work_dir + "/graph") - with GraphServer(work_dir).turn_off_index().start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).turn_off_index().start() as server: + client = server.get_client() with pytest.raises(Exception) as excinfo: client.query(query) assert ( diff --git a/python/tests/test_base_install/test_graphql/misc/test_latest.py b/python/tests/test_base_install/test_graphql/misc/test_latest.py index ee58148aed..8dfd3ba0dc 100644 --- a/python/tests/test_base_install/test_graphql/misc/test_latest.py +++ b/python/tests/test_base_install/test_graphql/misc/test_latest.py @@ -232,6 +232,6 @@ def test_latest_and_active(): g.add_node(2, 2, {"int_prop": 125}) g.save_to_file(work_dir + "/graph") - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() assert sort_by_gql_name_or_id(client.query(query)) == result diff --git a/python/tests/test_base_install/test_graphql/misc/test_map_props.py b/python/tests/test_base_install/test_graphql/misc/test_map_props.py index f5ca23d73a..5546b75bc8 100644 --- a/python/tests/test_base_install/test_graphql/misc/test_map_props.py +++ b/python/tests/test_base_install/test_graphql/misc/test_map_props.py @@ -14,9 +14,9 @@ def test_map_props(): work_dir = tempfile.mkdtemp() server = GraphServer(work_dir) - with server.start(): + with server.start() as server: temp_dir = tempfile.mkdtemp() - client = RaphtoryClient("http://localhost:1736") + client = server.get_client() g = Graph() g.update_metadata({"test": TEST_PROPS}) node = g.add_node(0, "test") @@ -28,7 +28,8 @@ def test_map_props(): work_dir = tempfile.mkdtemp() server = GraphServer(work_dir) - with server.start(): + with server.start() as server: + client = server.get_client() client.new_graph("test", "EVENT") rg = client.remote_graph("test") rg.update_metadata({"test": TEST_PROPS}) diff --git a/python/tests/test_base_install/test_graphql/misc/test_snapshot.py b/python/tests/test_base_install/test_graphql/misc/test_snapshot.py index de8c8f8501..1bb4dc8ba6 100644 --- a/python/tests/test_base_install/test_graphql/misc/test_snapshot.py +++ b/python/tests/test_base_install/test_graphql/misc/test_snapshot.py @@ -5,8 +5,8 @@ def test_snapshot(): work_dir = tempfile.mkdtemp() server = GraphServer(work_dir) - with server.start(): - client = RaphtoryClient("http://localhost:1736") + with server.start() as server: + client = server.get_client() def query(graph: str, window: str): return client.query(f"""{{ diff --git a/python/tests/test_base_install/test_graphql/misc/test_tracing.py b/python/tests/test_base_install/test_graphql/misc/test_tracing.py index f870d20e68..b7f0ebeca0 100644 --- a/python/tests/test_base_install/test_graphql/misc/test_tracing.py +++ b/python/tests/test_base_install/test_graphql/misc/test_tracing.py @@ -12,8 +12,8 @@ def test_server_start_on_default_port(): g.add_edge(3, "ben", "haaroon") tmp_work_dir = tempfile.mkdtemp() - with GraphServer(tmp_work_dir, tracing=True).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(tmp_work_dir, tracing=True).start() as server: + client = server.get_client() client.send_graph(path="g", graph=g) query = """{graph(path: "g") {nodes {list {name}}}}""" @@ -24,8 +24,8 @@ def test_server_start_on_default_port(): } } } - with GraphServer(tmp_work_dir, tracing=True).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(tmp_work_dir, tracing=True).start() as server: + client = server.get_client() client.send_graph(path="g2", graph=g) query = """{graph(path: "g2") {nodes {list {name}}}}""" diff --git a/python/tests/test_base_install/test_graphql/test_graph_file_time_stats.py b/python/tests/test_base_install/test_graphql/test_graph_file_time_stats.py index 63e6de6dba..04f31253d5 100644 --- a/python/tests/test_base_install/test_graphql/test_graph_file_time_stats.py +++ b/python/tests/test_base_install/test_graphql/test_graph_file_time_stats.py @@ -16,8 +16,8 @@ def test_graph_file_time_stats(): graph_file_path = os.path.join(work_dir, "shivam", "g3") g.save_to_file(graph_file_path) - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() query = """{graph(path: "shivam/g3") { created lastOpened lastUpdated }}""" result = client.query(query) diff --git a/python/tests/test_base_install/test_graphql/test_metadata_dispatch.py b/python/tests/test_base_install/test_graphql/test_metadata_dispatch.py index 01c60bfe11..0802ae4c31 100644 --- a/python/tests/test_base_install/test_graphql/test_metadata_dispatch.py +++ b/python/tests/test_base_install/test_graphql/test_metadata_dispatch.py @@ -33,8 +33,6 @@ reason="disk-backed graph tests require the storage feature", ) -SERVER_URL = "http://localhost:1736" - def _persist_disk_graph(graph_dir): """Build a disk-backed graph at `graph_dir`, populate it, flush it, @@ -112,8 +110,8 @@ def test_metadata_returned_for_both_disk_and_parquet_graphs(): _read_is_diskgraph(parquet_graph_dir) is False ), "parquet_graph was not saved as parquet" - with GraphServer(work_dir).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir).start() as server: + client = server.get_client() # ---- Path 2 (disk on-disk read) and Path 3 (parquet on-disk read). # Neither graph has been loaded into the server's cache yet, so @@ -160,8 +158,8 @@ def test_metadata_update_in_single_segment_returns_latest(): g.flush() del g - with GraphServer(work_dir).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir).start() as server: + client = server.get_client() meta = _list_metadata_by_path(client) assert ( meta["g"]["version"] == "v2" @@ -187,8 +185,8 @@ def test_metadata_update_across_flushes_returns_newest_segment(): g.flush() del g - with GraphServer(work_dir).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir).start() as server: + client = server.get_client() meta = _list_metadata_by_path(client) assert ( meta["g"]["version"] == "v2" @@ -211,8 +209,8 @@ def test_metadata_many_updates_across_flushes_returns_last(): g.flush() del g - with GraphServer(work_dir).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir).start() as server: + client = server.get_client() meta = _list_metadata_by_path(client) assert ( meta["g"]["version"] == "v499" @@ -278,8 +276,8 @@ def test_metadata_mixed_keys_across_flushes(): g.flush() del g - with GraphServer(work_dir).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir).start() as server: + client = server.get_client() meta = _list_metadata_by_path(client) assert meta["g"]["untouched"] == "stable" assert ( diff --git a/python/tests/test_base_install/test_graphql/test_misc.py b/python/tests/test_base_install/test_graphql/test_misc.py index a8a1992845..f37d1cd45e 100644 --- a/python/tests/test_base_install/test_graphql/test_misc.py +++ b/python/tests/test_base_install/test_graphql/test_misc.py @@ -8,6 +8,6 @@ def test_version_query(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() assert client.query("{version}")["version"] == raphtory.version() diff --git a/python/tests/test_base_install/test_graphql/test_namespace.py b/python/tests/test_base_install/test_graphql/test_namespace.py index 35fbb6de11..c3ed99538c 100644 --- a/python/tests/test_base_install/test_graphql/test_namespace.py +++ b/python/tests/test_base_install/test_graphql/test_namespace.py @@ -29,8 +29,8 @@ def sort_dict(d): def test_namespaces_and_metagraph(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() make_folder_structure(client) # tests list and page on namespaces and metagraphs @@ -188,8 +188,8 @@ def test_namespaces_and_metagraph(): def test_counting(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() make_folder_structure(client) query = """ @@ -227,8 +227,8 @@ def test_counting(): def test_escaping_parent(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() make_folder_structure(client) query = """{ @@ -265,8 +265,8 @@ def test_escaping_parent(): def test_wrong_paths(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() make_folder_structure(client) query = """{ @@ -347,8 +347,8 @@ def test_wrong_paths(): def test_namespaces(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() make_folder_structure(client) query = """ @@ -441,8 +441,8 @@ def test_namespace_listing_does_not_load_each_graph(): graph_props segment).""" n_graphs = 200 work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() g = Graph() g.add_node(1, "alice", {"role": "engineer"}) diff --git a/python/tests/test_base_install/test_graphql/test_read_only_load.py b/python/tests/test_base_install/test_graphql/test_read_only_load.py index b4a266f12d..af456943a5 100644 --- a/python/tests/test_base_install/test_graphql/test_read_only_load.py +++ b/python/tests/test_base_install/test_graphql/test_read_only_load.py @@ -31,8 +31,6 @@ reason="disk-backed graph tests require the storage feature", ) -SERVER_URL = "http://localhost:1736" - def _persist_graph(graph_dir): """Build a disk-backed graph at `graph_dir`, populate it, flush it, and @@ -65,8 +63,8 @@ def test_read_only_load_while_server_owns_directory(): read-only handle should both succeed and report the same data. """ work_dir, graph_dir = _make_work_dir_with_graph() - with GraphServer(work_dir).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir).start() as server: + client = server.get_client() # Trigger the server to load the graph so it holds the writer. client.query('{ graph(path: "g") { created } }') @@ -104,8 +102,8 @@ def test_graphql_and_read_only_handle_interleaved(): read-only handle to the same directory. Both pathways should keep serving consistent results across multiple round-trips.""" work_dir, graph_dir = _make_work_dir_with_graph() - with GraphServer(work_dir).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir).start() as server: + client = server.get_client() # Prime the server's load. client.query('{ graph(path: "g") { created } }') ro = Graph.load(graph_dir, read_only=True) @@ -130,8 +128,8 @@ def test_read_only_load_blocks_all_mutation_paths(): - graph-level metadata (never touches the id resolver at all) """ work_dir, graph_dir = _make_work_dir_with_graph() - with GraphServer(work_dir).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir).start() as server: + client = server.get_client() client.query('{ graph(path: "g") { created } }') ro = Graph.load(graph_dir, read_only=True) @@ -159,8 +157,8 @@ def test_writer_load_against_live_server_directory_fails(): `read_only=True` is not accidentally a no-op flag — it really is the only way to coexist with a live writer.""" work_dir, graph_dir = _make_work_dir_with_graph() - with GraphServer(work_dir).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir).start() as server: + client = server.get_client() # Force the server to take the writer lock by loading the graph. client.query('{ graph(path: "g") { created } }') @@ -172,8 +170,8 @@ def test_multiple_read_only_handles_can_coexist_with_server(): """Two simultaneous read-only handles + the server writer = three total attachments to the same graph directory.""" work_dir, graph_dir = _make_work_dir_with_graph() - with GraphServer(work_dir).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir).start() as server: + client = server.get_client() client.query('{ graph(path: "g") { created } }') ro1 = Graph.load(graph_dir, read_only=True) @@ -212,8 +210,8 @@ def test_flush_via_update_graph_makes_writes_visible_to_read_only_handle(): when it opens) wouldn't see writes the server has made via `updateGraph` since the last flush.""" work_dir, graph_dir = _make_work_dir_with_graph() - with GraphServer(work_dir).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir).start() as server: + client = server.get_client() # Server loads as writer. client.query('{ graph(path: "g") { created } }') @@ -234,8 +232,8 @@ def test_flush_via_update_graph_makes_writes_visible_to_read_only_handle(): def test_read_only_load_from_a_separate_python_process(): work_dir, graph_dir = _make_work_dir_with_graph() - with GraphServer(work_dir).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir).start() as server: + client = server.get_client() client.query('{ graph(path: "g") { created } }') result = subprocess.run( diff --git a/python/tests/test_base_install/test_graphql/test_schema.py b/python/tests/test_base_install/test_graphql/test_schema.py index 2ec0412efd..6c67fd58b3 100644 --- a/python/tests/test_base_install/test_graphql/test_schema.py +++ b/python/tests/test_base_install/test_graphql/test_schema.py @@ -58,8 +58,8 @@ def test_node_edge_properties_schema(): graph_file_path = os.path.join(work_dir, "graph") g.save_to_file(graph_file_path) - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() query = """{ graph(path: "graph") { diff --git a/python/tests/test_base_install/test_graphql/test_server_flags.py b/python/tests/test_base_install/test_graphql/test_server_flags.py index 1c80de56f1..409899b6bb 100644 --- a/python/tests/test_base_install/test_graphql/test_server_flags.py +++ b/python/tests/test_base_install/test_graphql/test_server_flags.py @@ -10,11 +10,11 @@ SERVER_URL = "http://localhost:1736" -def batch_query(body): +def batch_query(port, body): """POST a raw JSON body (needed for batch requests — the client only sends single queries).""" data = json.dumps(body).encode("utf-8") req = urllib.request.Request( - SERVER_URL + "/", + f"http://localhost:{port}/", data=data, headers={"Content-Type": "application/json"}, method="POST", @@ -40,16 +40,16 @@ def make_graph(client, path="g"): def test_introspection_enabled_by_default(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir).start() as server: + client = server.get_client() result = client.query("{ __schema { queryType { name } } }") assert result["__schema"]["queryType"]["name"] def test_disable_introspection(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, disable_introspection=True).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir, disable_introspection=True).start() as server: + client = server.get_client() client.query("{ version }") with pytest.raises(Exception) as excinfo: @@ -60,8 +60,8 @@ def test_disable_introspection(): def test_max_query_depth(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, max_query_depth=3).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir, max_query_depth=3).start() as server: + client = server.get_client() make_graph(client) client.query('{ graph(path: "g") { created } }') @@ -75,8 +75,8 @@ def test_max_query_depth(): def test_max_query_complexity(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, max_query_complexity=3).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir, max_query_complexity=3).start() as server: + client = server.get_client() make_graph(client) client.query("{ version }") @@ -253,8 +253,8 @@ def test_max_query_complexity(): def test_disable_lists_all_resolvers(): """Every `list` endpoint across every paginated type rejects with the same error.""" work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, disable_lists=True).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir, disable_lists=True).start() as server: + client = server.get_client() make_graph(client) for name, query in LIST_QUERIES: @@ -269,8 +269,8 @@ def test_disable_lists_all_resolvers(): def test_disable_lists_page_still_works(): """Even with `disable_lists=True`, `page` queries still succeed.""" work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, disable_lists=True).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir, disable_lists=True).start() as server: + client = server.get_client() make_graph(client) result = client.query( '{ graph(path: "g") { nodes { page(limit: 10) { name } } } }' @@ -281,8 +281,8 @@ def test_disable_lists_page_still_works(): def test_max_page_size_all_resolvers(): """Every `page` endpoint across every paginated type enforces max_page_size.""" work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, max_page_size=2).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir, max_page_size=2).start() as server: + client = server.get_client() make_graph(client) for name, query in PAGE_QUERIES: @@ -296,8 +296,8 @@ def test_max_page_size_all_resolvers(): def test_max_page_size_under_cap_works(): """Pages at or below max_page_size still succeed.""" work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, max_page_size=2).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir, max_page_size=2).start() as server: + client = server.get_client() make_graph(client) result = client.query( '{ graph(path: "g") { nodes { page(limit: 2) { name } } } }' @@ -307,30 +307,33 @@ def test_max_page_size_under_cap_works(): def test_disable_batching(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, disable_batching=True).start(): - RaphtoryClient(SERVER_URL).query("{ version }") + with GraphServer(work_dir, disable_batching=True).start() as server: + server.get_client().query("{ version }") - status, body = batch_query([{"query": "{ version }"}, {"query": "{ version }"}]) + status, body = batch_query( + server.port(), [{"query": "{ version }"}, {"query": "{ version }"}] + ) assert status == 400 assert "Query batching is disabled on this server" in str(body) def test_max_batch_size(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, max_batch_size=2).start(): - status, body = batch_query([{"query": "{ version }"}] * 2) + with GraphServer(work_dir, max_batch_size=2).start() as server: + port = server.port() + status, body = batch_query(port, [{"query": "{ version }"}] * 2) assert status == 200 assert isinstance(body, list) and len(body) == 2 - status, body = batch_query([{"query": "{ version }"}] * 3) + status, body = batch_query(port, [{"query": "{ version }"}] * 3) assert status == 400 assert "Batch size 3 exceeds the maximum allowed 2" in str(body) def test_max_recursive_depth(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, max_recursive_depth=2).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir, max_recursive_depth=2).start() as server: + client = server.get_client() make_graph(client) # depth 2: { graph { created } } — root selection set is depth 0, graph{...} pushes to 1 @@ -345,8 +348,8 @@ def test_max_recursive_depth(): def test_max_directives_per_field(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir, max_directives_per_field=1).start(): - client = RaphtoryClient(SERVER_URL) + with GraphServer(work_dir, max_directives_per_field=1).start() as server: + client = server.get_client() # 1 directive — allowed client.query("{ version @skip(if: false) }") @@ -370,8 +373,8 @@ def test_concurrency_flags_smoke(): work_dir, heavy_query_limit=4, exclusive_writes=True, - ).start(): - client = RaphtoryClient(SERVER_URL) + ).start() as server: + client = server.get_client() make_graph(client) # Read path: works under exclusive_writes's read lock. assert client.query('{ graph(path: "g") { nodes { count } } }') diff --git a/python/tests/test_base_install/test_graphql/update_graph/test_batch_updates.py b/python/tests/test_base_install/test_graphql/update_graph/test_batch_updates.py index 35988b24a3..13c2c425b0 100644 --- a/python/tests/test_base_install/test_graphql/update_graph/test_batch_updates.py +++ b/python/tests/test_base_install/test_graphql/update_graph/test_batch_updates.py @@ -70,8 +70,8 @@ def create_updates(timestamps: List[int]): def test_add_nodes(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() client.new_graph("path/to/event_graph", "EVENT") rg: RemoteGraph = client.remote_graph("path/to/event_graph") node_updates = [] @@ -124,8 +124,8 @@ def test_add_nodes(): def test_add_edges(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() client.new_graph("path/to/event_graph", "EVENT") rg: RemoteGraph = client.remote_graph("path/to/event_graph") edge_updates = [] diff --git a/python/tests/test_base_install/test_graphql/update_graph/test_edge_updates.py b/python/tests/test_base_install/test_graphql/update_graph/test_edge_updates.py index f7c0181542..ed9cef7c49 100644 --- a/python/tests/test_base_install/test_graphql/update_graph/test_edge_updates.py +++ b/python/tests/test_base_install/test_graphql/update_graph/test_edge_updates.py @@ -54,8 +54,8 @@ def make_props2(): def test_add_updates(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() client.new_graph("path/to/event_graph", "EVENT") rg = client.remote_graph("path/to/event_graph") props = make_props() @@ -78,8 +78,8 @@ def test_add_updates(): def test_add_metadata(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() client.new_graph("path/to/event_graph", "EVENT") rg = client.remote_graph("path/to/event_graph") props = make_props() @@ -100,8 +100,8 @@ def test_add_metadata(): def test_update_metadata(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() client.new_graph("path/to/event_graph", "EVENT") rg = client.remote_graph("path/to/event_graph") props = make_props() @@ -122,8 +122,8 @@ def test_update_metadata(): def test_delete(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() client.new_graph("path/to/event_graph", "EVENT") rg = client.remote_graph("path/to/event_graph") diff --git a/python/tests/test_base_install/test_graphql/update_graph/test_graph_updates.py b/python/tests/test_base_install/test_graphql/update_graph/test_graph_updates.py index 4ca5df2c6e..026fe7c64e 100644 --- a/python/tests/test_base_install/test_graphql/update_graph/test_graph_updates.py +++ b/python/tests/test_base_install/test_graphql/update_graph/test_graph_updates.py @@ -40,8 +40,8 @@ def make_props(): def test_add_metadata(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() client.new_graph("path/to/event_graph", "EVENT") rg = client.remote_graph("path/to/event_graph") props = make_props() @@ -56,8 +56,8 @@ def test_add_metadata(): def test_update_metadata(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() client.new_graph("path/to/event_graph", "EVENT") rg = client.remote_graph("path/to/event_graph") props = make_props() @@ -73,8 +73,8 @@ def test_update_metadata(): def test_add_properties(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() client.new_graph("path/to/event_graph", "EVENT") rg = client.remote_graph("path/to/event_graph") props = make_props() @@ -104,8 +104,8 @@ def test_add_properties(): def test_add_node(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() client.new_graph("path/to/event_graph", "EVENT") rg = client.remote_graph("path/to/event_graph") props = make_props() @@ -123,8 +123,8 @@ def test_add_node(): def test_add_edge(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() client.new_graph("path/to/event_graph", "EVENT") rg = client.remote_graph("path/to/event_graph") props = make_props() @@ -142,8 +142,8 @@ def test_add_edge(): def test_delete_edge(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() client.new_graph("path/to/event_graph", "EVENT") rg = client.remote_graph("path/to/event_graph") diff --git a/python/tests/test_base_install/test_graphql/update_graph/test_node_updates.py b/python/tests/test_base_install/test_graphql/update_graph/test_node_updates.py index c394dd657d..4690d4eb92 100644 --- a/python/tests/test_base_install/test_graphql/update_graph/test_node_updates.py +++ b/python/tests/test_base_install/test_graphql/update_graph/test_node_updates.py @@ -36,8 +36,8 @@ def make_props(): def test_set_node_type(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() client.new_graph("path/to/event_graph", "EVENT") rg = client.remote_graph("path/to/event_graph") node = rg.add_node(1, "ben") @@ -51,8 +51,8 @@ def test_set_node_type(): def test_add_updates(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() client.new_graph("path/to/event_graph", "EVENT") rg = client.remote_graph("path/to/event_graph") props = make_props() @@ -67,8 +67,8 @@ def test_add_updates(): def test_add_metadata(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() client.new_graph("path/to/event_graph", "EVENT") rg = client.remote_graph("path/to/event_graph") props = make_props() @@ -84,8 +84,8 @@ def test_add_metadata(): def test_update_metadata(): work_dir = tempfile.mkdtemp() - with GraphServer(work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(work_dir).start() as server: + client = server.get_client() client.new_graph("path/to/event_graph", "EVENT") rg = client.remote_graph("path/to/event_graph") props = make_props() diff --git a/python/tests/test_permissions.py b/python/tests/test_permissions.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/python/tests/test_search/test_gql_index_spec.py b/python/tests/test_search/test_gql_index_spec.py index 04a639757f..65811627b9 100644 --- a/python/tests/test_search/test_gql_index_spec.py +++ b/python/tests/test_search/test_gql_index_spec.py @@ -108,10 +108,9 @@ def test_create_index_with_default_spec(graph): @pytest.mark.parametrize("graph", [EVENT_GRAPH, PERSISTENT_GRAPH]) def test_create_index_using_client(graph): - tmp_work_dir = tempfile.mkdtemp() - with GraphServer(tmp_work_dir).start(): - client = RaphtoryClient("http://localhost:1736") + with GraphServer(tmp_work_dir).start() as server: + client = server.get_client() client.send_graph(path="g", graph=graph) query = """{graph(path: "g") {nodes {list {name}}}}""" diff --git a/python/tests/test_vectors/test_graphql_vectors.py b/python/tests/test_vectors/test_graphql_vectors.py index 6a22c0b35c..cf46800859 100644 --- a/python/tests/test_vectors/test_graphql_vectors.py +++ b/python/tests/test_vectors/test_graphql_vectors.py @@ -60,8 +60,8 @@ def test_new_graph(): work_dir = tempfile.TemporaryDirectory() server = GraphServer(work_dir.name) with embeddings.start(7340): - with server.start(): - client = RaphtoryClient("http://localhost:1736") + with server.start() as server: + client = server.get_client() client.new_graph("abb", "EVENT") rg = client.remote_graph("abb") setup_graph(rg) @@ -79,8 +79,8 @@ def test_upload_graph(): temp_dir = tempfile.TemporaryDirectory() server = GraphServer(work_dir.name) with embeddings.start(7340): - with server.start(): - client = RaphtoryClient("http://localhost:1736") + with server.start() as server: + client = server.get_client() g = Graph() setup_graph(g) g_path = temp_dir.name + "/abb" @@ -104,8 +104,8 @@ def test_vectorised_graph_window_accepts_time_input_shapes(): work_dir = tempfile.TemporaryDirectory() server = GraphServer(work_dir.name) with embeddings.start(7340): - with server.start(): - client = RaphtoryClient("http://localhost:1736") + with server.start() as server: + client = server.get_client() client.new_graph("abb", "EVENT") rg = client.remote_graph("abb") setup_graph(rg) @@ -163,6 +163,6 @@ def test_include_graph(): nodes="{{ name }}", edges=False, ) - with server.start(): - client = RaphtoryClient("http://localhost:1736") + with server.start() as server: + client = server.get_client() assert_correct_documents(client) diff --git a/python/tox.ini b/python/tox.ini index de2ce2de53..d600935672 100644 --- a/python/tox.ini +++ b/python/tox.ini @@ -18,6 +18,8 @@ pass_env = [testenv:.pkg] pass_env = MATURIN_PEP517_ARGS +set_env = + MATURIN_PEP517_ARGS="--features=test" [testenv:search] wheel_build_env = .pkg_search @@ -25,7 +27,7 @@ commands = pytest {posargs} {tty:--color=yes} tests/test_search [testenv:.pkg_search] set_env = - MATURIN_PEP517_ARGS="--features=search,extension-module" + MATURIN_PEP517_ARGS="--features=search,test" [testenv:export] diff --git a/raphtory-graphql/Cargo.toml b/raphtory-graphql/Cargo.toml index 081ea82f76..a6e625e239 100644 --- a/raphtory-graphql/Cargo.toml +++ b/raphtory-graphql/Cargo.toml @@ -42,6 +42,7 @@ tracing-opentelemetry = { workspace = true } tracing-subscriber = { workspace = true } walkdir = { workspace = true } ordered-float = { workspace = true } +dashmap.workspace = true chrono = { workspace = true } config = { workspace = true } url = { workspace = true } @@ -54,6 +55,7 @@ ahash = { workspace = true } strum = { workspace = true } strum_macros = { workspace = true } bigdecimal = { workspace = true, features = ["serde"] } +quick_cache = { workspace = true } # python binding optional dependencies pyo3 = { workspace = true, optional = true } @@ -69,7 +71,7 @@ rust-embed = { workspace = true } parking_lot = { workspace = true } tempfile = { workspace = true } pretty_assertions = { workspace = true } -raphtory = { workspace = true, features = ["test-utils"] } +raphtory = { workspace = true, features = ["test-utils", "panic-on-drop"] } arrow-array = { workspace = true } [features] diff --git a/raphtory-graphql/src/cache.rs b/raphtory-graphql/src/cache.rs new file mode 100644 index 0000000000..d607d860f2 --- /dev/null +++ b/raphtory-graphql/src/cache.rs @@ -0,0 +1,203 @@ +use crate::{ + data::{InsertionError, MutationErrorInner}, + graph::GraphWithVectors, + paths::ValidGraphPaths, + rayon::{blocking_compute, EVICT_POOL}, +}; +use quick_cache::{ + sync::{Cache, EntryAction, EntryResult}, + DefaultHashBuilder, Lifecycle, UnitWeighter, +}; +use raphtory::prelude::AdditionOps; +use raphtory_storage::core_ops::CoreGraphOps; +use std::future::Future; +use tracing::{debug, error}; + +#[derive(Default, Copy, Clone)] +pub struct ArcPinned; + +fn flush_graph(val: GraphWithVectors) -> () { + val.set_flushing(true); + val.set_dirty(false); // make sure this is reset before the flush so any mutation that gets triggered afterwards will set the graph back to dirty + let graph = val.graph(); + if let Err(e) = graph.flush() { + error!("Failed to flush graph {}: {e}", val.folder().local_path()) + } + if let Err(e) = val.folder().replace_graph_data(graph.clone()) { + error!("Failed to write graph {}: {e}", val.folder().local_path()) + } + val.set_flushing(false); +} + +impl Lifecycle for ArcPinned { + type RequestState = (); + + #[inline] + fn begin_request(&self) -> Self::RequestState {} + + #[inline] + fn is_pinned(&self, _key: &String, val: &GraphWithVectors) -> bool { + if val.ref_count() > 1 { + return true; + } + + if val.is_dirty() { + if !val.is_flushing() { + let graph = val.clone(); + EVICT_POOL.spawn(move || { + debug!("Flushing graph {}", graph.folder().local_path()); + flush_graph(graph); + }) + } + return true; + } + + val.is_flushing() + } + + #[inline] + fn on_evict(&self, state: &mut Self::RequestState, key: String, graph: GraphWithVectors) { + debug_assert_eq!( + graph.ref_count(), + 1, + "We should have the only reference to the graph on eviction" + ); + debug_assert!(!graph.is_dirty(), "Graph should be clean on eviction"); + debug_assert!( + !graph.is_flushing(), + "Graph should be already flushed on eviction" + ); + + debug!( + "Graph {} removed from cache (clean)", + graph.folder().local_path() + ); + } +} + +pub struct GraphCache { + cache: Cache, +} + +impl GraphCache { + pub fn new(items_capacity: usize) -> Self { + let cache = Cache::with( + items_capacity, + items_capacity as u64, + Default::default(), + Default::default(), + Default::default(), + ); + Self { cache } + } + + /// Get item for key if it is cached + pub fn get(&self, key: &str) -> Option { + self.cache.get(key) + } + + pub fn contains_key(&self, key: &str) -> bool { + self.cache.contains_key(key) + } + + pub fn iter(&self) -> impl Iterator + use<'_> { + self.cache.iter() + } + + /// Get item for key. If the item is not found, insert it using the provided future + pub async fn get_or_insert( + &self, + key: &str, + with: impl Future>, + ) -> Result { + self.cache.get_or_insert_async(key, with).await + } + + /// Insert a new item into the cache, replacing an existing item if it exists + /// The old item is dropped before the closure to create the new graph is invoked + pub async fn insert_with( + &self, + key: &str, + with: impl FnOnce() -> F, + ) -> Result<(), InsertionError> + where + F: Future>, + InsertionError: From, + { + let cache_guard = self + .cache + .entry_async(key, |key, value| EntryAction::<()>::ReplaceWithGuard) + .await; + let guard = match cache_guard { + EntryResult::Replaced(guard, old_graph) => { + drop(old_graph); + guard + } + EntryResult::Vacant(guard) => guard, + _ => { + unreachable!() + } + }; + let new_graph = with().await?; + guard + .insert(new_graph) + .map_err(|_| InsertionError::Insertion { + graph: key.to_string(), + error: MutationErrorInner::CacheReplacementError, + })?; + Ok(()) + } + + /// clear all items from the cache, flushing them if needed + pub fn flush_and_clear(&self) { + for (_, graph) in self.cache.drain() { + flush_graph(graph); + } + } + + /// remove a graph from the cache without triggering the eviction drop logic + /// Note that the cache entry is available again immediately! + pub async fn remove(&self, key: &str) -> Option { + let res = self + .cache + .entry_async(key, |key, graph| EntryAction::<()>::Remove) + .await; + match res { + EntryResult::Removed(_, graph) => Some(graph), + _ => None, + } + } + + /// remove a graph from the cache, locking the cache entry until the graph is dropped + /// this is different from remove which returns the graph and unlocks the entry immediately + pub async fn delete(&self, key: &str) { + let res = self + .cache + .entry_async(key, |key, graph| EntryAction::<()>::ReplaceWithGuard) + .await; + + match res { + EntryResult::Replaced(_guard, graph) => { + blocking_compute(move || drop(graph)).await; + } + _ => {} + } + } + + /// remove a graph from the cache, locking the cache entry until the graph is dropped and the future has completed. + /// if the graph exists, it is dropped first before the future runs + pub async fn invalidate_with(&self, key: &str, with: impl Future) -> E { + let guard = self + .cache + .entry_async(key, |key, graph| EntryAction::<()>::ReplaceWithGuard) + .await; + + match guard { + EntryResult::Replaced(_guard, graph) => { + blocking_compute(move || drop(graph)).await; + with.await + } + _ => with.await, + } + } +} diff --git a/raphtory-graphql/src/cli.rs b/raphtory-graphql/src/cli.rs index 56a7bbe242..36f009c1a5 100644 --- a/raphtory-graphql/src/cli.rs +++ b/raphtory-graphql/src/cli.rs @@ -4,7 +4,7 @@ use crate::{ config::{ app_config::AppConfigBuilder, auth_config::{DEFAULT_REQUIRE_AUTH_FOR_READS, PUBLIC_KEY_DECODING_ERR_MSG}, - cache_config::{DEFAULT_CAPACITY, DEFAULT_TTI_SECONDS}, + cache_config::DEFAULT_CAPACITY, concurrency_config::{ DEFAULT_DISABLE_BATCHING, DEFAULT_DISABLE_LISTS, DEFAULT_EXCLUSIVE_WRITES, }, @@ -24,14 +24,14 @@ use raphtory::db::api::storage::storage::Config; use std::path::PathBuf; use tokio::io::Result as IoResult; -#[derive(Parser)] +#[derive(Parser, Debug)] #[command(name = "raphtory", about = "Raphtory CLI", version = raphtory::version())] struct Args { #[command(subcommand)] command: Commands, } -#[derive(Subcommand)] +#[derive(Subcommand, Debug)] enum Commands { #[command(about = "Run the GraphQL server")] Server(ServerArgs), @@ -39,7 +39,7 @@ enum Commands { Schema, } -#[derive(clap::Args)] +#[derive(clap::Args, Debug)] struct ServerArgs { #[arg( long, @@ -49,15 +49,12 @@ struct ServerArgs { )] work_dir: PathBuf, - #[arg(long, env = "RAPHTORY_PORT", default_value_t = DEFAULT_PORT, help = "Port for Raphtory to run on")] - port: u16, + #[arg(long, env = "RAPHTORY_PORT", help = "Port for Raphtory to run on")] + port: Option, #[arg(long, env = "RAPHTORY_CACHE_CAPACITY", default_value_t = DEFAULT_CAPACITY, help = "Cache capacity")] cache_capacity: u64, - #[arg(long, env = "RAPHTORY_CACHE_TTI_SECONDS", default_value_t = DEFAULT_TTI_SECONDS, help = "Cache time-to-idle in seconds")] - cache_tti_seconds: u64, - #[arg(long, env = "RAPHTORY_LOG_LEVEL", default_value = DEFAULT_LOG_LEVEL, help = "Log level")] log_level: String, @@ -199,7 +196,6 @@ where Commands::Server(server_args) => { let mut builder = AppConfigBuilder::new() .with_cache_capacity(server_args.cache_capacity) - .with_cache_tti_seconds(server_args.cache_tti_seconds) .with_log_level(server_args.log_level) .with_tracing(server_args.tracing) .with_tracing_level(server_args.tracing_level) @@ -238,7 +234,14 @@ where .await?; let server = apply_server_extension(server, server_args.permissions_store_path.as_deref()); - server.run_with_port(server_args.port).await?; + match server_args.port { + None => { + server.run().await?; + } + Some(port) => { + server.run_with_port(port).await?; + } + } } } Ok(()) diff --git a/raphtory-graphql/src/config/app_config.rs b/raphtory-graphql/src/config/app_config.rs index 4157dd84df..bc66d5cff4 100644 --- a/raphtory-graphql/src/config/app_config.rs +++ b/raphtory-graphql/src/config/app_config.rs @@ -98,11 +98,6 @@ impl AppConfigBuilder { self } - pub fn with_cache_tti_seconds(mut self, tti_seconds: u64) -> Self { - self.cache.tti_seconds = tti_seconds; - self - } - pub fn with_auth_public_key( mut self, public_key: Option, @@ -254,7 +249,7 @@ pub fn load_config( app_config_builder = app_config_builder.with_cache_capacity(cache_capacity); } if let Ok(cache_tti_seconds) = settings.get::("cache.tti_seconds") { - app_config_builder = app_config_builder.with_cache_tti_seconds(cache_tti_seconds); + app_config_builder = app_config_builder; } if let Ok(public_key) = settings.get::>("auth.public_key") { diff --git a/raphtory-graphql/src/config/cache_config.rs b/raphtory-graphql/src/config/cache_config.rs index da97275717..49fbb974ed 100644 --- a/raphtory-graphql/src/config/cache_config.rs +++ b/raphtory-graphql/src/config/cache_config.rs @@ -1,19 +1,16 @@ use serde::Deserialize; pub const DEFAULT_CAPACITY: u64 = 30; -pub const DEFAULT_TTI_SECONDS: u64 = 1000000000; #[derive(Debug, Deserialize, PartialEq, Clone, serde::Serialize)] pub struct CacheConfig { pub capacity: u64, - pub tti_seconds: u64, } impl Default for CacheConfig { fn default() -> Self { Self { capacity: DEFAULT_CAPACITY, - tti_seconds: DEFAULT_TTI_SECONDS, } } } diff --git a/raphtory-graphql/src/config/mod.rs b/raphtory-graphql/src/config/mod.rs index 9ef7311dec..4cd7d851f5 100644 --- a/raphtory-graphql/src/config/mod.rs +++ b/raphtory-graphql/src/config/mod.rs @@ -42,7 +42,6 @@ mod tests { .with_tracing(true) .with_tracing_level(TracingLevel::ESSENTIAL) .with_cache_capacity(30) - .with_cache_tti_seconds(1000) .with_auth_public_key(Some( "MCowBQYDK2VwAyEADdrWr1kTLj+wSHlr45eneXmOjlHo3N1DjLIvDa2ozno=".to_owned(), )) @@ -57,10 +56,7 @@ mod tests { #[test] fn test_load_config_with_custom_cache() { - let app_config = AppConfigBuilder::new() - .with_cache_capacity(50) - .with_cache_tti_seconds(1200) - .build(); + let app_config = AppConfigBuilder::new().with_cache_capacity(50).build(); let result = load_config(Some(app_config.clone()), None); diff --git a/raphtory-graphql/src/data.rs b/raphtory-graphql/src/data.rs index 989a706eaf..8b4d1056e7 100644 --- a/raphtory-graphql/src/data.rs +++ b/raphtory-graphql/src/data.rs @@ -1,6 +1,7 @@ use crate::{ auth::ContextValidation, - auth_policy::{AuthorizationPolicy, GraphPermission, NamespacePermission}, + auth_policy::{AuthorizationPolicy, GraphPermission, NamespacePermission, PermissionLevel}, + cache::GraphCache, config::app_config::AppConfig, graph::GraphWithVectors, model::{ @@ -19,8 +20,6 @@ use crate::{ }; use async_graphql::Context; use dynamic_graphql::Enum; -use futures_util::FutureExt; -use moka::future::Cache; use raphtory::{ db::{ api::{ @@ -56,6 +55,8 @@ pub enum MutationErrorInner { IO(#[from] io::Error), #[error(transparent)] InvalidInternal(#[from] InternalPathValidationError), + #[error("Cache operation failed, simultaneous mutation occurred")] + CacheReplacementError, } #[derive(thiserror::Error, Debug)] @@ -142,7 +143,7 @@ pub(crate) fn get_relative_path( /// Inner struct with a drop implementation that cleans up the graphs pub struct DataInner { pub(crate) work_dir: PathBuf, - pub(crate) cache: Cache, + pub(crate) cache: GraphCache, pub(crate) vector_cache: LazyDiskVectorCache, pub(crate) graph_conf: Config, pub(crate) auth_policy: Option>, @@ -167,25 +168,7 @@ impl Data { pub fn new(work_dir: &Path, configs: &AppConfig, graph_conf: Config) -> Self { let cache_configs = &configs.cache; - let cache = Cache::::builder() - .max_capacity(cache_configs.capacity) - .time_to_idle(std::time::Duration::from_secs(cache_configs.tti_seconds)) - .async_eviction_listener(|_, graph, cause| { - // The eviction listener gets called any time a graph is removed from the cache, - // not just when it is evicted. Only serialize on evictions. - async move { - if !cause.was_evicted() { - return; - } - if let Err(e) = - blocking_compute(move || graph.folder.replace_graph_data(graph.graph)).await - { - error!("Error encoding graph to disk on eviction: {e}"); - } - } - .boxed() - }) - .build(); + let cache = GraphCache::new(cache_configs.capacity as usize); #[cfg(feature = "search")] let create_index = configs.index.create_index; @@ -212,11 +195,6 @@ impl Data { .auth_policy = Some(policy); } - async fn invalidate(&self, path: &str) { - self.cache.invalidate(path).await; - self.cache.run_pending_tasks().await; // make sure the item is actually dropped - } - pub fn validate_path_for_insert( &self, path: &str, @@ -232,9 +210,9 @@ impl Data { /// # ⚠ Bypasses all permission checks — do not call from resolvers directly. /// Use `get_graph_with_read_permission`, `get_raw_graph_with_read_permission`, or /// `get_graph_with_write_permission` instead. - async fn get_graph(&self, path: &str) -> Result> { + async fn get_graph(&self, path: &str) -> Result { self.cache - .try_get_with(path.into(), self.read_graph_from_disk(path)) + .get_or_insert(path.into(), self.read_graph_from_disk(path)) .await } @@ -243,17 +221,12 @@ impl Data { pub(crate) async fn get_graph_for_test( &self, path: &str, - ) -> Result> { + ) -> Result { self.get_graph(path).await } pub async fn get_cached_graph(&self, path: &str) -> Option { - self.cache.get(path).await - } - - pub fn has_graph(&self, path: &str) -> bool { - self.cache.contains_key(path) - || ExistingGraphFolder::try_from(self.work_dir.clone(), path).is_ok() + self.cache.get(path.into()) } pub async fn insert_graph( @@ -261,25 +234,19 @@ impl Data { writeable_folder: ValidWriteableGraphFolder, graph: MaterializedGraph, ) -> Result<(), InsertionError> { - self.invalidate(writeable_folder.local_path()).await; + let key = writeable_folder.local_path().to_owned(); let config = self.graph_conf.clone(); - let graph = blocking_compute(move || { - writeable_folder.write_graph_data(graph.clone(), config)?; - let folder = writeable_folder.finish()?; - let graph = GraphWithVectors::new(graph, None, folder.as_existing()?); - Ok::<_, InsertionError>(graph) - }) - .await?; self.cache - .insert(graph.folder.local_path().into(), graph) - .await; - // moka's `insert(..).await` is eventually consistent — the entry is - // queued and may not be visible to `cache.get(..)` immediately. Force - // the pending insert through so a follow-up `MetaGraph.metadata` - // hitting the listing path sees the cached graph instead of falling - // through to `read_constant_graph_properties`, which would read the - // on-disk graph_props before the writer has flushed them. - self.cache.run_pending_tasks().await; + .insert_with(&key, || { + blocking_compute(move || { + let (is_dirty, new_graph) = writeable_folder.write_graph_data(graph, config)?; + let folder = writeable_folder.finish()?; + let graph = GraphWithVectors::new(new_graph, None, folder.as_existing()?); + graph.set_dirty(is_dirty); + Ok::<_, InsertionError>(graph) + }) + }) + .await?; Ok(()) } @@ -289,13 +256,16 @@ impl Data { folder: ValidWriteableGraphFolder, bytes: R, ) -> Result<(), InsertionError> { - self.invalidate(folder.local_path()).await; let conf = self.graph_conf.clone(); - blocking_io(move || { - folder.write_graph_bytes(bytes, conf)?; - folder.finish() - }) - .await?; + self.cache + .invalidate_with( + &folder.local_path().to_string(), + blocking_io(move || { + folder.write_graph_bytes(bytes, conf)?; + folder.finish() + }), + ) + .await?; Ok(()) } @@ -303,14 +273,18 @@ impl Data { &self, graph_folder: ExistingGraphFolder, ) -> Result<(), MutationErrorInner> { + let key = graph_folder.local_path().to_string(); let dirty_file = mark_dirty(graph_folder.root())?; - self.invalidate(graph_folder.local_path()).await; - blocking_io(move || { - fs::remove_dir_all(graph_folder.root())?; - fs::remove_file(dirty_file)?; - Ok::<_, MutationErrorInner>(()) - }) - .await?; + self.cache + .invalidate_with( + &key, + blocking_io(move || { + fs::remove_dir_all(graph_folder.root())?; + fs::remove_file(dirty_file)?; + Ok::<_, MutationErrorInner>(()) + }), + ) + .await?; Ok(()) } @@ -319,7 +293,6 @@ impl Data { self.delete_graph_inner(graph_folder) .await .map_err(|err| DeletionError::from_inner(path, err))?; - self.cache.remove(path).await; Ok(()) } @@ -355,10 +328,14 @@ impl Data { model: CachedEmbeddingModel, ) -> Result<(), GQLError> { let graph = match self.get_cached_graph(folder.local_path()).await { - None => self.read_graph_from_disk_inner(folder.clone()).await?, - Some(graph) => graph, + None => self + .read_graph_from_disk_inner(folder.clone()) + .await? + .graph() + .clone(), + Some(graph) => graph.graph().clone(), }; - self.vectorise_with_template(graph.graph, folder, template, model) + self.vectorise_with_template(graph, folder, template, model) .await; self.cache.remove(folder.local_path()).await; Ok(()) @@ -411,6 +388,10 @@ pub(crate) enum PermissionError { /// Caller has read-only access but the operation requires write. #[error("Access denied: WRITE permission required for graph '{graph}'")] GraphWriteRequired { graph: String }, + + /// Caller has filtered read-only access but the opration requires unfiltered read + #[error("Access denied: unfiltered READ permissions required for graph '{graph}'")] + GraphUnfilteredReadRequired { graph: String }, /// Caller lacks write permission on the destination namespace. #[error( "Access denied: WRITE required on namespace '{namespace}' to {operation} graph '{graph}'" @@ -588,19 +569,21 @@ impl Data { ) -> async_graphql::Result<(ExistingGraphFolder, DynamicGraph)> { let gwv = self.get_graph(path).await?; let typed_graph = match graph_type { - Some(GqlGraphType::Event) => match gwv.graph { - MaterializedGraph::EventGraph(g) => MaterializedGraph::EventGraph(g), + Some(GqlGraphType::Event) => match gwv.graph() { + MaterializedGraph::EventGraph(g) => MaterializedGraph::EventGraph(g.clone()), MaterializedGraph::PersistentGraph(g) => { MaterializedGraph::EventGraph(g.event_graph()) } }, - Some(GqlGraphType::Persistent) => match gwv.graph { + Some(GqlGraphType::Persistent) => match gwv.graph() { MaterializedGraph::EventGraph(g) => { MaterializedGraph::PersistentGraph(g.persistent_graph()) } - MaterializedGraph::PersistentGraph(g) => MaterializedGraph::PersistentGraph(g), + MaterializedGraph::PersistentGraph(g) => { + MaterializedGraph::PersistentGraph(g.clone()) + } }, - None => gwv.graph, + None => gwv.graph().clone(), }; let raw = typed_graph.into_dynamic(); let graph = if let GraphPermission::Read { @@ -611,7 +594,7 @@ impl Data { } else { raw }; - Ok((gwv.folder, graph)) + Ok((gwv.folder().clone(), graph)) } /// For the `graph()` resolver: permission denial → `Ok(None)` (null to client, hides @@ -651,10 +634,14 @@ impl Data { ctx: &Context<'_>, path: &str, ) -> async_graphql::Result { - require_at_least_read(ctx, &self.auth_policy, path)?; - self.get_graph(path) - .await - .map_err(|e| async_graphql::Error::new(e.to_string())) + let res = require_at_least_read(ctx, &self.auth_policy, path)?; + if res.level() < PermissionLevel::Read { + Err(PermissionError::GraphUnfilteredReadRequired { + graph: path.to_string(), + })?; + } + let graph = self.get_graph(path).await?; + Ok(graph) } /// Checks write permission then returns the raw `GraphWithVectors` for mutation operations. @@ -664,9 +651,8 @@ impl Data { path: &str, ) -> async_graphql::Result { require_graph_write(ctx, &self.auth_policy, path)?; - self.get_graph(path) - .await - .map_err(|e| async_graphql::Error::new(e.to_string())) + let graph = self.get_graph(path).await?; + Ok(graph) } /// Checks read permission then returns the vectorised graph, if any. @@ -681,25 +667,14 @@ impl Data { if matches!(perm, GraphPermission::Read { filter: Some(_) }) { return Ok(None); } - Ok(self - .get_graph(path) - .await - .ok() - .and_then(|g| g.vectors) - .map(Into::into)) + let graph = self.get_graph(path).await?; + Ok(graph.vectors().cloned().map(|g| g.into())) } } impl Drop for DataInner { fn drop(&mut self) { - // On drop, serialize graphs that don't have underlying storage. - for (_, graph) in self.cache.iter() { - if graph.is_dirty() { - if let Err(e) = graph.folder.replace_graph_data(graph.graph) { - error!("Error encoding graph to disk on drop: {e}"); - } - } - } + self.cache.flush_and_clear(); } } @@ -776,10 +751,7 @@ pub(crate) mod data_tests { graph.encode(&tmp_work_dir.path().join("test_g")).unwrap(); graph.encode(&tmp_work_dir.path().join("test_g2")).unwrap(); - let configs = AppConfigBuilder::new() - .with_cache_capacity(1) - .with_cache_tti_seconds(2) - .build(); + let configs = AppConfigBuilder::new().with_cache_capacity(1).build(); let data = Data::new(tmp_work_dir.path(), &configs, Default::default()); @@ -792,20 +764,7 @@ pub(crate) mod data_tests { assert!(!data.cache.contains_key("test_g")); data.get_graph("test_g").await.unwrap(); // wait for any eviction - data.cache.run_pending_tasks().await; assert_eq!(data.cache.iter().count(), 1); - - sleep(Duration::from_secs(3)).await; - assert!(!data.cache.contains_key("test_g")); - assert!(!data.cache.contains_key("test_g2")); - // FIXME: this test is not doing anything because calling cache.contains_key() runs - // any pending evictions. To actually test it we need this assertion: - // assert_eq!(data.cache.entry_count(), 0); - // Which currently does not work because the server task to trigger evictions is not running - // in this context. The problem is if we do run it by creating a server and calling - // server.start() the server gets consumed and we loose access to the cache to be able to run - // the check. If rework the server implementation and this becomes feasible we should change - // this test } #[tokio::test] @@ -836,10 +795,7 @@ pub(crate) mod data_tests { fs::create_dir_all(&g6_path).unwrap(); fs::write(g6_path.join("random-file"), "some-random-content").unwrap(); - let configs = AppConfigBuilder::new() - .with_cache_capacity(1) - .with_cache_tti_seconds(2) - .build(); + let configs = AppConfigBuilder::new().with_cache_capacity(1).build(); let data = Data::new(work_dir, &configs, Default::default()); @@ -900,10 +856,7 @@ pub(crate) mod data_tests { let graph1_original_time = graph1_metadata.modified().unwrap(); let graph2_original_time = graph2_metadata.modified().unwrap(); - let configs = AppConfigBuilder::new() - .with_cache_capacity(10) - .with_cache_tti_seconds(300) - .build(); + let configs = AppConfigBuilder::new().with_cache_capacity(10).build(); let data = Data::new(tmp_work_dir.path(), &configs, Default::default()); @@ -911,7 +864,7 @@ pub(crate) mod data_tests { let loaded_graph2 = data.get_graph("test_graph2").await.unwrap(); // TODO: This test doesn't work with disk storage right now, make sure modification dates actually update correctly! - if loaded_graph1.graph.disk_storage_path().is_some() { + if loaded_graph1.graph().disk_storage_path().is_some() { assert!( !loaded_graph1.is_dirty(), "Graph1 should not be dirty when loaded from disk" @@ -984,10 +937,7 @@ pub(crate) mod data_tests { let graph2_original_time = graph2_metadata.modified().unwrap(); // Create cache with time to idle 3 seconds to force eviction - let configs = AppConfigBuilder::new() - .with_cache_capacity(10) - .with_cache_tti_seconds(3) - .build(); + let configs = AppConfigBuilder::new().with_cache_capacity(10).build(); let data = Data::new(tmp_work_dir.path(), &configs, Default::default()); @@ -1015,10 +965,9 @@ pub(crate) mod data_tests { // Sleep to trigger eviction sleep(Duration::from_secs(3)).await; - data.cache.run_pending_tasks().await; // TODO: This test doesn't work with disk storage right now, make sure modification dates actually update correctly! - if loaded_graph1.graph.disk_storage_path().is_some() { + if loaded_graph1.graph().disk_storage_path().is_some() { // Check modification times after eviction let graph1_metadata_after = fs::metadata(&graph1_path).unwrap(); let graph2_metadata_after = fs::metadata(&graph2_path).unwrap(); diff --git a/raphtory-graphql/src/graph.rs b/raphtory-graphql/src/graph.rs index 663adbcd23..02fd01f4fb 100644 --- a/raphtory-graphql/src/graph.rs +++ b/raphtory-graphql/src/graph.rs @@ -34,32 +34,62 @@ use raphtory::prelude::IndexMutationOps; #[derive(Clone)] pub struct GraphWithVectors { - pub graph: MaterializedGraph, - pub vectors: Option>, - pub(crate) folder: ExistingGraphFolder, - pub(crate) is_dirty: Arc, + inner: Arc, +} + +struct GraphWithVectorsInner { + graph: MaterializedGraph, + vectors: Option>, + folder: ExistingGraphFolder, + is_dirty: AtomicBool, + is_flushing: AtomicBool, } impl GraphWithVectors { - pub(crate) fn new( + pub fn new( graph: MaterializedGraph, vectors: Option>, folder: ExistingGraphFolder, ) -> Self { - Self { + let inner = Arc::new(GraphWithVectorsInner { graph, vectors, folder, - is_dirty: Arc::new(AtomicBool::new(false)), - } + is_dirty: AtomicBool::new(false), + is_flushing: AtomicBool::new(false), + }); + Self { inner } + } + + pub fn graph(&self) -> &MaterializedGraph { + &self.inner.graph + } + + pub fn vectors(&self) -> Option<&VectorisedGraph> { + self.inner.vectors.as_ref() } - pub(crate) fn set_dirty(&self, is_dirty: bool) { - self.is_dirty.store(is_dirty, Ordering::SeqCst); + pub fn folder(&self) -> &ExistingGraphFolder { + &self.inner.folder + } + pub fn set_dirty(&self, is_dirty: bool) { + self.inner.is_dirty.store(is_dirty, Ordering::Release); } - pub(crate) fn is_dirty(&self) -> bool { - self.is_dirty.load(Ordering::SeqCst) + pub fn is_dirty(&self) -> bool { + self.inner.is_dirty.load(Ordering::Acquire) + } + + pub fn is_flushing(&self) -> bool { + self.inner.is_flushing.load(Ordering::Acquire) + } + + pub fn set_flushing(&self, is_flushing: bool) { + self.inner.is_flushing.store(is_flushing, Ordering::Release) + } + + pub fn ref_count(&self) -> usize { + Arc::strong_count(&self.inner) } /// Generates and stores embeddings for a batch of nodes. @@ -67,7 +97,7 @@ impl GraphWithVectors { &self, nodes: Vec, ) -> GraphResult<()> { - if let Some(vectors) = &self.vectors { + if let Some(vectors) = &self.inner.vectors { vectors.update_nodes(nodes).await?; } @@ -79,7 +109,7 @@ impl GraphWithVectors { &self, edges: Vec<(T, T)>, ) -> GraphResult<()> { - if let Some(vectors) = &self.vectors { + if let Some(vectors) = &self.inner.vectors { vectors.update_edges(edges).await?; } @@ -116,12 +146,7 @@ impl GraphWithVectors { graph.create_index()?; } - Ok(Self { - graph: graph.clone(), - vectors, - folder: folder.clone().into(), - is_dirty: Arc::new(AtomicBool::new(false)), - }) + Ok(Self::new(graph, vectors, folder.clone())) } } @@ -129,7 +154,7 @@ impl Base for GraphWithVectors { type Base = MaterializedGraph; #[inline] fn base(&self) -> &Self::Base { - &self.graph + &self.inner.graph } } diff --git a/raphtory-graphql/src/lib.rs b/raphtory-graphql/src/lib.rs index 5a4c7b0cf6..b342a59dce 100644 --- a/raphtory-graphql/src/lib.rs +++ b/raphtory-graphql/src/lib.rs @@ -20,6 +20,7 @@ mod routes; pub mod server; pub mod url_encode; +pub mod cache; pub mod cli; pub mod config; #[cfg(feature = "python")] @@ -1304,6 +1305,63 @@ mod graphql_test { save_graphs_to_work_dir(&data, &graphs).await.unwrap(); let schema = App::create_schema().data(data).finish().unwrap(); + let all = r#"{ + graph(path: "graph1") { + nodes { + list { + name + } + } + edges { + list { + id + } + } + } + }"#; + + let res = schema.execute(Request::new(all)).await; + let data = res.data.into_json().unwrap(); + + let all_nodes: Vec<_> = data + .get("graph") + .unwrap() + .get("nodes") + .unwrap() + .get("list") + .unwrap() + .as_array() + .unwrap() + .iter() + .filter_map(|v| v.get("name").unwrap().as_str()) + .collect(); + + let all_edges: Vec<(_, _)> = data + .get("graph") + .unwrap() + .get("edges") + .unwrap() + .get("list") + .unwrap() + .as_array() + .unwrap() + .iter() + .filter_map(|v| v.get("id").unwrap().as_array()) + .filter_map(|ids| ids.iter().filter_map(|v| v.as_u64()).collect_tuple()) + .collect(); + + // make sure we have the correct edges + assert_eq!( + all_edges.iter().cloned().sorted().collect_vec(), + [(1, 2), (2, 4), (3, 2), (3, 6), (4, 5), (4, 6), (5, 6),] + ); + + // make sure we have the correct nodes + assert_eq!( + all_nodes.iter().copied().sorted().collect_vec(), + ["1", "2", "3", "4", "5", "6"] + ); + let req = r#" { graph(path: "graph1") { diff --git a/raphtory-graphql/src/model/graph/graph.rs b/raphtory-graphql/src/model/graph/graph.rs index e64ccc03d8..25bf609365 100644 --- a/raphtory-graphql/src/model/graph/graph.rs +++ b/raphtory-graphql/src/model/graph/graph.rs @@ -66,7 +66,7 @@ pub(crate) struct GqlGraph { impl From for GqlGraph { fn from(value: GraphWithVectors) -> Self { - GqlGraph::new(value.folder, value.graph) + GqlGraph::new(value.folder().clone(), value.graph().clone()) } } @@ -694,7 +694,8 @@ impl GqlGraph { let other_g = data .get_graph_with_write_permission(ctx, path.as_ref()) .await? - .graph; + .graph() + .clone(); let g = self.graph.clone(); blocking_compute(move || { other_g.import_nodes(g.nodes(), true)?; diff --git a/raphtory-graphql/src/model/graph/meta_graph.rs b/raphtory-graphql/src/model/graph/meta_graph.rs index e9da3277a4..7033a81d1e 100644 --- a/raphtory-graphql/src/model/graph/meta_graph.rs +++ b/raphtory-graphql/src/model/graph/meta_graph.rs @@ -116,7 +116,7 @@ impl MetaGraph { let data: &Data = ctx.data_unchecked(); if let Some(graph) = data.get_cached_graph(self.folder.local_path()).await { return Ok(graph - .graph + .graph() .metadata() .iter() .filter_map(|(key, value)| value.map(|prop| GqlProperty::new(key.into(), prop))) diff --git a/raphtory-graphql/src/model/graph/mutable_graph.rs b/raphtory-graphql/src/model/graph/mutable_graph.rs index 4dc852518a..91a3a111ac 100644 --- a/raphtory-graphql/src/model/graph/mutable_graph.rs +++ b/raphtory-graphql/src/model/graph/mutable_graph.rs @@ -166,17 +166,17 @@ impl GqlMutableGraph { )] graph_type: Option, ) -> GqlGraph { - let folder = self.graph.folder.clone(); + let folder = self.graph.folder().clone(); match graph_type { - Some(GqlGraphType::Event) => match self.graph.graph.clone() { + Some(GqlGraphType::Event) => match self.graph.graph().clone() { MaterializedGraph::EventGraph(g) => GqlGraph::new(folder, g), MaterializedGraph::PersistentGraph(g) => GqlGraph::new(folder, g.event_graph()), }, - Some(GqlGraphType::Persistent) => match self.graph.graph.clone() { + Some(GqlGraphType::Persistent) => match self.graph.graph().clone() { MaterializedGraph::EventGraph(g) => GqlGraph::new(folder, g.persistent_graph()), MaterializedGraph::PersistentGraph(g) => GqlGraph::new(folder, g), }, - None => GqlGraph::new(folder, self.graph.graph.clone()), + None => GqlGraph::new(folder, self.graph.graph().clone()), } } @@ -524,8 +524,14 @@ impl GqlMutableGraph { async fn flush(&self) -> Result { let self_clone = self.clone(); blocking_write(move || { - self_clone.graph.graph.flush()?; - Ok(true) + self_clone.graph.set_flushing(true); + self_clone.graph.set_dirty(false); + let res = self_clone.graph.graph().flush(); + if res.is_err() { + self_clone.graph.set_dirty(true) + } + self_clone.graph.set_flushing(false); + res.map(|_| true) }) .await } @@ -859,7 +865,7 @@ mod tests { template::DocumentTemplate, }, }; - use tempfile::tempdir; + use tempfile::{tempdir, TempDir}; fn fake_embedding(_: &str) -> Vec { vec![1.0] @@ -870,9 +876,16 @@ mod tests { graph.into() } - async fn create_mutable_graph( - port: u16, - ) -> (GqlMutableGraph, Data, tempfile::TempDir, EmbeddingServer) { + /// Struct returned by `create_mutable_graph` to make sure the directory is dropped last. + /// Otherwise, the drop of the graph might fail as the directory no longer exists. + pub struct GraphTestContext { + mutable_graph: GqlMutableGraph, + embedding_server: EmbeddingServer, + data: Data, + tmp_dir: TempDir, + } + + async fn create_mutable_graph(port: u16) -> GraphTestContext { let graph = create_test_graph(); let tmp_dir = tempdir().unwrap(); @@ -907,12 +920,21 @@ mod tests { let graph_with_vectors = data.get_graph_for_test(graph_name).await.unwrap(); let mutable_graph = GqlMutableGraph::from(graph_with_vectors); - (mutable_graph, data, tmp_dir, embedding_server) + GraphTestContext { + mutable_graph, + data, + tmp_dir, + embedding_server, + } } #[tokio::test] async fn test_add_nodes_empty_list() { - let (mutable_graph, _data, _tmp_dir, embedding_server) = create_mutable_graph(1745).await; + let GraphTestContext { + mutable_graph, + embedding_server, + .. + } = create_mutable_graph(0).await; let nodes = vec![]; let result = mutable_graph.add_nodes(nodes).await; @@ -925,7 +947,8 @@ mod tests { #[tokio::test] #[ignore = "TODO: #2384"] async fn test_add_nodes_simple() { - let (mutable_graph, _data, _tmp_dir, es) = create_mutable_graph(1746).await; + let context = create_mutable_graph(1746).await; + let mutable_graph = &context.mutable_graph; let nodes = vec![ NodeAddition { @@ -959,7 +982,7 @@ mod tests { let limit = 5; let result = mutable_graph .graph - .vectors + .vectors() .unwrap() .nodes_by_similarity(&embedding.into(), limit, None) .execute() @@ -967,13 +990,14 @@ mod tests { assert!(result.is_ok()); assert!(result.unwrap().get_documents().await.unwrap().len() == 2); - es.stop().await; + context.embedding_server.stop().await; } #[tokio::test] #[ignore = "TODO: #2384"] async fn test_add_nodes_with_properties() { - let (mutable_graph, _data, _tmp_dir, es) = create_mutable_graph(1747).await; + let context = create_mutable_graph(1747).await; + let mutable_graph = &context.mutable_graph; let nodes = vec![ NodeAddition { @@ -1035,7 +1059,7 @@ mod tests { let limit = 5; let result = mutable_graph .graph - .vectors + .vectors() .unwrap() .nodes_by_similarity(&embedding.into(), limit, None) .execute() @@ -1043,13 +1067,14 @@ mod tests { assert!(result.is_ok()); assert!(result.unwrap().get_documents().await.unwrap().len() == 3); - es.stop().await; + context.embedding_server.stop().await; } #[tokio::test] #[ignore = "TODO: #2384"] async fn test_add_edges_simple() { - let (mutable_graph, _data, _tmp_dir, es) = create_mutable_graph(1748).await; + let context = create_mutable_graph(1748).await; + let mutable_graph = &context.mutable_graph; // First add some nodes. let nodes = vec![ @@ -1116,7 +1141,7 @@ mod tests { let limit = 5; let result = mutable_graph .graph - .vectors + .vectors() .unwrap() .edges_by_similarity(&embedding.into(), limit, None) .execute() @@ -1124,6 +1149,6 @@ mod tests { assert!(result.is_ok()); assert!(result.unwrap().get_documents().await.unwrap().len() == 2); - es.stop().await; + context.embedding_server.stop().await; } } diff --git a/raphtory-graphql/src/model/mod.rs b/raphtory-graphql/src/model/mod.rs index 41b846552a..521f7ff355 100644 --- a/raphtory-graphql/src/model/mod.rs +++ b/raphtory-graphql/src/model/mod.rs @@ -2,7 +2,6 @@ use crate::{ auth::ContextValidation, auth_policy::{AuthorizationPolicy, NamespacePermission}, data::{parent_namespace, require_graph_write, Data, GqlGraphType, PermissionError}, - graph::GraphWithVectors, model::{ graph::{ collection::GqlCollection, graph::GqlGraph, index::IndexSpecInput, @@ -413,8 +412,11 @@ impl Mut { let src_ns = parent_namespace(path); require_namespace_write(ctx, &data.auth_policy, src_ns, path, "move")?; // copy_graph handles dst namespace WRITE check (and src READ, which WRITE implies) - Self::copy_graph(ctx, path, new_path, overwrite).await?; - data.delete_graph(path).await?; + if path != new_path { + // moving with the same path should be a no-op, not delete the graph + Self::copy_graph(ctx, path, new_path, overwrite).await?; + data.delete_graph(path).await?; + } Ok(true) } @@ -438,7 +440,7 @@ impl Mut { let overwrite = overwrite.unwrap_or(false); let src = data.get_raw_graph_with_read_permission(ctx, path).await?; let folder = data.validate_path_for_insert(new_path, overwrite)?; - data.insert_graph(folder, src.graph).await?; + data.insert_graph(folder, src.graph().clone()).await?; Ok(true) } @@ -541,7 +543,11 @@ impl Mut { let data = ctx.data_unchecked::(); #[cfg(feature = "search")] { - let graph = data.get_graph_with_write_permission(ctx, path).await?.graph; + let graph = data + .get_graph_with_write_permission(ctx, path) + .await? + .graph() + .clone(); match index_spec { Some(index_spec) => { let index_spec = index_spec.to_index_spec(graph.clone())?; diff --git a/raphtory-graphql/src/paths.rs b/raphtory-graphql/src/paths.rs index b3f8093663..26771265e2 100644 --- a/raphtory-graphql/src/paths.rs +++ b/raphtory-graphql/src/paths.rs @@ -1,4 +1,6 @@ -use crate::{data::DIRTY_PATH, model::blocking_io, rayon::blocking_compute}; +use crate::{ + data::DIRTY_PATH, graph::GraphWithVectors, model::blocking_io, rayon::blocking_compute, +}; use futures_util::io; use raphtory::{ db::api::{ @@ -6,7 +8,7 @@ use raphtory::{ view::{internal::InternalStorageOps, MaterializedGraph}, }, errors::{GraphError, InvalidPathReason}, - prelude::GraphViewOps, + prelude::{AdditionOps, GraphViewOps}, serialise::{ metadata::GraphMetadata, GraphFolder, GraphPaths, RelativePath, StableDecode, WriteableGraphFolder, ROOT_META_PATH, @@ -345,31 +347,35 @@ impl ValidWriteableGraphFolder { Self::new(path, relative_path) } + /// write graph data to folder (returns a flag to indicate if the graph should be considered dirty) fn write_graph_data_inner( &self, graph: MaterializedGraph, config: Config, - ) -> Result<(), InternalPathValidationError> { - if Extension::disk_storage_enabled() { + ) -> Result<(bool, MaterializedGraph), InternalPathValidationError> { + let is_dirty = if Extension::disk_storage_enabled() { let graph_path = self.graph_folder().graph_path()?; if graph .disk_storage_path() .is_some_and(|path| path == &graph_path) { self.global_path.write_metadata(&graph)?; + (true, graph) } else { - graph.materialize_at_with_config(self.graph_folder(), config)?; + let new_graph = graph.materialize_at_with_config(self.graph_folder(), config)?; + (true, new_graph) } } else { - self.global_path.data_path()?.replace_graph(graph)?; - } - Ok(()) + self.global_path.data_path()?.replace_graph(graph.clone())?; + (false, graph) + }; + Ok(is_dirty) } pub fn write_graph_data( &self, graph: MaterializedGraph, config: Config, - ) -> Result<(), PathValidationError> { + ) -> Result<(bool, MaterializedGraph), PathValidationError> { self.write_graph_data_inner(graph, config) .with_path(self.local_path()) } @@ -390,16 +396,17 @@ impl ValidWriteableGraphFolder { config: Config, ) -> Result<(), PathValidationError> { self.with_internal_errors(|| { - if Extension::disk_storage_enabled() { + let is_dirty = if Extension::disk_storage_enabled() { MaterializedGraph::decode_from_zip_at( ZipArchive::new(bytes)?, self.graph_folder(), config, - )?; + )? + .flush()?; } else { self.global_path.data_path()?.unzip_to_folder(bytes)?; - } - Ok::<(), GraphError>(()) + }; + Ok::<_, GraphError>(is_dirty) }) } diff --git a/raphtory-graphql/src/python/server/mod.rs b/raphtory-graphql/src/python/server/mod.rs index 4a33de6e01..5b9e24a7cb 100644 --- a/raphtory-graphql/src/python/server/mod.rs +++ b/raphtory-graphql/src/python/server/mod.rs @@ -10,6 +10,10 @@ pub(crate) enum BridgeCommand { StopListening, } +pub(crate) struct ServerStarted { + port: u16, +} + pub(crate) fn wait_server(running_server: &mut Option) -> PyResult<()> { let owned_running_server = running_server .take() diff --git a/raphtory-graphql/src/python/server/running_server.rs b/raphtory-graphql/src/python/server/running_server.rs index 7b464d0bdd..27ebb56bb2 100644 --- a/raphtory-graphql/src/python/server/running_server.rs +++ b/raphtory-graphql/src/python/server/running_server.rs @@ -51,26 +51,6 @@ impl PyRunningGraphServer { } } - pub(crate) fn wait_for_server_online(&self, url: &String, timeout_ms: u64) -> PyResult<()> { - let num_intervals = timeout_ms / WAIT_CHECK_INTERVAL_MILLIS; - for _ in 0..num_intervals { - let join_handle = &self.server_handler.as_ref().unwrap().join_handle; - if join_handle.is_finished() { - // this error will never be presented to the user, the result coming from the server task will instead - return Err(PyException::new_err("Server task finished too early")); - } - if is_online(url) { - return Ok(()); - } else { - sleep(Duration::from_millis(WAIT_CHECK_INTERVAL_MILLIS)) - } - } - Err(PyException::new_err(format!( - "Failed to start server in {} milliseconds", - timeout_ms - ))) - } - pub(crate) fn stop_server(&mut self, py: Python) -> PyResult<()> { Self::apply_if_alive(self, |handler| { match handler.sender.send(BridgeCommand::StopServer) { @@ -101,6 +81,11 @@ impl PyRunningGraphServer { }) } + /// Get the port the server is listening on + pub fn port(&self) -> PyResult { + self.apply_if_alive(|handler| Ok(handler.port)) + } + /// Stop the server and wait for it to finish. /// /// Returns: diff --git a/raphtory-graphql/src/python/server/server.rs b/raphtory-graphql/src/python/server/server.rs index 3fccbe0aa0..a8c23c50d5 100644 --- a/raphtory-graphql/src/python/server/server.rs +++ b/raphtory-graphql/src/python/server/server.rs @@ -3,12 +3,15 @@ use crate::{ app_config::AppConfigBuilder, auth_config::PUBLIC_KEY_DECODING_ERR_MSG, otlp_config::TracingLevel, }, - python::server::{running_server::PyRunningGraphServer, wait_server, BridgeCommand}, + python::server::{ + running_server::PyRunningGraphServer, wait_server, BridgeCommand, ServerStarted, + }, server::apply_server_extension, GraphServer, }; +use crossbeam_channel::RecvTimeoutError; use pyo3::{ - exceptions::{PyAttributeError, PyException, PyValueError}, + exceptions::{PyAttributeError, PyException, PyRuntimeError, PyValueError}, prelude::*, }; use raphtory::{ @@ -19,7 +22,7 @@ use raphtory::{ }, vectors::template::{DocumentTemplate, DEFAULT_EDGE_TEMPLATE, DEFAULT_NODE_TEMPLATE}, }; -use std::{path::PathBuf, thread}; +use std::{path::PathBuf, thread, time::Duration}; /// A class for defining and running a Raphtory GraphQL server /// @@ -85,7 +88,6 @@ impl PyGraphServer { signature = ( work_dir, cache_capacity = None, - cache_tti_seconds = None, log_level = None, tracing=None, tracing_level=None, @@ -113,7 +115,6 @@ impl PyGraphServer { fn py_new( work_dir: PathBuf, cache_capacity: Option, - cache_tti_seconds: Option, log_level: Option, tracing: Option, tracing_level: Option, @@ -167,9 +168,6 @@ impl PyGraphServer { if let Some(cache_capacity) = cache_capacity { app_config_builder = app_config_builder.with_cache_capacity(cache_capacity); } - if let Some(cache_tti_seconds) = cache_tti_seconds { - app_config_builder = app_config_builder.with_cache_tti_seconds(cache_tti_seconds); - } app_config_builder = app_config_builder .with_auth_public_key(auth_public_key) .map_err(|_| PyValueError::new_err(PUBLIC_KEY_DECODING_ERR_MSG))?; @@ -304,7 +302,8 @@ impl PyGraphServer { /// Start the server and return a handle to it. /// /// Arguments: - /// port (int): the port to use. Defaults to 1736. + /// port (int, optional): the port to use. If not specified, tries 1736 by default and if that is not available starts on an arbitrary port. + /// If specified and the port is in use, the server will fail to start. /// timeout_ms (int): wait for server to be online. Defaults to 5000. /// /// The server is stopped if not online within timeout_ms but manages to come online as soon as timeout_ms finishes! @@ -312,17 +311,28 @@ impl PyGraphServer { /// Returns: /// RunningGraphServer: The running server #[pyo3( - signature = (port = 1736, timeout_ms = 5000) + signature = (port = None, timeout_ms = 5000) )] - pub fn start(&self, py: Python, port: u16, timeout_ms: u64) -> PyResult { + pub fn start(&self, port: Option, timeout_ms: u64) -> PyResult { let (sender, receiver) = crossbeam_channel::bounded::(1); + let (start_sender, start_receiver) = crossbeam_channel::bounded::(1); let cloned_sender = sender.clone(); let server = self.0.clone(); let join_handle = thread::spawn(move || { block_on(async move { - let handler = server.start_with_port(port); - let running_server = handler.await?; + let running_server = match port { + None => server.start().await?, + Some(port) => server.start_with_port(port).await?, + }; + if let Err(_) = start_sender.send(ServerStarted { + port: running_server.port(), + }) { + // This happens if the other end of the channel doesn't exist + running_server.stop().await; + return Ok(()); + }; + let tokio_sender = running_server._get_sender().clone(); tokio::task::spawn_blocking(move || { match receiver.recv().expect("Failed to wait for cancellation") { @@ -338,35 +348,37 @@ impl PyGraphServer { }) }); - let mut server = PyRunningGraphServer::new(join_handle, sender, port)?; - if let Some(_server_handler) = &server.server_handler { - let url = format!("http://localhost:{port}"); - let result = server.wait_for_server_online(&url, timeout_ms); - match result { - Ok(_) => return Ok(server), - Err(e) => { - PyRunningGraphServer::stop_server(&mut server, py)?; - Err(e) + let ServerStarted { port } = start_receiver + .recv_timeout(Duration::from_millis(timeout_ms)) + .map_err(|err| { + let _ = sender.try_send(BridgeCommand::StopServer); // best effort cleanup + match err { + RecvTimeoutError::Timeout => PyRuntimeError::new_err(format!( + "Failed to start server in {timeout_ms} milliseconds" + )), + RecvTimeoutError::Disconnected => { + PyRuntimeError::new_err("Failed to start server") + } } - } - } else { - Err(PyException::new_err("Failed to start server")) - } + })?; + let server = PyRunningGraphServer::new(join_handle, sender, port)?; + Ok(server) } /// Run the server until completion. /// /// Arguments: - /// port (int): The port to use. Defaults to 1736. + /// port (int, optional): The port to use. If not specified, tries 1736 by default and if that is not available starts on an arbitrary port. + /// If specified and the port is in use, the server will fail to start. /// timeout_ms (int): Timeout for waiting for the server to start. Defaults to 180000. /// /// Returns: /// None: #[pyo3( - signature = (port = 1736, timeout_ms = 180000) + signature = (port = None, timeout_ms = 180000) )] - pub fn run(&self, py: Python, port: u16, timeout_ms: u64) -> PyResult<()> { - let mut server = self.start(py, port, timeout_ms)?.server_handler; + pub fn run(&self, py: Python, port: Option, timeout_ms: u64) -> PyResult<()> { + let mut server = self.start(port, timeout_ms)?.server_handler; py.detach(|| wait_server(&mut server)) } } diff --git a/raphtory-graphql/src/rayon.rs b/raphtory-graphql/src/rayon.rs index cb9352ab98..7bba4dcad4 100644 --- a/raphtory-graphql/src/rayon.rs +++ b/raphtory-graphql/src/rayon.rs @@ -2,14 +2,14 @@ use rayon::{ThreadPool, ThreadPoolBuilder}; use std::sync::LazyLock; use tokio::sync::oneshot; -static WRITE_POOL: LazyLock = LazyLock::new(|| { +pub static WRITE_POOL: LazyLock = LazyLock::new(|| { ThreadPoolBuilder::new() .thread_name(|t| format!("RAP-write-{t}")) .build() .unwrap() }); -static COMPUTE_POOL: LazyLock = LazyLock::new(|| { +pub static COMPUTE_POOL: LazyLock = LazyLock::new(|| { ThreadPoolBuilder::new() .stack_size(16 * 1024 * 1024) .thread_name(|t| format!("RAP-compute-{t}")) @@ -17,6 +17,15 @@ static COMPUTE_POOL: LazyLock = LazyLock::new(|| { .unwrap() }); +pub static EVICT_POOL: LazyLock = LazyLock::new(|| { + ThreadPoolBuilder::new() + .stack_size(16 * 1024 * 1024) + .num_threads(1) + .thread_name(|t| format!("RAP-evict-{t}")) + .build() + .unwrap() +}); + /// Use the rayon threadpool to execute a task /// /// Use this for long-running, compute-heavy work diff --git a/raphtory-graphql/src/server.rs b/raphtory-graphql/src/server.rs index eb41886281..02760119a0 100644 --- a/raphtory-graphql/src/server.rs +++ b/raphtory-graphql/src/server.rs @@ -8,7 +8,7 @@ use crate::{ App, }, observability::open_telemetry::OpenTelemetry, - paths::ExistingGraphFolder, + paths::{ExistingGraphFolder, PathValidationError::IOError}, routes::{health, version, PublicFilesEndpoint}, server::ServerError::SchemaError, GQLError, @@ -19,7 +19,7 @@ use opentelemetry::trace::TracerProvider; use opentelemetry_sdk::trace::{Tracer, TracerProvider as TP}; use poem::{ get, - listener::TcpListener, + listener::{Acceptor, Listener, TcpListener}, middleware::{Compression, CompressionEndpoint, Cors, CorsEndpoint}, web::CompressionLevel, EndpointExt, Route, Server, @@ -32,6 +32,7 @@ use serde_json::json; use std::{ fs::create_dir_all, future::Future, + io::ErrorKind, ops::Deref, path::{Path, PathBuf}, pin::Pin, @@ -50,7 +51,7 @@ use tokio::{ task, task::JoinHandle, }; -use tracing::{debug, info}; +use tracing::{debug, info, warn}; use tracing_subscriber::{ fmt, fmt::format::FmtSpan, layer::SubscriberExt, util::SubscriberInitExt, Registry, }; @@ -232,12 +233,26 @@ impl GraphServer { } /// Start the server on the default port and return a handle to it. + /// If the default port is in use, pub async fn start(&self) -> IoResult { - self.start_with_port(DEFAULT_PORT).await + match self.start_with_port(DEFAULT_PORT).await { + Ok(server) => Ok(server), + Err(err) => { + if matches!(err.kind(), ErrorKind::AddrInUse) { + warn!("Default port {DEFAULT_PORT} already in use, retrying with port=0"); + self.start_with_port(0).await + } else { + Err(err) + } + } + } } /// Start the server on the given port and return a handle to it. pub async fn start_with_port(&self, port: u16) -> IoResult { + let acceptor = TcpListener::bind(format!("0.0.0.0:{port}")) + .into_acceptor() + .await?; // set up opentelemetry first of all let config = self.config.clone(); let filter = config.logging.get_log_env(); @@ -263,16 +278,6 @@ impl GraphServer { let work_dir = self.data.work_dir.clone(); - // Otherwise evictions are only triggered when the cache is actively touched - let cache_clone = self.data.cache.clone(); - let cache_task: AbortOnDrop<()> = AbortOnDrop(tokio::spawn(async move { - let mut interval = tokio::time::interval(std::time::Duration::from_secs(1)); - loop { - interval.tick().await; - cache_clone.run_pending_tasks().await; - } - })); - // it is important that this runs after algorithms have been pushed to PLUGIN_ALGOS static variable let app = self .generate_endpoint(tp.clone().map(|tp| tp.tracer(tracer_name))) @@ -280,7 +285,22 @@ impl GraphServer { let (signal_sender, signal_receiver) = mpsc::channel(1); - info!("UI listening on 0.0.0.0:{port}, live at: http://localhost:{port}"); + let actual_port = acceptor + .local_addr() + .into_iter() + .next() + .unwrap() + .as_socket_addr() + .unwrap() + .port(); + let server_task = Server::new_with_acceptor(acceptor).run_with_graceful_shutdown( + app, + server_termination(signal_receiver, tp), + None, + ); + let server_result = AbortOnDrop(tokio::spawn(server_task)); + + info!("UI listening on 0.0.0.0:{actual_port}, live at: http://localhost:{actual_port}"); debug!( "Server configurations: {}", json!({ @@ -289,14 +309,10 @@ impl GraphServer { }) ); - let server_task = Server::new(TcpListener::bind(format!("0.0.0.0:{port}"))) - .run_with_graceful_shutdown(app, server_termination(signal_receiver, tp), None); - let server_result = AbortOnDrop(tokio::spawn(server_task)); - Ok(RunningGraphServer { signal_sender, server_result, - cache_task, + port: actual_port, }) } @@ -393,13 +409,12 @@ impl Future for AbortOnDrop { pub struct RunningGraphServer { signal_sender: Sender<()>, server_result: AbortOnDrop>, - cache_task: AbortOnDrop<()>, + port: u16, } impl RunningGraphServer { /// Stop the server. pub async fn stop(&self) { - self.cache_task.abort(); let _ignored = self.signal_sender.send(()).await; } @@ -408,6 +423,10 @@ impl RunningGraphServer { self.server_result.await.expect("Server panicked") } + pub fn port(&self) -> u16 { + self.port + } + // TODO: make this optional with some python feature flag pub fn _get_sender(&self) -> &Sender<()> { &self.signal_sender diff --git a/raphtory/Cargo.toml b/raphtory/Cargo.toml index 1629456088..13028b3f55 100644 --- a/raphtory/Cargo.toml +++ b/raphtory/Cargo.toml @@ -187,5 +187,11 @@ proto = [ test-utils = [ "dep:proptest", - "dep:proptest-derive" + "dep:proptest-derive", + "storage/test-utils", + "panic-on-drop" +] + +panic-on-drop = [ + "storage/panic-on-drop" ] diff --git a/raphtory/src/db/api/storage/graph/storage_ops/property_schema.rs b/raphtory/src/db/api/storage/graph/storage_ops/property_schema.rs index 99e88ecc4e..94f44793bc 100644 --- a/raphtory/src/db/api/storage/graph/storage_ops/property_schema.rs +++ b/raphtory/src/db/api/storage/graph/storage_ops/property_schema.rs @@ -1,11 +1,9 @@ +use super::GraphStorage; use crate::db::api::{ properties::internal::{EdgePropertySchemaOps, NodePropertySchemaOps}, view::BoxedLIter, }; use raphtory_api::{core::storage::arc_str::ArcStr, iter::IntoDynBoxed}; -use raphtory_storage::core_ops::CoreGraphOps; - -use super::GraphStorage; impl NodePropertySchemaOps for GraphStorage { fn node_visible_temporal_prop_ids(&self) -> BoxedLIter<'_, usize> { diff --git a/raphtory/src/db/api/storage/storage.rs b/raphtory/src/db/api/storage/storage.rs index 273acd3589..e145f472ae 100644 --- a/raphtory/src/db/api/storage/storage.rs +++ b/raphtory/src/db/api/storage/storage.rs @@ -383,12 +383,10 @@ impl InheritViewOps for Storage {} #[derive(Clone)] pub struct StorageWriteSession<'a> { session: UnlockedSession<'a>, - storage: &'a Storage, } pub struct AtomicAddEdgeSession<'a> { session: AtomicAddEdge<'a, Extension>, - storage: &'a Storage, } impl EdgeWriteLock for AtomicAddEdgeSession<'_> { @@ -538,10 +536,7 @@ impl InternalAdditionOps for Storage { fn write_session(&self) -> Result, Self::Error> { let session = self.graph.write_session()?; - Ok(StorageWriteSession { - session, - storage: self, - }) + Ok(StorageWriteSession { session }) } fn atomic_add_edge( @@ -551,10 +546,7 @@ impl InternalAdditionOps for Storage { e_id: Option, ) -> Result, Self::Error> { let session = self.graph.atomic_add_edge(src, dst, e_id)?; - Ok(AtomicAddEdgeSession { - session, - storage: self, - }) + Ok(AtomicAddEdgeSession { session }) } fn internal_add_node( diff --git a/raphtory/src/db/graph/views/property_redacted_graph.rs b/raphtory/src/db/graph/views/property_redacted_graph.rs index e6eace9e47..c06aa001d7 100644 --- a/raphtory/src/db/graph/views/property_redacted_graph.rs +++ b/raphtory/src/db/graph/views/property_redacted_graph.rs @@ -1,8 +1,7 @@ use crate::db::api::{ properties::internal::{ - EdgePropertySchemaOps, InheritEdgePropertySchemaOps, InheritNodePropertySchemaOps, - InheritTemporalPropertyViewOps, InternalMetadataOps, InternalTemporalPropertiesOps, - NodePropertySchemaOps, + EdgePropertySchemaOps, InheritTemporalPropertyViewOps, InternalMetadataOps, + InternalTemporalPropertiesOps, NodePropertySchemaOps, }, view::{ internal::{ diff --git a/raphtory/src/graph_loader/mod.rs b/raphtory/src/graph_loader/mod.rs index 0234660bdb..586e994227 100644 --- a/raphtory/src/graph_loader/mod.rs +++ b/raphtory/src/graph_loader/mod.rs @@ -103,7 +103,7 @@ use std::{ path::{Path, PathBuf}, time::Duration, }; -use tempfile::{NamedTempFile, PersistError}; +use tempfile::NamedTempFile; use zip::read::ZipArchive; pub mod company_house;