diff --git a/Cargo.lock b/Cargo.lock index f0bd74eec6..b3c26bb8fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2206,7 +2206,7 @@ dependencies = [ "itertools 0.14.0", "parking_lot", "paste", - "petgraph 0.8.3", + "petgraph", "tokio", ] @@ -3950,7 +3950,7 @@ dependencies = [ "permutation", "pin-project", "prost 0.14.3", - "prost-types 0.14.3", + "prost-types", "rand 0.9.4", "roaring 0.11.3", "semver", @@ -4064,7 +4064,7 @@ dependencies = [ "log", "pin-project", "prost 0.14.3", - "prost-build 0.14.3", + "prost-build", "snafu 0.9.0", "tokio", "tracing", @@ -4118,8 +4118,8 @@ dependencies = [ "lz4", "num-traits", "prost 0.14.3", - "prost-build 0.14.3", - "prost-types 0.14.3", + "prost-build", + "prost-types", "rand 0.9.4", "snafu 0.9.0", "strum 0.26.3", @@ -4156,8 +4156,8 @@ dependencies = [ "num-traits", "object_store", "prost 0.14.3", - "prost-build 0.14.3", - "prost-types 0.14.3", + "prost-build", + "prost-types", "snafu 0.9.0", "tokio", "tracing", @@ -4210,8 +4210,8 @@ dependencies = [ "num-traits", "object_store", "prost 0.14.3", - "prost-build 0.14.3", - "prost-types 0.14.3", + "prost-build", + "prost-types", "rand 0.9.4", "rand_distr 0.5.1", "rangemap", @@ -4366,8 +4366,8 @@ dependencies = [ "log", "object_store", "prost 0.14.3", - "prost-build 0.14.3", - "prost-types 0.14.3", + "prost-build", + "prost-types", "rand 0.9.4", "rangemap", "roaring 0.11.3", @@ -5568,16 +5568,6 @@ dependencies = [ "sha2", ] -[[package]] -name = "petgraph" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" -dependencies = [ - "fixedbitset", - "indexmap 2.14.0", -] - [[package]] name = "petgraph" version = "0.8.3" @@ -5932,26 +5922,6 @@ dependencies = [ "prost-derive 0.14.3", ] -[[package]] -name = "prost-build" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" -dependencies = [ - "heck 0.5.0", - "itertools 0.14.0", - "log", - "multimap", - "once_cell", - "petgraph 0.7.1", - "prettyplease", - "prost 0.13.5", - "prost-types 0.13.5", - "regex", - "syn 2.0.117", - "tempfile", -] - [[package]] name = "prost-build" version = "0.14.3" @@ -5962,10 +5932,10 @@ dependencies = [ "itertools 0.14.0", "log", "multimap", - "petgraph 0.8.3", + "petgraph", "prettyplease", "prost 0.14.3", - "prost-types 0.14.3", + "prost-types", "regex", "syn 2.0.117", "tempfile", @@ -5997,15 +5967,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "prost-types" -version = "0.13.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" -dependencies = [ - "prost 0.13.5", -] - [[package]] name = "prost-types" version = "0.14.3" @@ -6373,7 +6334,6 @@ dependencies = [ "itertools 0.13.0", "kdam", "lancedb", - "memmap2", "minijinja", "minijinja-contrib", "moka", @@ -6390,9 +6350,6 @@ dependencies = [ "pretty_assertions", "proptest", "proptest-derive", - "prost 0.13.5", - "prost-build 0.13.5", - "prost-types 0.13.5", "pyo3", "pyo3-arrow", "pythonize", diff --git a/Cargo.toml b/Cargo.toml index 22f903e388..0bf9774c4f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -159,15 +159,11 @@ bytemuck = { version = "1.25.0", features = ["derive"] } ouroboros = "0.18.5" url = "2.5" base64-compat = { package = "base64-compat", version = "1.0.0" } -prost = "0.13.5" -prost-types = "0.13.5" -prost-build = "0.13.5" lazy_static = "1.5.0" pest = "2.8.6" pest_derive = "2.8.6" minijinja = "2.18.0" minijinja-contrib = { version = "2.18.0", features = ["datetime"] } - lancedb = { version = "0.27.2", features = [] } heed = "0.22.0" sqlparser = "0.59.0" diff --git a/Dockerfile b/Dockerfile index 736edc4261..e6f5e5ddcc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,6 @@ RUN cargo chef prepare --recipe-path recipe.json FROM chef AS builder ARG RAPHTORY_PROFILE -RUN apt-get update && apt-get install -y protobuf-compiler COPY --from=planner /app/recipe.json recipe.json RUN cargo chef cook --profile=${RAPHTORY_PROFILE} --recipe-path recipe.json COPY . . diff --git a/python.Dockerfile b/python.Dockerfile index 5862ee75bd..46bf272397 100644 --- a/python.Dockerfile +++ b/python.Dockerfile @@ -7,7 +7,6 @@ WORKDIR /app ENV HOME=/root ENV PYENV_ROOT=$HOME/.pyenv ENV PATH=$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH -RUN apt-get update && apt-get install -y protobuf-compiler RUN curl -fsSL https://pyenv.run | bash RUN PYTHON_VERSION=$(echo ${BASE_PYTHON_IMAGE_TAG} | cut -d'-' -f1) && \ pyenv install ${PYTHON_VERSION} && \ diff --git a/python/Cargo.toml b/python/Cargo.toml index 3fcef1949a..93ed6979c4 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -33,7 +33,6 @@ clam-core = { workspace = true, features = ["python"] } [features] extension-module = ["pyo3/extension-module"] search = ["raphtory/search", "raphtory-graphql/search"] -proto = ["raphtory/proto"] [build-dependencies] diff --git a/python/tox.ini b/python/tox.ini index de2ce2de53..62895cd31f 100644 --- a/python/tox.ini +++ b/python/tox.ini @@ -17,6 +17,7 @@ pass_env = DISK_TEST_MARK [testenv:.pkg] +set_env = MATURIN_PEP517_ARGS="--profile=build-fast" pass_env = MATURIN_PEP517_ARGS [testenv:search] @@ -25,7 +26,7 @@ commands = pytest {posargs} {tty:--color=yes} tests/test_search [testenv:.pkg_search] set_env = - MATURIN_PEP517_ARGS="--features=search,extension-module" + MATURIN_PEP517_ARGS="--features=search,extension-module --profile=build-fast" [testenv:export] diff --git a/raphtory-api/Cargo.toml b/raphtory-api/Cargo.toml index 7e4ca6b83d..14818af8fa 100644 --- a/raphtory-api/Cargo.toml +++ b/raphtory-api/Cargo.toml @@ -56,7 +56,6 @@ python = [ "dep:pyo3", "dep:pyo3-arrow", "dep:display-error-chain", "dep:numpy" ] -proto = [] vectors = [] template = ["dep:minijinja"] search = [] diff --git a/raphtory-benchmark/Cargo.toml b/raphtory-benchmark/Cargo.toml index 5e3ffa56c2..50db5fa3e3 100644 --- a/raphtory-benchmark/Cargo.toml +++ b/raphtory-benchmark/Cargo.toml @@ -57,14 +57,12 @@ name = "materialise" harness = false [[bench]] -name = "proto_encode" +name = "encode" harness = false -required-features = ["proto"] [[bench]] -name = "proto_decode" +name = "decode" harness = false -required-features = ["proto"] [[bench]] name = "search_bench" @@ -87,5 +85,4 @@ harness = false [features] search = ["raphtory/search"] -proto = ["raphtory/proto"] vectors = ["raphtory/vectors"] diff --git a/raphtory-benchmark/benches/base.rs b/raphtory-benchmark/benches/base.rs index c6f65479c3..930f83801d 100644 --- a/raphtory-benchmark/benches/base.rs +++ b/raphtory-benchmark/benches/base.rs @@ -2,8 +2,8 @@ use criterion::{criterion_group, criterion_main, Criterion, Throughput}; use raphtory::{graph_loader::lotr_graph::lotr_graph, prelude::*}; use raphtory_api::core::storage::timeindex::AsTime; use raphtory_benchmark::common::{ - bootstrap_graph, run_graph_ops_benches, run_large_ingestion_benchmarks, - run_proto_decode_benchmark, run_proto_encode_benchmark, + bootstrap_graph, run_decode_benchmark, run_encode_benchmark, run_graph_ops_benches, + run_large_ingestion_benchmarks, }; pub fn base(c: &mut Criterion) { @@ -50,10 +50,10 @@ pub fn base(c: &mut Criterion) { } run_graph_ops_benches(c, "lotr_graph", graph.clone(), layered_graph); - let mut proto_group = c.benchmark_group("lotr_graph"); - run_proto_decode_benchmark(&mut proto_group, graph.clone()); - run_proto_encode_benchmark(&mut proto_group, graph.clone()); - proto_group.finish(); + let mut encode_decode_group = c.benchmark_group("lotr_graph"); + run_decode_benchmark(&mut encode_decode_group, graph.clone()); + run_encode_benchmark(&mut encode_decode_group, graph.clone()); + encode_decode_group.finish(); } criterion_group!(benches, base); diff --git a/raphtory-benchmark/benches/proto_decode.rs b/raphtory-benchmark/benches/decode.rs similarity index 60% rename from raphtory-benchmark/benches/proto_decode.rs rename to raphtory-benchmark/benches/decode.rs index 65b3ca9cff..f739039b19 100644 --- a/raphtory-benchmark/benches/proto_decode.rs +++ b/raphtory-benchmark/benches/decode.rs @@ -1,18 +1,18 @@ use criterion::{criterion_group, criterion_main, Criterion}; use raphtory::graph_loader::{lotr_graph::lotr_graph, sx_superuser_graph::sx_superuser_graph}; -use raphtory_benchmark::common::run_proto_decode_benchmark; +use raphtory_benchmark::common::run_decode_benchmark; fn bench(c: &mut Criterion) { let graph = sx_superuser_graph().unwrap(); - let mut group = c.benchmark_group("proto_sx_superuser"); + let mut group = c.benchmark_group("decode_sx_superuser"); group.sample_size(10); - run_proto_decode_benchmark(&mut group, graph); + run_decode_benchmark(&mut group, graph); group.finish(); - let mut group = c.benchmark_group("proto_lotr"); + let mut group = c.benchmark_group("decode_lotr"); let graph = lotr_graph(); group.sample_size(100); - run_proto_decode_benchmark(&mut group, graph); + run_decode_benchmark(&mut group, graph); group.finish(); } diff --git a/raphtory-benchmark/benches/proto_encode.rs b/raphtory-benchmark/benches/encode.rs similarity index 64% rename from raphtory-benchmark/benches/proto_encode.rs rename to raphtory-benchmark/benches/encode.rs index fc3df41047..45d1946368 100644 --- a/raphtory-benchmark/benches/proto_encode.rs +++ b/raphtory-benchmark/benches/encode.rs @@ -1,12 +1,12 @@ use criterion::{criterion_group, criterion_main, Criterion}; use raphtory::graph_loader::sx_superuser_graph::sx_superuser_graph; -use raphtory_benchmark::common::run_proto_encode_benchmark; +use raphtory_benchmark::common::run_encode_benchmark; fn bench(c: &mut Criterion) { let graph = sx_superuser_graph().unwrap(); - let mut group = c.benchmark_group("proto_sx_superuser"); + let mut group = c.benchmark_group("encode_sx_superuser"); group.sample_size(10); - run_proto_encode_benchmark(&mut group, graph); + run_encode_benchmark(&mut group, graph); group.finish(); } diff --git a/raphtory-benchmark/src/common/mod.rs b/raphtory-benchmark/src/common/mod.rs index 6a0b1384bf..d4f8ed54f8 100644 --- a/raphtory-benchmark/src/common/mod.rs +++ b/raphtory-benchmark/src/common/mod.rs @@ -603,8 +603,8 @@ pub fn run_graph_ops_benches( ); } -pub fn run_proto_encode_benchmark(group: &mut BenchmarkGroup, graph: Graph) { - bench(group, "proto_encode", None, |b: &mut Bencher| { +pub fn run_encode_benchmark(group: &mut BenchmarkGroup, graph: Graph) { + bench(group, "encode", None, |b: &mut Bencher| { b.iter_batched( || TempDir::new().unwrap(), |f| graph.encode(f.path()).unwrap(), @@ -613,10 +613,10 @@ pub fn run_proto_encode_benchmark(group: &mut BenchmarkGroup, graph: G }); } -pub fn run_proto_decode_benchmark(group: &mut BenchmarkGroup, graph: Graph) { +pub fn run_decode_benchmark(group: &mut BenchmarkGroup, graph: Graph) { let f = TempDir::new().unwrap(); graph.encode(f.path()).unwrap(); - bench(group, "proto_decode", None, |b| { + bench(group, "decode", None, |b| { b.iter(|| Graph::decode(f.path()).unwrap()) }) } diff --git a/raphtory-tests/tests/db_tests.rs b/raphtory-tests/tests/db_tests.rs index 8c655cc4d5..6af8bab3de 100644 --- a/raphtory-tests/tests/db_tests.rs +++ b/raphtory-tests/tests/db_tests.rs @@ -2,8 +2,6 @@ use bigdecimal::BigDecimal; use chrono::NaiveDateTime; use itertools::Itertools; use proptest::{arbitrary::any, prop_assert, prop_assert_eq, proptest, sample::subsequence}; -#[cfg(feature = "proto")] -use raphtory::serialise::StableDecode; use raphtory::{ algorithms::{ centrality::{degree_centrality::degree_centrality, pagerank::unweighted_page_rank}, @@ -52,10 +50,11 @@ use std::{ ops::{Deref, Range}, sync::Arc, }; -#[cfg(feature = "proto")] -use tempfile::TempDir; use tracing::{error, info}; +#[cfg(feature = "io")] +use {raphtory::serialise::StableDecode, tempfile::TempDir}; + #[test] fn edge_metadata() -> Result<(), GraphError> { let g = Graph::new(); @@ -759,7 +758,7 @@ fn props_with_layers() { } #[test] -#[cfg(feature = "proto")] +#[cfg(feature = "io")] fn graph_save_to_load_from_file() { let vs = vec![ (1, 1, 2), @@ -2962,18 +2961,6 @@ fn test_node_state_merge() { assert_eq!(m2.values().num_rows(), graph.count_nodes()); } -#[test] -#[cfg(feature = "proto")] -fn save_load_serial() { - let g = Graph::new(); - g.add_edge(0, 0, 1, NO_PROPS, None).unwrap(); - let dir = tempfile::tempdir().unwrap(); - let file_path = dir.path().join("abcd11"); - g.encode(&file_path).unwrap(); - let gg = Graph::decode(&file_path).unwrap(); - assert_graph_equal(&g, &gg); -} - #[test] fn test_node_type_changes() { let g = Graph::new(); diff --git a/raphtory-tests/tests/serialise_test.rs b/raphtory-tests/tests/serialise_test.rs index 5e69d1fdc1..797cc7d370 100644 --- a/raphtory-tests/tests/serialise_test.rs +++ b/raphtory-tests/tests/serialise_test.rs @@ -1,10 +1,8 @@ -#[cfg(all(test, feature = "test-utils", feature = "proto"))] +#[cfg(test)] mod serialise_test { - use arrow::{array::types::Int32Type, datatypes::UInt8Type}; use chrono::{DateTime, NaiveDateTime}; use itertools::Itertools; - #[cfg(feature = "proto")] use proptest::proptest; use raphtory::{ db::{ diff --git a/raphtory/Cargo.toml b/raphtory/Cargo.toml index 47d877aecd..73f26f9efa 100644 --- a/raphtory/Cargo.toml +++ b/raphtory/Cargo.toml @@ -75,10 +75,6 @@ reqwest = { workspace = true, features = ["blocking"], optional = true } tokio = { workspace = true, optional = true } kdam = { workspace = true, optional = true } -# proto optional dependencies -memmap2 = { workspace = true, optional = true } -prost = { workspace = true, optional = true } -prost-types = { workspace = true, optional = true } # search optional dependencies tantivy = { workspace = true, optional = true } @@ -122,8 +118,6 @@ proptest-derive.workspace = true [target.'cfg(target_os = "macos")'.dependencies] tikv-jemallocator = "0.6.1" -[build-dependencies] -prost-build = { workspace = true, optional = true } [features] default = [] # we can't depend on ourselves but we want to share test-utils @@ -144,7 +138,7 @@ io = [ ] # search -search = ["dep:tantivy", "dep:tempfile", "proto", "io"] +search = ["dep:tantivy", "dep:tempfile", "io"] # vectors vectors = [ "dep:futures-util", @@ -176,12 +170,3 @@ python = [ "raphtory-core/python", "kdam/notebook", ] - - -proto = [ - "dep:prost", - "dep:prost-types", - "dep:prost-build", - "dep:memmap2", - "io", -] diff --git a/raphtory/build.rs b/raphtory/build.rs index f424e39b19..7acbc3f99d 100644 --- a/raphtory/build.rs +++ b/raphtory/build.rs @@ -1,17 +1,5 @@ use std::io::Result; -#[cfg(feature = "proto")] -fn main() -> Result<()> { - prost_build::compile_protos(&["src/serialise/graph.proto"], &["src/serialise"])?; - println!("cargo::rerun-if-changed=src/serialise/graph.proto"); - - println!("cargo::rustc-check-cfg=cfg(has_debug_symbols)"); - if let Ok("true" | "1" | "2") = std::env::var("DEBUG").as_deref() { - println!("cargo::rustc-cfg=has_debug_symbols"); - } - Ok(()) -} -#[cfg(not(feature = "proto"))] fn main() -> Result<()> { println!("cargo::rustc-check-cfg=cfg(has_debug_symbols)"); if let Ok("true" | "1" | "2") = std::env::var("DEBUG").as_deref() { diff --git a/raphtory/src/errors.rs b/raphtory/src/errors.rs index 691a1df284..e75edf6899 100644 --- a/raphtory/src/errors.rs +++ b/raphtory/src/errors.rs @@ -105,18 +105,6 @@ pub fn into_load_err(err: impl Into) -> LoadError { err.into() } -#[cfg(feature = "proto")] -#[derive(thiserror::Error, Debug)] -pub enum WriteError { - #[cfg(feature = "proto")] - #[error("Unrecoverable disk error: {0}, resetting file size failed: {1}")] - FatalWriteError(io::Error, io::Error), - - #[cfg(feature = "proto")] - #[error("Failed to write delta to cache: {0}")] - WriteError(#[from] io::Error), -} - pub type GraphResult = Result; pub fn into_graph_err(err: impl Into) -> GraphError { @@ -335,35 +323,10 @@ pub enum GraphError { #[error("Illegal set error {0}")] IllegalSet(String), - #[cfg(feature = "proto")] - #[error("Protobuf encode error{0}")] - DecodeError(#[from] prost::DecodeError), - - #[cfg(feature = "proto")] - #[error( - "Cannot recover from write failure {write_err}, new updates are invalid: {decode_err}" - )] - FatalDecodeError { - write_err: WriteError, - decode_err: prost::DecodeError, - }, - - #[cfg(feature = "proto")] - #[error("Cache write error: {0}")] - CacheWriteError(#[from] WriteError), - - #[cfg(feature = "proto")] - #[error("Protobuf decode error{0}")] - EncodeError(#[from] prost::EncodeError), - #[cfg(feature = "io")] #[error("Cannot write graph into non empty folder {0}")] NonEmptyGraphFolder(PathBuf), - #[cfg(feature = "proto")] - #[error("Cache is not initialised")] - CacheNotInnitialised, - #[error("Immutable graph is .. immutable!")] AttemptToMutateImmutableGraph, diff --git a/raphtory/src/search/searcher.rs b/raphtory/src/search/searcher.rs index a77661c4af..cda7379fd9 100644 --- a/raphtory/src/search/searcher.rs +++ b/raphtory/src/search/searcher.rs @@ -277,30 +277,4 @@ mod search_tests { assert_eq!(results, vec![("raphtory".into(), "pometry".into())]); } } - - #[test] - #[cfg(feature = "proto")] - #[ignore = "this test is for experiments with the jira graph"] - fn load_jira_graph() -> Result<(), GraphError> { - global_info_logger(); - - let graph = Graph::decode("/tmp/graphs/jira").expect("failed to load graph"); - assert!(graph.count_nodes() > 0); - - let now = SystemTime::now(); - - let elapsed = now.elapsed()?.as_secs(); - info!("indexing took: {:?}", elapsed); - graph.create_index_in_ram()?; - - let filter = NodeFilter::name().eq("DEV-1690"); - let issues = graph.search_nodes(filter, 5, 0)?; - - assert!(!issues.is_empty()); - - let names = issues.into_iter().map(|v| v.name()).collect::>(); - info!("names: {:?}", names); - - Ok(()) - } } diff --git a/raphtory/src/serialise/graph.proto b/raphtory/src/serialise/graph.proto deleted file mode 100644 index 230953f484..0000000000 --- a/raphtory/src/serialise/graph.proto +++ /dev/null @@ -1,273 +0,0 @@ -syntax = "proto3"; - -package serialise; - -enum GraphType { - EVENT = 0; - PERSISTENT = 1; -} - -message Graph { - GraphType graph_type = 1; - repeated NewMeta metas = 2; - repeated NewNode nodes = 3; - repeated NewEdge edges = 4; - repeated GraphUpdate updates = 5; -} - -message NewMeta { - oneof meta { - NewNodeType new_node_type = 2; - NewNodeCProp new_node_cprop = 3; - NewNodeTProp new_node_tprop = 5; - NewGraphCProp new_graph_cprop = 7; - NewGraphTProp new_graph_tprop = 9; - NewLayer new_layer = 11; - NewEdgeCProp new_edge_cprop = 14; - NewEdgeTProp new_edge_tprop = 16; - } - - message NewNodeType { - string name = 1; - uint64 id = 2; - } - - message NewGraphCProp { - string name = 1; - uint64 id = 2; - } - - message NewGraphTProp { - string name = 1; - uint64 id = 2; - PropType.PropType p_type = 3 [deprecated = true]; - PropType.PType p_type2 = 4; - } - - message NewNodeCProp { - string name = 1; - uint64 id = 2; - PropType.PropType p_type = 3 [deprecated = true]; - PropType.PType p_type2 = 4; - } - - message NewNodeTProp { - string name = 1; - uint64 id = 2; - PropType.PropType p_type = 3[deprecated = true]; - PropType.PType p_type2 = 4; - } - - message NewEdgeCProp { - string name = 1; - uint64 id = 2; - PropType.PropType p_type = 3[deprecated = true]; - PropType.PType p_type2 = 4; - } - - message NewEdgeTProp { - string name = 1; - uint64 id = 2; - PropType.PropType p_type = 3 [deprecated = true]; - PropType.PType p_type2 = 4; - } - - message NewLayer { - string name = 1; - uint64 id = 2; - } -} - -message NewNode { - oneof gid { - string gid_str = 1; - uint64 gid_u64 = 2; - }; - uint64 vid = 3; - uint64 type_id = 4; -} - -message NewEdge{ - uint64 src = 1; - uint64 dst = 2; - uint64 eid = 3; -} - -message GraphUpdate { - oneof update { - UpdateNodeCProps update_node_cprops = 1; - UpdateNodeTProps update_node_tprops = 2; - - UpdateGraphCProps update_graph_cprops = 3; - UpdateGraphTProps update_graph_tprops = 4; - - DelEdge del_edge = 5; - UpdateEdgeCProps update_edge_cprops = 6; - UpdateEdgeTProps update_edge_tprops = 7; - - UpdateNodeType update_node_type = 8; - } - - message UpdateNodeTProps { - uint64 id = 1; - int64 time = 2; - uint64 secondary = 3; - repeated PropPair properties = 5; - } - - message UpdateNodeCProps { - uint64 id = 1; - repeated PropPair properties = 2; - } - - message UpdateNodeType { - uint64 id = 1; - uint64 type_id = 2; - } - - message UpdateEdgeTProps { - uint64 eid = 1; - int64 time = 2; - uint64 secondary = 3; - uint64 layer_id = 4; - repeated PropPair properties = 5; - } - - message DelEdge{ - uint64 eid = 1; - int64 time = 2; - uint64 secondary = 3; - uint64 layer_id = 4; - } - - message UpdateEdgeCProps { - uint64 eid = 1; - uint64 layer_id = 2; - repeated PropPair properties = 3; - } - - message UpdateGraphCProps { - repeated PropPair properties = 1; - } - - message UpdateGraphTProps { - int64 time = 1; - uint64 secondary = 2; - repeated PropPair properties = 3; - } - - message PropPair { - uint64 key = 1; - Prop value = 2; - } -} - -message PropType { - enum PropType { - Str = 0; - U8 = 1; - U16 = 2; - I32 = 3; - I64 = 4; - U32 = 5; - U64 = 6; - F32 = 7; - F64 = 8; - Bool = 9; - List = 10; - Map = 11; - NDTime = 12; - Graph = 13 [deprecated = true]; - PersistentGraph = 14 [deprecated = true]; - Document = 15; - DTime = 16; - } - - message Array { - PropType p_type = 1; - } - - message Scalar { - PropType p_type = 1; - } - - message Decimal{ - uint32 scale = 1; - } - - message PType{ - oneof kind { - Scalar scalar = 1; - Array array = 2; - Decimal decimal = 3; - } - } -} - -message Prop { - oneof value { - string str = 1; - uint32 u8 = 2; // Note: Protobuf does not have a UInt8 type, using uint32 instead. - uint32 u16 = 3; // Note: Protobuf does not have a UInt16 type, using uint32 instead. - int32 i32 = 4; - int64 i64 = 5; - uint32 u32 = 6; - uint64 u64 = 7; - float f32 = 8; - double f64 = 9; - bool bool_value = 10; - Props prop = 11; - Dict map = 12; - Graph graph = 13 [deprecated = true]; // Assuming Data can be represented as bytes. - Graph persistentGraph = 14 [deprecated = true]; // Assuming Data can be represented as bytes. - NDTime ndTime = 15; - string dTime = 16; - DocumentInput documentInput = 17; - Array array = 18; - string decimal = 19; - } - - message Array { - bytes data = 1; - } - - message NDTime{ - uint32 year = 1; - uint32 month = 2; - uint32 day = 3; - uint32 hour = 4; - uint32 minute = 5; - uint32 second = 6; - uint32 nanos = 7; - } - - message Dict{ - map map = 1; - } - - message Props{ - repeated Prop properties = 1; - } - - message DocumentInput { - string content = 1; - Lifespan life = 2; - } - - message Lifespan { - oneof l_type { - Interval interval = 1; - Event event = 2; - // Inherited is represented by the absence of both interval and event. - } - - message Interval { - int64 start = 1; - int64 end = 2; - } - - message Event { - int64 time = 1; - } - } -} diff --git a/raphtory/src/serialise/mod.rs b/raphtory/src/serialise/mod.rs index ec33629745..7b49f01200 100644 --- a/raphtory/src/serialise/mod.rs +++ b/raphtory/src/serialise/mod.rs @@ -3,12 +3,7 @@ pub mod metadata; pub mod parquet; -#[cfg(feature = "proto")] -pub mod proto; mod serialise; pub use graph_folder::*; pub use serialise::{StableDecode, StableEncode}; - -#[cfg(feature = "proto")] -pub use proto::proto_generated::Graph as ProtoGraph; diff --git a/raphtory/src/serialise/proto/ext.rs b/raphtory/src/serialise/proto/ext.rs deleted file mode 100644 index bd93a9843c..0000000000 --- a/raphtory/src/serialise/proto/ext.rs +++ /dev/null @@ -1,733 +0,0 @@ -use super::proto_generated::{ - self, - graph_update::{ - DelEdge, PropPair, Update, UpdateEdgeCProps, UpdateEdgeTProps, UpdateGraphCProps, - UpdateGraphTProps, UpdateNodeCProps, UpdateNodeTProps, UpdateNodeType, - }, - new_meta::{ - Meta, NewEdgeCProp, NewEdgeTProp, NewGraphCProp, NewGraphTProp, NewLayer, NewNodeCProp, - NewNodeTProp, NewNodeType, - }, - new_node, prop, - prop_type::{Array as ArrayType, PType, PropType as SPropType, Scalar as ScalarType}, - GraphUpdate, NewEdge, NewMeta, NewNode, -}; -use crate::errors::GraphError; -use chrono::{DateTime, Datelike, NaiveDate, NaiveDateTime, NaiveTime, Timelike}; -use raphtory_api::core::{ - entities::{ - properties::prop::{Prop, PropType}, - GidRef, LayerId, EID, VID, - }, - storage::{ - arc_str::ArcStr, - timeindex::{AsTime, EventTime}, - }, -}; -use std::{borrow::Borrow, collections::HashMap, sync::Arc}; - -fn as_proto_prop_type(p_type: &PropType) -> Option { - let val = match p_type { - PropType::Str => SPropType::Str, - PropType::U8 => SPropType::U8, - PropType::U16 => SPropType::U16, - PropType::U32 => SPropType::U32, - PropType::I32 => SPropType::I32, - PropType::I64 => SPropType::I64, - PropType::U64 => SPropType::U64, - PropType::F32 => SPropType::F32, - PropType::F64 => SPropType::F64, - PropType::Bool => SPropType::Bool, - PropType::NDTime => SPropType::NdTime, - PropType::DTime => SPropType::DTime, - PropType::Map(_) => SPropType::Map, - PropType::List(_) => SPropType::List, - _ => { - return None; - } - }; - Some(val) -} - -fn as_proto_prop_type2(p_type: &PropType) -> Option { - match p_type { - PropType::List(tpe) => { - let prop_type = as_proto_prop_type(tpe)?; - Some(PType { - kind: Some(proto_generated::prop_type::p_type::Kind::Array(ArrayType { - p_type: prop_type.into(), - })), - }) - } - _ => Some(PType { - kind: Some(proto_generated::prop_type::p_type::Kind::Scalar( - ScalarType { - p_type: as_proto_prop_type(p_type)?.into(), - }, - )), - }), - } -} - -fn as_prop_type2(p_type: PType) -> Option { - match p_type.kind? { - proto_generated::prop_type::p_type::Kind::Scalar(scalar) => as_prop_type(scalar.p_type()), - proto_generated::prop_type::p_type::Kind::Array(array) => { - let p_type = as_prop_type(array.p_type())?; - Some(PropType::List(Box::new(p_type))) - } - proto_generated::prop_type::p_type::Kind::Decimal(decimal) => Some(PropType::Decimal { - scale: decimal.scale as i64, - }), - } -} - -pub fn as_prop_type(p_type: SPropType) -> Option { - // for backwards compatibility we may skip some types - match p_type { - SPropType::Str => Some(PropType::Str), - SPropType::U8 => Some(PropType::U8), - SPropType::U16 => Some(PropType::U16), - SPropType::U32 => Some(PropType::U32), - SPropType::I32 => Some(PropType::I32), - SPropType::I64 => Some(PropType::I64), - SPropType::U64 => Some(PropType::U64), - SPropType::F32 => Some(PropType::F32), - SPropType::F64 => Some(PropType::F64), - SPropType::Bool => Some(PropType::Bool), - SPropType::List => Some(PropType::List(Box::new(PropType::Empty))), - SPropType::Map => Some(PropType::Map(HashMap::new().into())), - SPropType::NdTime => Some(PropType::NDTime), - SPropType::DTime => Some(PropType::DTime), - SPropType::Document => None, - SPropType::Graph => None, - SPropType::PersistentGraph => None, - } -} - -impl NewEdge { - pub fn src(&self) -> VID { - VID(self.src as usize) - } - - pub fn dst(&self) -> VID { - VID(self.dst as usize) - } - - pub fn eid(&self) -> EID { - EID(self.eid as usize) - } -} - -impl DelEdge { - pub fn eid(&self) -> EID { - EID(self.eid as usize) - } - - pub fn layer_id(&self) -> usize { - self.layer_id as usize - } - - pub fn time(&self) -> EventTime { - EventTime(self.time, self.secondary as usize) - } -} - -impl UpdateEdgeCProps { - pub fn eid(&self) -> EID { - EID(self.eid as usize) - } - - pub fn layer_id(&self) -> usize { - self.layer_id as usize - } - - pub fn props(&self) -> impl Iterator> + '_ { - self.properties - .iter() - .map(as_prop) - .filter_map(|r| r.transpose()) - } -} - -impl UpdateEdgeTProps { - pub fn eid(&self) -> EID { - EID(self.eid as usize) - } - - pub fn layer_id(&self) -> usize { - self.layer_id as usize - } - - pub fn time(&self) -> EventTime { - EventTime(self.time, self.secondary as usize) - } - - pub fn has_props(&self) -> bool { - !self.properties.is_empty() - } - - pub fn props(&self) -> impl Iterator> + '_ { - self.properties - .iter() - .map(as_prop) - .filter_map(|r| r.transpose()) - } -} - -impl UpdateNodeType { - pub fn vid(&self) -> VID { - VID(self.id as usize) - } - - pub fn type_id(&self) -> usize { - self.type_id as usize - } -} - -impl UpdateNodeCProps { - pub fn vid(&self) -> VID { - VID(self.id as usize) - } - - pub fn props(&self) -> impl Iterator> + '_ { - self.properties - .iter() - .map(as_prop) - .filter_map(|r| r.transpose()) - } -} - -impl UpdateNodeTProps { - pub fn vid(&self) -> VID { - VID(self.id as usize) - } - - pub fn time(&self) -> EventTime { - EventTime(self.time, self.secondary as usize) - } - - pub fn props(&self) -> impl Iterator> + '_ { - self.properties - .iter() - .map(as_prop) - .filter_map(|r| r.transpose()) - } -} - -pub(crate) trait PropTypeExt { - fn p_type(&self) -> SPropType; - fn p_type2(&self) -> Option<&PType>; - - fn prop_type(&self) -> PropType { - self.p_type2() - .and_then(|p_type| as_prop_type2(*p_type)) - .or_else(|| as_prop_type(self.p_type())) - .unwrap_or(PropType::Empty) - } -} - -impl PropTypeExt for NewNodeCProp { - #[allow(deprecated)] - fn p_type(&self) -> SPropType { - self.p_type() - } - - fn p_type2(&self) -> Option<&PType> { - self.p_type2.as_ref() - } -} - -impl PropTypeExt for NewNodeTProp { - #[allow(deprecated)] - fn p_type(&self) -> SPropType { - self.p_type() - } - - fn p_type2(&self) -> Option<&PType> { - self.p_type2.as_ref() - } -} - -impl PropTypeExt for NewEdgeCProp { - #[allow(deprecated)] - fn p_type(&self) -> SPropType { - self.p_type() - } - - fn p_type2(&self) -> Option<&PType> { - self.p_type2.as_ref() - } -} - -impl PropTypeExt for NewEdgeTProp { - #[allow(deprecated)] - fn p_type(&self) -> SPropType { - self.p_type() - } - - fn p_type2(&self) -> Option<&PType> { - self.p_type2.as_ref() - } -} - -impl PropTypeExt for NewGraphTProp { - #[allow(deprecated)] - fn p_type(&self) -> SPropType { - self.p_type() - } - - fn p_type2(&self) -> Option<&PType> { - self.p_type2.as_ref() - } -} - -impl NewMeta { - fn new(new_meta: Meta) -> Self { - Self { - meta: Some(new_meta), - } - } - - fn new_graph_cprop(key: &str, id: usize) -> Self { - let inner = NewGraphCProp { - name: key.to_string(), - id: id as u64, - }; - Self::new(Meta::NewGraphCprop(inner)) - } - - fn new_graph_tprop(key: &str, id: usize, dtype: &PropType) -> Self { - let mut inner = NewGraphTProp::default(); - inner.name = key.to_string(); - inner.id = id as u64; - inner.p_type2 = as_proto_prop_type2(dtype); - Self::new(Meta::NewGraphTprop(inner)) - } - - fn new_node_cprop(key: &str, id: usize, dtype: &PropType) -> Self { - let mut inner = NewNodeCProp::default(); - inner.name = key.to_string(); - inner.id = id as u64; - inner.p_type2 = as_proto_prop_type2(dtype); - Self::new(Meta::NewNodeCprop(inner)) - } - - fn new_node_tprop(key: &str, id: usize, dtype: &PropType) -> Self { - let mut inner = NewNodeTProp::default(); - inner.name = key.to_string(); - inner.id = id as u64; - inner.p_type2 = as_proto_prop_type2(dtype); - Self::new(Meta::NewNodeTprop(inner)) - } - - fn new_edge_cprop(key: &str, id: usize, dtype: &PropType) -> Self { - let mut inner = NewEdgeCProp::default(); - inner.name = key.to_string(); - inner.id = id as u64; - inner.p_type2 = as_proto_prop_type2(dtype); - Self::new(Meta::NewEdgeCprop(inner)) - } - - fn new_edge_tprop(key: &str, id: usize, dtype: &PropType) -> Self { - let mut inner = NewEdgeTProp::default(); - inner.name = key.to_string(); - inner.id = id as u64; - inner.p_type2 = as_proto_prop_type2(dtype); - Self::new(Meta::NewEdgeTprop(inner)) - } - - fn new_layer(layer: &str, id: usize) -> Self { - let mut inner = NewLayer::default(); - inner.name = layer.to_string(); - inner.id = id as u64; - Self::new(Meta::NewLayer(inner)) - } - - fn new_node_type(node_type: &str, id: usize) -> Self { - let mut inner = NewNodeType::default(); - inner.name = node_type.to_string(); - inner.id = id as u64; - Self::new(Meta::NewNodeType(inner)) - } -} - -impl GraphUpdate { - fn new(update: Update) -> Self { - Self { - update: Some(update), - } - } - - fn update_graph_cprops(values: impl Iterator)>) -> Self { - let inner = UpdateGraphCProps::new(values); - Self::new(Update::UpdateGraphCprops(inner)) - } - - fn update_graph_tprops( - time: EventTime, - values: impl IntoIterator)>, - ) -> Self { - let inner = UpdateGraphTProps::new(time, values); - Self::new(Update::UpdateGraphTprops(inner)) - } - - fn update_node_type(node_id: VID, type_id: usize) -> Self { - let inner = UpdateNodeType { - id: node_id.as_u64(), - type_id: type_id as u64, - }; - Self::new(Update::UpdateNodeType(inner)) - } - - fn update_node_cprops( - node_id: VID, - properties: impl Iterator)>, - ) -> Self { - let properties = collect_proto_props(properties); - let inner = UpdateNodeCProps { - id: node_id.as_u64(), - properties, - }; - Self::new(Update::UpdateNodeCprops(inner)) - } - - fn update_node_tprops( - node_id: VID, - time: EventTime, - properties: impl Iterator)>, - ) -> Self { - let properties = collect_proto_props(properties); - let inner = UpdateNodeTProps { - id: node_id.as_u64(), - time: time.t(), - secondary: time.i() as u64, - properties, - }; - Self::new(Update::UpdateNodeTprops(inner)) - } - - fn update_edge_tprops( - eid: EID, - time: EventTime, - layer_id: LayerId, - properties: impl Iterator)>, - ) -> Self { - let properties = collect_proto_props(properties); - let inner = UpdateEdgeTProps { - eid: eid.0 as u64, - time: time.t(), - secondary: time.i() as u64, - layer_id: layer_id.0 as u64, - properties, - }; - Self::new(Update::UpdateEdgeTprops(inner)) - } - - fn update_edge_cprops( - eid: EID, - layer_id: LayerId, - properties: impl Iterator)>, - ) -> Self { - let properties = collect_proto_props(properties); - let inner = UpdateEdgeCProps { - eid: eid.0 as u64, - layer_id: layer_id.0 as u64, - properties, - }; - Self::new(Update::UpdateEdgeCprops(inner)) - } - - fn del_edge(eid: EID, layer_id: LayerId, time: EventTime) -> Self { - let inner = DelEdge { - eid: eid.as_u64(), - time: time.t(), - secondary: time.i() as u64, - layer_id: layer_id.0 as u64, - }; - Self::new(Update::DelEdge(inner)) - } -} - -impl UpdateGraphCProps { - fn new(values: impl Iterator)>) -> Self { - let properties = collect_proto_props(values); - UpdateGraphCProps { properties } - } -} - -impl UpdateGraphTProps { - fn new(time: EventTime, values: impl IntoIterator)>) -> Self { - let properties = collect_proto_props(values); - UpdateGraphTProps { - time: time.t(), - secondary: time.i() as u64, - properties, - } - } -} - -impl PropPair { - fn new(key: usize, value: &Prop) -> Self { - PropPair { - key: key as u64, - value: Some(as_proto_prop(value)), - } - } -} - -impl proto_generated::Graph { - pub fn new_edge(&mut self, src: VID, dst: VID, eid: EID) { - let edge = NewEdge { - src: src.as_u64(), - dst: dst.as_u64(), - eid: eid.as_u64(), - }; - self.edges.push(edge); - } - - pub fn new_node(&mut self, gid: GidRef, vid: VID, type_id: usize) { - let type_id = type_id as u64; - let gid = match gid { - GidRef::U64(id) => new_node::Gid::GidU64(id), - GidRef::Str(name) => new_node::Gid::GidStr(name.to_string()), - }; - let node = NewNode { - type_id, - gid: Some(gid), - vid: vid.as_u64(), - }; - self.nodes.push(node); - } - - pub fn new_graph_cprop(&mut self, key: &str, id: usize) { - self.metas.push(NewMeta::new_graph_cprop(key, id)); - } - - pub fn new_graph_tprop(&mut self, key: &str, id: usize, dtype: &PropType) { - self.metas.push(NewMeta::new_graph_tprop(key, id, dtype)); - } - - pub fn new_node_cprop(&mut self, key: &str, id: usize, dtype: &PropType) { - self.metas.push(NewMeta::new_node_cprop(key, id, dtype)); - } - - pub fn new_node_tprop(&mut self, key: &str, id: usize, dtype: &PropType) { - self.metas.push(NewMeta::new_node_tprop(key, id, dtype)); - } - - pub fn new_edge_cprop(&mut self, key: &str, id: usize, dtype: &PropType) { - self.metas.push(NewMeta::new_edge_cprop(key, id, dtype)); - } - - pub fn new_edge_tprop(&mut self, key: &str, id: usize, dtype: &PropType) { - self.metas.push(NewMeta::new_edge_tprop(key, id, dtype)) - } - - pub fn new_layer(&mut self, layer: &str, id: usize) { - self.metas.push(NewMeta::new_layer(layer, id)); - } - - pub fn new_node_type(&mut self, node_type: &str, id: usize) { - self.metas.push(NewMeta::new_node_type(node_type, id)); - } - - pub fn update_graph_cprops( - &mut self, - values: impl Iterator)>, - ) { - self.updates.push(GraphUpdate::update_graph_cprops(values)); - } - - pub fn update_graph_tprops( - &mut self, - time: EventTime, - values: impl IntoIterator)>, - ) { - self.updates - .push(GraphUpdate::update_graph_tprops(time, values)); - } - - pub fn update_node_type(&mut self, node_id: VID, type_id: usize) { - self.updates - .push(GraphUpdate::update_node_type(node_id, type_id)) - } - pub fn update_node_cprops( - &mut self, - node_id: VID, - properties: impl Iterator)>, - ) { - self.updates - .push(GraphUpdate::update_node_cprops(node_id, properties)); - } - - pub fn update_node_tprops( - &mut self, - node_id: VID, - time: EventTime, - properties: impl Iterator)>, - ) { - self.updates - .push(GraphUpdate::update_node_tprops(node_id, time, properties)); - } - - pub fn update_edge_tprops( - &mut self, - eid: EID, - time: EventTime, - layer_id: LayerId, - properties: impl Iterator)>, - ) { - self.updates.push(GraphUpdate::update_edge_tprops( - eid, time, layer_id, properties, - )); - } - - pub fn update_edge_cprops( - &mut self, - eid: EID, - layer_id: LayerId, - properties: impl Iterator)>, - ) { - self.updates - .push(GraphUpdate::update_edge_cprops(eid, layer_id, properties)); - } - - pub fn del_edge(&mut self, eid: EID, layer_id: LayerId, time: EventTime) { - self.updates - .push(GraphUpdate::del_edge(eid, layer_id, time)) - } -} - -fn as_prop(prop_pair: &PropPair) -> Result, GraphError> { - let PropPair { key, value } = prop_pair; - let value = value.as_ref().expect("Missing prop value"); - let value = value.value.as_ref(); - let value = as_prop_value(value)?; - - Ok(value.map(|value| (*key as usize, value))) -} - -fn as_prop_value(value: Option<&prop::Value>) -> Result, GraphError> { - let value = match value.expect("Missing prop value") { - prop::Value::BoolValue(b) => Some(Prop::Bool(*b)), - prop::Value::U8(u) => Some(Prop::U8((*u).try_into().unwrap())), - prop::Value::U16(u) => Some(Prop::U16((*u).try_into().unwrap())), - prop::Value::U32(u) => Some(Prop::U32(*u)), - prop::Value::I32(i) => Some(Prop::I32(*i)), - prop::Value::I64(i) => Some(Prop::I64(*i)), - prop::Value::U64(u) => Some(Prop::U64(*u)), - prop::Value::F32(f) => Some(Prop::F32(*f)), - prop::Value::F64(f) => Some(Prop::F64(*f)), - prop::Value::Str(s) => Some(Prop::Str(ArcStr::from(s.as_str()))), - prop::Value::Prop(props) => Some(Prop::List( - props - .properties - .iter() - .filter_map(|prop| as_prop_value(prop.value.as_ref()).transpose()) - .collect::, _>>()? - .into(), - )), - prop::Value::Map(dict) => Some(Prop::Map(Arc::new( - dict.map - .iter() - .filter_map(|(k, v)| { - as_prop_value(v.value.as_ref()) - .map(|v| v.map(|v| (ArcStr::from(k.as_str()), v))) - .transpose() - }) - .collect::>()?, - ))), - prop::Value::NdTime(ndt) => { - let prop::NdTime { - year, - month, - day, - hour, - minute, - second, - nanos, - } = ndt; - let ndt = NaiveDateTime::new( - NaiveDate::from_ymd_opt(*year as i32, *month, *day).unwrap(), - NaiveTime::from_hms_nano_opt(*hour, *minute, *second, *nanos).unwrap(), - ); - Some(Prop::NDTime(ndt)) - } - prop::Value::DTime(dt) => Some(Prop::DTime( - DateTime::parse_from_rfc3339(dt).unwrap().into(), - )), - prop::Value::Array(_) => None, - _ => None, - }; - Ok(value) -} - -fn collect_proto_props( - iter: impl IntoIterator)>, -) -> Vec { - iter.into_iter() - .map(|(key, value)| PropPair::new(key, value.borrow())) - .collect() -} - -pub fn collect_props<'a>( - iter: impl IntoIterator, -) -> Result, GraphError> { - iter.into_iter() - .map(as_prop) - .filter_map(|r| r.transpose()) - .collect() -} - -fn as_proto_prop(prop: &Prop) -> proto_generated::Prop { - let value: Option = match prop { - Prop::Bool(b) => Some(prop::Value::BoolValue(*b)), - Prop::U8(u) => Some(prop::Value::U8((*u).into())), - Prop::U16(u) => Some(prop::Value::U16((*u).into())), - Prop::U32(u) => Some(prop::Value::U32(*u)), - Prop::I32(i) => Some(prop::Value::I32(*i)), - Prop::I64(i) => Some(prop::Value::I64(*i)), - Prop::U64(u) => Some(prop::Value::U64(*u)), - Prop::F32(f) => Some(prop::Value::F32(*f)), - Prop::F64(f) => Some(prop::Value::F64(*f)), - Prop::Str(s) => Some(prop::Value::Str(s.to_string())), - Prop::List(list) => { - let properties = list.iter().map(|p| as_proto_prop(&p)).collect(); - Some(prop::Value::Prop(prop::Props { properties })) - } - Prop::Map(map) => { - let map = map - .iter() - .map(|(k, v)| (k.to_string(), as_proto_prop(v))) - .collect(); - Some(prop::Value::Map(prop::Dict { map })) - } - Prop::NDTime(ndt) => { - let (year, month, day) = (ndt.date().year(), ndt.date().month(), ndt.date().day()); - let (hour, minute, second, nanos) = ( - ndt.time().hour(), - ndt.time().minute(), - ndt.time().second(), - ndt.time().nanosecond(), - ); - - let proto_ndt = prop::NdTime { - year: year as u32, - month: month, - day: day, - hour: hour, - minute: minute, - second: second, - nanos: nanos, - }; - Some(prop::Value::NdTime(proto_ndt)) - } - Prop::DTime(dt) => Some(prop::Value::DTime( - dt.to_rfc3339_opts(chrono::SecondsFormat::AutoSi, true), - )), - Prop::Decimal(bd) => Some(prop::Value::Decimal(bd.to_string())), - }; - - proto_generated::Prop { value } -} diff --git a/raphtory/src/serialise/proto/mod.rs b/raphtory/src/serialise/proto/mod.rs deleted file mode 100644 index 6d23bcc28d..0000000000 --- a/raphtory/src/serialise/proto/mod.rs +++ /dev/null @@ -1,227 +0,0 @@ -use crate::{ - core::entities::LayerIds, - db::{ - api::{ - properties::internal::{InternalMetadataOps, InternalTemporalPropertyViewOps}, - view::MaterializedGraph, - }, - graph::views::deletion_graph::PersistentGraph, - }, - errors::GraphError, - prelude::Graph, -}; - -// Load the generated protobuf code from the build directory -pub mod proto_generated { - include!(concat!(env!("OUT_DIR"), "/serialise.rs")); -} - -use itertools::Itertools; -use raphtory_api::core::{ - entities::{ - properties::{prop::Prop, tprop::TPropOps}, - LayerId, VID, - }, - storage::timeindex::TimeIndexOps, -}; -use raphtory_storage::{ - core_ops::CoreGraphOps, - graph::{ - edges::edge_storage_ops::EdgeStorageOps, graph::GraphStorage, - nodes::node_storage_ops::NodeStorageOps, - }, -}; -use std::{iter, sync::Arc}; - -pub mod ext; - -/// Trait for encoding a graph to protobuf format -pub trait ProtoEncoder { - fn encode_to_proto(&self) -> proto_generated::Graph; -} - -/// Trait for decoding a graph from protobuf format -pub trait ProtoDecoder: Sized { - fn decode_from_proto(graph: &proto_generated::Graph) -> Result; -} - -macro_rules! zip_tprop_updates { - ($iter:expr) => { - &$iter - .map(|(key, values)| values.iter().map(move |(t, v)| (t, (key, v)))) - .kmerge_by(|(left_t, _), (right_t, _)| left_t <= right_t) - .chunk_by(|(t, _)| *t) - }; -} - -impl ProtoEncoder for GraphStorage { - fn encode_to_proto(&self) -> proto_generated::Graph { - let storage = self.lock(); - let mut graph = proto_generated::Graph::default(); - - // Graph Properties - let graph_meta = storage.graph_props_meta(); - for (id, key) in graph_meta.metadata_mapper().read().iter_ids() { - graph.new_graph_cprop(key, id); - } - graph.update_graph_cprops( - storage - .metadata_ids() - .filter_map(|id| Some((id, storage.get_metadata(id)?))), - ); - - for (id, key, dtype) in graph_meta - .temporal_prop_mapper() - .locked() - .iter_ids_and_types() - { - graph.new_graph_tprop(key, id, dtype); - } - - let t_props = graph_meta - .temporal_prop_mapper() - .locked() - .iter_ids_and_types() - .map(|(id, _, _)| storage.temporal_iter(id).map(move |(t, v)| (t, (id, v)))) - .kmerge_by(|(left_t, _), (right_t, _)| left_t <= right_t) - .chunk_by(|(t, _)| *t); - - for (t, group) in t_props.into_iter() { - graph.update_graph_tprops(t, group.map(|(_, v)| v)); - } - - // Layers - for (id, layer) in storage.edge_meta().layer_meta().read().iter_ids() { - graph.new_layer(layer, id); - } - - // Node Types - for (id, node_type) in storage.node_meta().node_type_meta().read().iter_ids() { - graph.new_node_type(node_type, id); - } - - // Node Properties - let n_const_meta = self.node_meta().metadata_mapper(); - for (id, key, dtype) in n_const_meta.locked().iter_ids_and_types() { - graph.new_node_cprop(key, id, dtype); - } - let n_temporal_meta = self.node_meta().temporal_prop_mapper(); - for (id, key, dtype) in n_temporal_meta.locked().iter_ids_and_types() { - graph.new_node_tprop(key, id, dtype); - } - - // Nodes - let all_node_prop_ids: Arc<[usize]> = n_temporal_meta.ids().collect(); - let nodes = storage.nodes(); - for node_id in 0..nodes.len() { - let node = nodes.node(VID(node_id)); - graph.new_node(node.id(), node.vid(), node.node_type_id()); - - for (time, _, row) in node.temp_prop_rows(Arc::clone(&all_node_prop_ids)) { - graph.update_node_tprops(node.vid(), time, row.into_iter()); - } - - graph.update_node_cprops( - node.vid(), - n_const_meta - .ids() - .flat_map(|i| node.constant_prop_layer(LayerId(0), i).map(|v| (i, v))), - ); - } - - // Edge Properties - let e_const_meta = self.edge_meta().metadata_mapper(); - for (id, key, dtype) in e_const_meta.locked().iter_ids_and_types() { - graph.new_edge_cprop(key, id, dtype); - } - let e_temporal_meta = self.edge_meta().temporal_prop_mapper(); - for (id, key, dtype) in e_temporal_meta.locked().iter_ids_and_types() { - graph.new_edge_tprop(key, id, dtype); - } - - // Edges - let edges = storage.edges(); - for edge in edges.iter(&LayerIds::All) { - let eid = edge.eid(); - let edge = edge.as_ref(); - graph.new_edge(edge.src(), edge.dst(), eid); - for layer_id in storage.unfiltered_layer_ids() { - for (t, props) in zip_tprop_updates!(e_temporal_meta - .ids() - .map(|i| (i, edge.temporal_prop_layer(layer_id, i)))) - { - graph.update_edge_tprops(eid, t, layer_id, props.map(|(_, v)| v)); - } - for t in edge.additions(layer_id).iter() { - graph.update_edge_tprops(eid, t, layer_id, iter::empty::<(usize, Prop)>()); - } - for t in edge.deletions(layer_id).iter() { - graph.del_edge(eid, layer_id, t); - } - graph.update_edge_cprops( - eid, - layer_id, - e_const_meta - .ids() - .filter_map(|i| edge.metadata_layer(layer_id, i).map(|prop| (i, prop))), - ); - } - } - graph - } -} - -impl ProtoEncoder for Graph { - fn encode_to_proto(&self) -> proto_generated::Graph { - let mut graph = self.core_graph().encode_to_proto(); - graph.set_graph_type(proto_generated::GraphType::Event); - graph - } -} - -impl ProtoEncoder for PersistentGraph { - fn encode_to_proto(&self) -> proto_generated::Graph { - let mut graph = self.core_graph().encode_to_proto(); - graph.set_graph_type(proto_generated::GraphType::Persistent); - graph - } -} - -impl ProtoEncoder for MaterializedGraph { - fn encode_to_proto(&self) -> proto_generated::Graph { - match self { - MaterializedGraph::EventGraph(graph) => graph.encode_to_proto(), - MaterializedGraph::PersistentGraph(graph) => graph.encode_to_proto(), - } - } -} - -impl ProtoDecoder for GraphStorage { - fn decode_from_proto(_graph: &proto_generated::Graph) -> Result { - todo!("implement this") - } -} - -impl ProtoDecoder for Graph { - fn decode_from_proto(_graph: &proto_generated::Graph) -> Result { - todo!("implement this") - } -} - -impl ProtoDecoder for PersistentGraph { - fn decode_from_proto(graph: &proto_generated::Graph) -> Result { - match graph.graph_type() { - proto_generated::GraphType::Event => Err(GraphError::GraphLoadError), - proto_generated::GraphType::Persistent => { - let storage = GraphStorage::decode_from_proto(graph)?; - Ok(PersistentGraph::from_internal_graph(storage)) - } - } - } -} - -impl ProtoDecoder for MaterializedGraph { - fn decode_from_proto(_graph: &proto_generated::Graph) -> Result { - todo!("implement this") - } -}