diff --git a/Cargo.lock b/Cargo.lock index 1a5ce66f44..6b08546d51 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2651,8 +2651,9 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] name = "dynamic-graphql" -version = "0.10.1" -source = "git+https://github.com/miratepuffin/dynamic-graphql?branch=add-arg-descriptions#69a07c5fe3c16b4baf76f676c96cde5865cae1de" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d58055cecef1736f42bee8819059054e18239cda3d5de56b0c8836c2a2322b" dependencies = [ "async-graphql", "dynamic-graphql-derive", @@ -2661,8 +2662,9 @@ dependencies = [ [[package]] name = "dynamic-graphql-derive" -version = "0.10.1" -source = "git+https://github.com/miratepuffin/dynamic-graphql?branch=add-arg-descriptions#69a07c5fe3c16b4baf76f676c96cde5865cae1de" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff1e2aa3d6affef7ce88263e83c0110b11f57409cf25aa81689bcc5f023a602e" dependencies = [ "Inflector", "darling 0.20.11", @@ -6397,7 +6399,6 @@ dependencies = [ "quad-rand", "rand 0.9.4", "rand_distr 0.5.1", - "raphtory", "raphtory-api", "raphtory-core", "raphtory-itertools", @@ -6554,6 +6555,7 @@ dependencies = [ "raphtory", "raphtory-api", "raphtory-storage", + "raphtory-test-utils", "rayon", "reqwest", "rust-embed", @@ -6630,6 +6632,31 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "raphtory-test-utils" +version = "0.18.0" +dependencies = [ + "ahash", + "arrow", + "bigdecimal", + "chrono", + "db4-storage", + "itertools 0.13.0", + "parquet", + "proptest", + "proptest-derive", + "rand 0.9.4", + "raphtory", + "raphtory-api", + "raphtory-storage", + "rayon", + "serde", + "serde_json", + "tempfile", + "tracing", + "zip", +] + [[package]] name = "raphtory_custom_gql_apis" version = "0.9.3" diff --git a/Cargo.toml b/Cargo.toml index 4baf275165..0494af6608 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,11 +13,12 @@ members = [ "raphtory-core", "raphtory-storage", "raphtory-api-macros", + "raphtory-test-utils", "raphtory-itertools", "clam-core", "clam-core/snb", "raphtory-itertools" -] +, "raphtory-test-utils"] default-members = ["raphtory"] exclude = ["optd"] resolver = "2" @@ -62,6 +63,7 @@ opt-level = 1 # much smaller files db4-graph = { version = "0.18.0", path = "db4-graph", default-features = false } db4-storage = { version = "0.18.0", path = "db4-storage" } raphtory = { version = "0.18.0", path = "raphtory", default-features = false } +raphtory-test-utils = { version = "0.18.0", path = "raphtory-test-utils", default-features = false } raphtory-api = { version = "0.18.0", path = "raphtory-api", default-features = false } raphtory-api-macros = { version = "0.18.0", path = "raphtory-api-macros", default-features = false } raphtory-core = { version = "0.18.0", path = "raphtory-core", default-features = false } @@ -73,7 +75,7 @@ optd-core = { version = "0.18.0", path = "optd/optd/core" } async-graphql = { version = "7.2.1", features = ["dynamic-schema"] } bincode = { version = "2", features = ["serde"] } async-graphql-poem = "7.2.1" -dynamic-graphql = { git = "https://github.com/miratepuffin/dynamic-graphql", branch = "add-arg-descriptions" } +dynamic-graphql = "0.10.2" derive_more = "2.1.1" tikv-jemallocator = "0.6.1" reqwest = { version = "0.12.28", default-features = false, features = [ diff --git a/raphtory-api/src/core/entities/mod.rs b/raphtory-api/src/core/entities/mod.rs index ab7f914cbb..2959371455 100644 --- a/raphtory-api/src/core/entities/mod.rs +++ b/raphtory-api/src/core/entities/mod.rs @@ -705,7 +705,6 @@ impl From for LayerIds { mod tests { use crate::core::entities::{LayerId, EID, MAX_EID}; use proptest::{prop_assert, prop_assert_eq, proptest}; - use std::char::MAX; #[test] fn test_elid_layer() { diff --git a/raphtory-benchmark/Cargo.toml b/raphtory-benchmark/Cargo.toml index 0c288e3ec1..5e3ffa56c2 100644 --- a/raphtory-benchmark/Cargo.toml +++ b/raphtory-benchmark/Cargo.toml @@ -1,5 +1,6 @@ [package] name = "raphtory-benchmark" +publish = false version.workspace = true edition = "2021" diff --git a/raphtory-graphql/Cargo.toml b/raphtory-graphql/Cargo.toml index 081ea82f76..04232dc7e6 100644 --- a/raphtory-graphql/Cargo.toml +++ b/raphtory-graphql/Cargo.toml @@ -63,13 +63,11 @@ zip = { workspace = true } clap = { workspace = true } rust-embed = { workspace = true } - - [dev-dependencies] parking_lot = { workspace = true } tempfile = { workspace = true } pretty_assertions = { workspace = true } -raphtory = { workspace = true, features = ["test-utils"] } +raphtory-test-utils = { workspace = true } arrow-array = { workspace = true } [features] diff --git a/raphtory-test-utils/Cargo.toml b/raphtory-test-utils/Cargo.toml new file mode 100644 index 0000000000..56b74a58a8 --- /dev/null +++ b/raphtory-test-utils/Cargo.toml @@ -0,0 +1,39 @@ +[package] +name = "raphtory-test-utils" +publish = false +version.workspace = true +documentation.workspace = true +repository.workspace = true +license.workspace = true +readme.workspace = true +homepage.workspace = true +keywords.workspace = true +authors.workspace = true +rust-version.workspace = true +edition.workspace = true + +[dependencies] +raphtory-api.workspace = true +raphtory = {workspace = true, features = ["io"]} +ahash.workspace = true +proptest.workspace = true +proptest-derive.workspace = true +serde.workspace = true +serde_json.workspace = true +itertools.workspace = true +storage.workspace = true +raphtory-storage.workspace = true +rand.workspace = true +rayon.workspace = true +bigdecimal.workspace = true +chrono.workspace = true + +[dev-dependencies] +tempfile.workspace = true +arrow.workspace = true +parquet.workspace = true +tracing.workspace = true +zip.workspace = true + +[features] +search = ["raphtory/search"] diff --git a/raphtory/resources/test/test.csv b/raphtory-test-utils/resources/test/test.csv similarity index 100% rename from raphtory/resources/test/test.csv rename to raphtory-test-utils/resources/test/test.csv diff --git a/raphtory/src/db/graph/assertions.rs b/raphtory-test-utils/src/assertions.rs similarity index 98% rename from raphtory/src/db/graph/assertions.rs rename to raphtory-test-utils/src/assertions.rs index 30b31dc1f9..f13b503202 100644 --- a/raphtory/src/db/graph/assertions.rs +++ b/raphtory-test-utils/src/assertions.rs @@ -1,14 +1,12 @@ -use crate::{ +use raphtory::{ db::api::view::{filter_ops::Filter, StaticGraphViewOps}, prelude::{EdgeViewOps, Graph, GraphViewOps, NodeViewOps}, }; use std::ops::Range; #[cfg(feature = "search")] -pub use crate::db::api::view::SearchableGraphOps; -#[cfg(feature = "search")] -use crate::prelude::IndexMutationOps; -use crate::{ +use raphtory::prelude::{IndexMutationOps, SearchableGraphOps}; +use raphtory::{ db::{ api::view::filter_ops::{EdgeSelect, NodeSelect}, graph::views::{ @@ -378,7 +376,7 @@ pub fn assert_search_edges_results( #[track_caller] fn assert_results( init_graph: impl FnOnce(Graph) -> Graph, - pre_transform: impl Fn(&Graph) -> (), + pre_transform: impl Fn(&Graph), transform: impl GraphTransformer, expected: &[&str], variants: Vec, diff --git a/raphtory-test-utils/src/lib.rs b/raphtory-test-utils/src/lib.rs new file mode 100644 index 0000000000..843bb9f8a0 --- /dev/null +++ b/raphtory-test-utils/src/lib.rs @@ -0,0 +1,2 @@ +pub mod assertions; +pub mod test_utils; diff --git a/raphtory/src/test_utils.rs b/raphtory-test-utils/src/test_utils.rs similarity index 99% rename from raphtory/src/test_utils.rs rename to raphtory-test-utils/src/test_utils.rs index a2df97d642..bc5ea41cea 100644 --- a/raphtory/src/test_utils.rs +++ b/raphtory-test-utils/src/test_utils.rs @@ -1,10 +1,3 @@ -use crate::{ - db::{ - api::storage::storage::Storage, - graph::{edge::EdgeView, node::NodeView}, - }, - prelude::*, -}; use ahash::HashSet; use bigdecimal::BigDecimal; use chrono::{DateTime, NaiveDateTime, Utc}; @@ -12,6 +5,13 @@ use itertools::Itertools; use proptest::{arbitrary::any, prelude::*}; use proptest_derive::Arbitrary; use rand::seq::SliceRandom; +use raphtory::{ + db::{ + api::storage::storage::Storage, + graph::{edge::EdgeView, node::NodeView}, + }, + prelude::*, +}; use raphtory_api::core::{ entities::properties::prop::{PropType, DECIMAL_MAX}, storage::{ diff --git a/raphtory/tests/algo_tests/centrality.rs b/raphtory-test-utils/tests/algo_tests/centrality.rs similarity index 99% rename from raphtory/tests/algo_tests/centrality.rs rename to raphtory-test-utils/tests/algo_tests/centrality.rs index b836fd2753..882dc0dee4 100644 --- a/raphtory/tests/algo_tests/centrality.rs +++ b/raphtory-test-utils/tests/algo_tests/centrality.rs @@ -8,8 +8,8 @@ use raphtory::{ pagerank::unweighted_page_rank, }, prelude::*, - test_storage, }; +use raphtory_test_utils::test_storage; #[test] fn test_betweenness_centrality() { diff --git a/raphtory/tests/algo_tests/community_detection.rs b/raphtory-test-utils/tests/algo_tests/community_detection.rs similarity index 98% rename from raphtory/tests/algo_tests/community_detection.rs rename to raphtory-test-utils/tests/algo_tests/community_detection.rs index b07c3b70b2..3f7a444743 100644 --- a/raphtory/tests/algo_tests/community_detection.rs +++ b/raphtory-test-utils/tests/algo_tests/community_detection.rs @@ -4,11 +4,11 @@ use raphtory::{ louvain::louvain, modularity::{ComID, ModularityFunction, ModularityUnDir, Partition}, }, + core::entities::VID, logging::global_info_logger, prelude::*, - test_storage, }; -use raphtory_core::entities::VID; +use raphtory_test_utils::test_storage; use std::collections::{HashMap, HashSet}; use tracing::info; @@ -134,7 +134,6 @@ proptest! { } } -#[cfg(feature = "io")] #[test] fn lfr_test() { use raphtory::io::csv_loader::CsvLoader; diff --git a/raphtory/tests/algo_tests/components.rs b/raphtory-test-utils/tests/algo_tests/components.rs similarity index 99% rename from raphtory/tests/algo_tests/components.rs rename to raphtory-test-utils/tests/algo_tests/components.rs index d11e560e32..61000fba7a 100644 --- a/raphtory/tests/algo_tests/components.rs +++ b/raphtory-test-utils/tests/algo_tests/components.rs @@ -8,8 +8,8 @@ use raphtory::{ view::internal::GraphView, }, prelude::*, - test_storage, }; +use raphtory_test_utils::test_storage; use serde::{Deserialize, Serialize}; use std::{ collections::{BTreeSet, HashMap}, @@ -270,8 +270,8 @@ mod in_component_test { }, }, prelude::*, - test_storage, }; + use raphtory_test_utils::test_storage; use std::collections::HashMap; fn check_node(graph: &Graph, node_id: u64, mut correct: Vec<(u64, usize)>) { @@ -540,8 +540,8 @@ mod components_test { }, }, prelude::*, - test_storage, }; + use raphtory_test_utils::test_storage; use std::collections::HashMap; fn check_node(graph: &Graph, node_id: u64, mut correct: Vec<(u64, usize)>) { @@ -803,8 +803,8 @@ mod strongly_connected_components_tests { use raphtory::{ algorithms::components::strongly_connected_components, prelude::{AdditionOps, Graph, NodeStateGroupBy, NodeStateOps, NodeViewOps, NO_PROPS}, - test_storage, }; + use raphtory_test_utils::test_storage; use std::collections::HashSet; #[test] diff --git a/raphtory/tests/algo_tests/cores.rs b/raphtory-test-utils/tests/algo_tests/cores.rs similarity index 96% rename from raphtory/tests/algo_tests/cores.rs rename to raphtory-test-utils/tests/algo_tests/cores.rs index e15a11baff..8df54a10f3 100644 --- a/raphtory/tests/algo_tests/cores.rs +++ b/raphtory-test-utils/tests/algo_tests/cores.rs @@ -1,6 +1,6 @@ -#[cfg(test)] mod k_core_test { - use raphtory::{algorithms::cores::k_core::k_core_set, prelude::*, test_storage}; + use raphtory::{algorithms::cores::k_core::k_core_set, prelude::*}; + use raphtory_test_utils::test_storage; use std::collections::HashSet; #[test] diff --git a/raphtory/tests/algo_tests/embeddings.rs b/raphtory-test-utils/tests/algo_tests/embeddings.rs similarity index 99% rename from raphtory/tests/algo_tests/embeddings.rs rename to raphtory-test-utils/tests/algo_tests/embeddings.rs index d263804709..41de4db5ee 100644 --- a/raphtory/tests/algo_tests/embeddings.rs +++ b/raphtory-test-utils/tests/algo_tests/embeddings.rs @@ -1,10 +1,9 @@ -#[cfg(test)] mod fast_rp_test { use raphtory::{ algorithms::embeddings::fast_rp::fast_rp, db::api::mutation::AdditionOps, prelude::*, - test_storage, }; + use raphtory_test_utils::test_storage; use std::collections::HashMap; #[test] diff --git a/raphtory/tests/algo_tests/metrics.rs b/raphtory-test-utils/tests/algo_tests/metrics.rs similarity index 96% rename from raphtory/tests/algo_tests/metrics.rs rename to raphtory-test-utils/tests/algo_tests/metrics.rs index fce5fd54f7..086b5b51bc 100644 --- a/raphtory/tests/algo_tests/metrics.rs +++ b/raphtory-test-utils/tests/algo_tests/metrics.rs @@ -1,12 +1,10 @@ -#[cfg(test)] mod cc_test { - use pretty_assertions::assert_eq; use raphtory::{ algorithms::metrics::clustering_coefficient::global_clustering_coefficient::global_clustering_coefficient, db::{api::mutation::AdditionOps, graph::graph::Graph}, prelude::NO_PROPS, - test_storage, }; + use raphtory_test_utils::test_storage; /// Test the global clustering coefficient #[test] @@ -48,7 +46,6 @@ mod cc_test { } } -#[cfg(test)] mod clustering_coefficient_tests { use raphtory::{ algorithms::metrics::clustering_coefficient::{ @@ -60,8 +57,8 @@ mod clustering_coefficient_tests { graph::graph::Graph, }, prelude::NO_PROPS, - test_storage, }; + use raphtory_test_utils::test_storage; use std::collections::HashMap; #[test] @@ -121,15 +118,13 @@ mod clustering_coefficient_tests { } } -#[cfg(test)] mod sum_weight_test { - use pretty_assertions::assert_eq; use raphtory::{ algorithms::metrics::balance::balance, db::{api::mutation::AdditionOps, graph::graph::Graph}, - test_storage, }; use raphtory_api::core::{entities::properties::prop::Prop, Direction}; + use raphtory_test_utils::test_storage; use std::collections::HashMap; #[test] @@ -203,7 +198,6 @@ mod sum_weight_test { } } -#[cfg(test)] mod degree_test { use raphtory::{ algorithms::metrics::degree::{ @@ -212,8 +206,8 @@ mod degree_test { }, db::{api::mutation::AdditionOps, graph::graph::Graph}, prelude::NO_PROPS, - test_storage, }; + use raphtory_test_utils::test_storage; #[test] fn degree_test() { @@ -264,14 +258,13 @@ mod degree_test { } } -#[cfg(test)] mod directed_graph_density_tests { use raphtory::{ algorithms::metrics::directed_graph_density::directed_graph_density, db::{api::mutation::AdditionOps, graph::graph::Graph}, prelude::*, - test_storage, }; + use raphtory_test_utils::test_storage; #[test] fn low_graph_density() { @@ -319,16 +312,14 @@ mod directed_graph_density_tests { } } -#[cfg(test)] mod reciprocity_test { - use pretty_assertions::assert_eq; use raphtory::{ algorithms::metrics::reciprocity::{all_local_reciprocity, global_reciprocity}, prelude::*, - test_storage, }; use std::collections::HashMap; + use raphtory_test_utils::test_storage; #[test] fn test_global_recip() { let graph = Graph::new(); diff --git a/raphtory/tests/algo_tests/mod.rs b/raphtory-test-utils/tests/algo_tests/mod.rs similarity index 88% rename from raphtory/tests/algo_tests/mod.rs rename to raphtory-test-utils/tests/algo_tests/mod.rs index 8d3bb8812d..bc098b5314 100644 --- a/raphtory/tests/algo_tests/mod.rs +++ b/raphtory-test-utils/tests/algo_tests/mod.rs @@ -1,12 +1,20 @@ use std::collections::HashMap; +#[cfg(test)] mod centrality; +#[cfg(test)] mod community_detection; +#[cfg(test)] mod components; +#[cfg(test)] mod cores; +#[cfg(test)] mod embeddings; +#[cfg(test)] mod metrics; +#[cfg(test)] mod motifs; +#[cfg(test)] mod pathing; fn assert_eq_f64(a: f64, b: f64, precision: f64) { diff --git a/raphtory/tests/algo_tests/motifs.rs b/raphtory-test-utils/tests/algo_tests/motifs.rs similarity index 98% rename from raphtory/tests/algo_tests/motifs.rs rename to raphtory-test-utils/tests/algo_tests/motifs.rs index abc135a5e6..3fcd6df8c0 100644 --- a/raphtory/tests/algo_tests/motifs.rs +++ b/raphtory-test-utils/tests/algo_tests/motifs.rs @@ -1,11 +1,10 @@ -#[cfg(test)] mod global_motifs_test { use raphtory::{ algorithms::motifs::global_temporal_three_node_motifs::temporal_three_node_motif_multi, db::{api::mutation::AdditionOps, graph::graph::Graph}, prelude::*, - test_storage, }; + use raphtory_test_utils::test_storage; fn load_graph(edges: Vec<(i64, u64, u64)>) -> Graph { let graph = Graph::new(); @@ -66,15 +65,14 @@ mod global_motifs_test { } } -#[cfg(test)] mod local_motifs_test { use raphtory::{ algorithms::motifs::local_temporal_three_node_motifs::temporal_three_node_motif, db::{api::mutation::AdditionOps, graph::graph::Graph}, prelude::*, - test_storage, }; use raphtory_api::core::utils::logging::global_debug_logger; + use raphtory_test_utils::test_storage; use std::collections::HashMap; use tracing::info; @@ -376,7 +374,6 @@ mod local_motifs_test { } } -#[cfg(test)] mod triangle_count_tests { use raphtory::{ algorithms::motifs::local_triangle_count::local_triangle_count, @@ -385,8 +382,8 @@ mod triangle_count_tests { graph::graph::Graph, }, prelude::*, - test_storage, }; + use raphtory_test_utils::test_storage; #[test] fn counts_triangles() { @@ -409,7 +406,6 @@ mod triangle_count_tests { } } -#[cfg(test)] mod rich_club_test { use crate::algo_tests::assert_eq_f64; use raphtory::{ @@ -475,14 +471,13 @@ mod rich_club_test { } } -#[cfg(test)] mod triangle_count_tests_alt { use raphtory::{ algorithms::motifs::triangle_count::triangle_count, db::{api::mutation::AdditionOps, graph::graph::Graph}, prelude::NO_PROPS, - test_storage, }; + use raphtory_test_utils::test_storage; use std::time::Instant; #[ignore] @@ -575,15 +570,13 @@ mod triangle_count_tests_alt { } } -#[cfg(test)] mod triplet_test { - use pretty_assertions::assert_eq; use raphtory::{ algorithms::motifs::triplet_count::triplet_count, db::{api::mutation::AdditionOps, graph::graph::Graph}, prelude::*, - test_storage, }; + use raphtory_test_utils::test_storage; /// Test the global clustering coefficient #[test] diff --git a/raphtory/tests/algo_tests/pathing.rs b/raphtory-test-utils/tests/algo_tests/pathing.rs similarity index 99% rename from raphtory/tests/algo_tests/pathing.rs rename to raphtory-test-utils/tests/algo_tests/pathing.rs index b0dd065dfb..6cd9e8ab54 100644 --- a/raphtory/tests/algo_tests/pathing.rs +++ b/raphtory-test-utils/tests/algo_tests/pathing.rs @@ -1,14 +1,13 @@ -#[cfg(test)] mod dijkstra_tests { use itertools::Itertools; use raphtory::{ algorithms::pathing::dijkstra::dijkstra_single_source_shortest_paths, db::{api::mutation::AdditionOps, graph::graph::Graph}, prelude::*, - test_storage, }; use raphtory_api::core::Direction; + use raphtory_test_utils::test_storage; fn load_graph(edges: Vec<(i64, &str, &str, Vec<(&str, f32)>)>) -> Graph { let graph = Graph::new(); @@ -425,16 +424,15 @@ mod dijkstra_tests { } } -#[cfg(test)] mod sssp_tests { use itertools::Itertools; use raphtory::{ algorithms::pathing::single_source_shortest_path::single_source_shortest_path, db::{api::mutation::AdditionOps, graph::graph::Graph}, prelude::*, - test_storage, }; use raphtory_api::core::utils::logging::global_info_logger; + use raphtory_test_utils::test_storage; use std::collections::HashMap; fn load_graph(edges: Vec<(i64, u64, u64)>) -> Graph { @@ -499,19 +497,18 @@ mod sssp_tests { } } -#[cfg(test)] mod generic_taint_tests { use raphtory::{ algorithms::pathing::temporal_reachability::temporally_reachable_nodes, + core::entities::nodes::node_ref::AsNodeRef, db::{ api::{mutation::AdditionOps, view::StaticGraphViewOps}, graph::graph::Graph, }, prelude::*, - test_storage, }; - use raphtory_core::entities::nodes::node_ref::AsNodeRef; + use raphtory_test_utils::test_storage; use std::collections::HashMap; fn sort_inner_by_string( diff --git a/raphtory/tests/algorithms.rs b/raphtory-test-utils/tests/algorithms.rs similarity index 100% rename from raphtory/tests/algorithms.rs rename to raphtory-test-utils/tests/algorithms.rs diff --git a/raphtory/tests/cached_view.rs b/raphtory-test-utils/tests/cached_view.rs similarity index 93% rename from raphtory/tests/cached_view.rs rename to raphtory-test-utils/tests/cached_view.rs index be46bff733..9293595aa5 100644 --- a/raphtory/tests/cached_view.rs +++ b/raphtory-test-utils/tests/cached_view.rs @@ -2,9 +2,10 @@ use itertools::Itertools; use proptest::prelude::*; use raphtory::{ algorithms::motifs::triangle_count::triangle_count, db::graph::graph::assert_graph_equal, - prelude::*, test_storage, + prelude::*, }; use raphtory_api::core::storage::timeindex::AsTime; +use raphtory_test_utils::test_storage; #[test] fn empty_graph() { @@ -125,13 +126,11 @@ mod test_filters_cached_view { use raphtory::{ db::{ api::view::StaticGraphViewOps, - graph::{ - assertions::GraphTransformer, - views::{cached_view::CachedView, window_graph::WindowedGraph}, - }, + graph::views::{cached_view::CachedView, window_graph::WindowedGraph}, }, prelude::{GraphViewOps, TimeOps}, }; + use raphtory_test_utils::assertions::GraphTransformer; use std::ops::Range; struct CachedGraphTransformer; @@ -156,20 +155,18 @@ mod test_filters_cached_view { use raphtory::{ db::{ api::view::StaticGraphViewOps, - graph::{ - assertions::{ - assert_filter_nodes_results, assert_search_nodes_results, - TestGraphVariants, TestVariants, - }, - views::filter::model::{ - node_filter::NodeFilter, property_filter::ops::PropertyFilterOps, - PropertyFilterFactory, - }, + graph::views::filter::model::{ + node_filter::NodeFilter, property_filter::ops::PropertyFilterOps, + PropertyFilterFactory, }, }, prelude::AdditionOps, }; use raphtory_api::core::entities::properties::prop::Prop; + use raphtory_test_utils::assertions::{ + assert_filter_nodes_results, assert_search_nodes_results, TestGraphVariants, + TestVariants, + }; use crate::test_filters_cached_view::{ CachedGraphTransformer, WindowedCachedGraphTransformer, @@ -266,15 +263,13 @@ mod test_filters_cached_view { mod test_edges_filter_cached_view_graph { use raphtory::{ - db::{ - api::view::StaticGraphViewOps, - graph::assertions::{ - assert_filter_edges_results, assert_search_edges_results, TestVariants, - }, - }, + db::api::view::StaticGraphViewOps, prelude::{AdditionOps, EdgeFilter}, }; use raphtory_api::core::entities::properties::prop::Prop; + use raphtory_test_utils::assertions::{ + assert_filter_edges_results, assert_search_edges_results, TestVariants, + }; use crate::test_filters_cached_view::{ CachedGraphTransformer, WindowedCachedGraphTransformer, diff --git a/raphtory/tests/db_tests.rs b/raphtory-test-utils/tests/db_tests.rs similarity index 99% rename from raphtory/tests/db_tests.rs rename to raphtory-test-utils/tests/db_tests.rs index e09a1ff1ef..cd61ef8d78 100644 --- a/raphtory/tests/db_tests.rs +++ b/raphtory-test-utils/tests/db_tests.rs @@ -26,11 +26,6 @@ use raphtory::{ errors::GraphError, graphgen::random_attachment::random_attachment, prelude::*, - test_storage, - test_utils::{ - build_graph, build_graph_strat, EdgeFixture, EdgeUpdatesFixture, GraphFixture, NodeFixture, - PropUpdatesFixture, - }, }; use raphtory_api::core::{ entities::{LayerId, GID, VID}, @@ -44,6 +39,13 @@ use raphtory_api::core::{ }, }; use raphtory_storage::{core_ops::CoreGraphOps, mutation::addition_ops::InternalAdditionOps}; +use raphtory_test_utils::{ + test_storage, + test_utils::{ + build_graph, build_graph_strat, EdgeFixture, EdgeUpdatesFixture, GraphFixture, NodeFixture, + PropUpdatesFixture, + }, +}; use rayon::{join, prelude::*}; use std::{ collections::{HashMap, HashSet}, diff --git a/raphtory/tests/deletion_graph.rs b/raphtory-test-utils/tests/deletion_graph.rs similarity index 100% rename from raphtory/tests/deletion_graph.rs rename to raphtory-test-utils/tests/deletion_graph.rs diff --git a/raphtory/tests/df_loaders.rs b/raphtory-test-utils/tests/df_loaders.rs similarity index 99% rename from raphtory/tests/df_loaders.rs rename to raphtory-test-utils/tests/df_loaders.rs index 970399c7c7..cfba2daa4b 100644 --- a/raphtory/tests/df_loaders.rs +++ b/raphtory-test-utils/tests/df_loaders.rs @@ -1,4 +1,4 @@ -#[cfg(feature = "io")] +#[cfg(test)] mod io_tests { use arrow::array::builder::{ ArrayBuilder, Int64Builder, LargeStringBuilder, StringViewBuilder, UInt64Builder, @@ -13,17 +13,19 @@ mod io_tests { nodes::{load_node_props_from_df, load_nodes_from_df}, }, }, + core::storage::timeindex::EventTime, db::graph::graph::assert_graph_equal, errors::GraphError, prelude::*, - test_utils::{build_edge_list, build_edge_list_str, build_edge_list_with_secondary_index}, }; use raphtory_api::core::{entities::LayerIds, storage::arc_str::ArcStr}; - use raphtory_core::storage::timeindex::EventTime; use raphtory_storage::{ core_ops::CoreGraphOps, mutation::addition_ops::{InternalAdditionOps, SessionAdditionOps}, }; + use raphtory_test_utils::test_utils::{ + build_edge_list, build_edge_list_str, build_edge_list_with_secondary_index, + }; fn build_df( chunk_size: usize, @@ -777,7 +779,6 @@ mod io_tests { } #[cfg(test)] -#[cfg(feature = "io")] mod parquet_tests { use bigdecimal::BigDecimal; use chrono::{DateTime, Utc}; @@ -788,11 +789,11 @@ mod parquet_tests { views::deletion_graph::PersistentGraph, }, prelude::*, - test_utils::{ - assert_valid_graph, build_edge_list_dyn, build_graph, build_graph_strat, - build_nodes_dyn, build_props_dyn, EdgeFixture, EdgeUpdatesFixture, GraphFixture, - NodeFixture, NodeUpdatesFixture, PropUpdatesFixture, - }, + }; + use raphtory_test_utils::test_utils::{ + assert_valid_graph, build_edge_list_dyn, build_graph, build_graph_strat, build_nodes_dyn, + build_props_dyn, EdgeFixture, EdgeUpdatesFixture, GraphFixture, NodeFixture, + NodeUpdatesFixture, PropUpdatesFixture, }; use serde_json::json; use std::{io::Cursor, str::FromStr}; diff --git a/raphtory/tests/edge_property_filter.rs b/raphtory-test-utils/tests/edge_property_filter.rs similarity index 99% rename from raphtory/tests/edge_property_filter.rs rename to raphtory-test-utils/tests/edge_property_filter.rs index 06bb051190..dfaf958e86 100644 --- a/raphtory/tests/edge_property_filter.rs +++ b/raphtory-test-utils/tests/edge_property_filter.rs @@ -4,7 +4,6 @@ use raphtory::{ db::{ api::view::Filter, graph::{ - assertions::{assert_ok_or_missing_edges, EdgeRow}, graph::{assert_graph_equal, assert_persistent_materialize_graph_equal}, views::{ deletion_graph::PersistentGraph, @@ -16,10 +15,13 @@ use raphtory::{ }, }, prelude::*, - test_utils::{build_edge_deletions, build_edge_list, build_graph_from_edge_list, build_window}, }; use raphtory_api::core::{entities::properties::prop::PropType, storage::timeindex::AsTime}; use raphtory_storage::mutation::addition_ops::{InternalAdditionOps, SessionAdditionOps}; +use raphtory_test_utils::{ + assertions::{assert_ok_or_missing_edges, EdgeRow}, + test_utils::{build_edge_deletions, build_edge_list, build_graph_from_edge_list, build_window}, +}; #[test] fn test_edge_filter() { diff --git a/raphtory/tests/exploded_edge_property_filter.rs b/raphtory-test-utils/tests/exploded_edge_property_filter.rs similarity index 99% rename from raphtory/tests/exploded_edge_property_filter.rs rename to raphtory-test-utils/tests/exploded_edge_property_filter.rs index 8f993e8f24..03bccb2a30 100644 --- a/raphtory/tests/exploded_edge_property_filter.rs +++ b/raphtory-test-utils/tests/exploded_edge_property_filter.rs @@ -1,10 +1,10 @@ use itertools::Itertools; use proptest::{arbitrary::any, proptest}; use raphtory::{ + core::entities::nodes::node_ref::AsNodeRef, db::{ api::view::{Filter, StaticGraphViewOps}, graph::{ - assertions::assert_ok_or_missing_edges, edge::EdgeView, graph::{ assert_graph_equal, assert_node_equal, assert_nodes_equal, @@ -20,20 +20,22 @@ use raphtory::{ }, }, prelude::*, - test_utils::{ - build_edge_deletions, build_edge_list, build_edge_list_with_deletions, - build_graph_from_edge_list, build_window, Update, - }, }; use raphtory_api::core::{ entities::properties::prop::PropType, storage::{arc_str::ArcStr, timeindex::AsTime}, }; -use raphtory_core::entities::nodes::node_ref::AsNodeRef; use raphtory_storage::{ core_ops::CoreGraphOps, mutation::addition_ops::{InternalAdditionOps, SessionAdditionOps}, }; +use raphtory_test_utils::{ + assertions::assert_ok_or_missing_edges, + test_utils::{ + build_edge_deletions, build_edge_list, build_edge_list_with_deletions, + build_graph_from_edge_list, build_window, Update, + }, +}; use std::collections::HashMap; fn build_filtered_graph( @@ -420,7 +422,7 @@ fn test_filter_window() { let expected_filtered_g = build_filtered_graph(&edges, |vv| vv == v); assert_graph_equal(&filtered.window(start, end), &expected_filtered_g.window(start, end)); }); - }) + }); } #[test] diff --git a/raphtory-test-utils/tests/graph_index.rs b/raphtory-test-utils/tests/graph_index.rs new file mode 100644 index 0000000000..915cc688d4 --- /dev/null +++ b/raphtory-test-utils/tests/graph_index.rs @@ -0,0 +1,981 @@ +#[cfg(all(test, feature = "search"))] +mod graph_index_test { + use raphtory::prelude::{AdditionOps, Graph, GraphViewOps, IndexMutationOps}; + + use raphtory::db::graph::views::filter::model::{ + edge_filter::EdgeFilter, node_filter::NodeFilter, property_filter::ops::PropertyFilterOps, + PropertyFilterFactory, + }; + use raphtory_test_utils::assertions::{search_edges, search_nodes}; + + fn init_nodes_graph(graph: Graph) -> Graph { + graph + .add_node(1, 1, [("p1", 1), ("p2", 2)], Some("fire_nation"), None) + .unwrap(); + graph + .add_node(2, 1, [("p6", 6)], Some("fire_nation"), None) + .unwrap(); + graph + .add_node(2, 2, [("p4", 5)], Some("fire_nation"), None) + .unwrap(); + graph + .add_node(3, 3, [("p2", 4), ("p3", 3)], Some("water_tribe"), None) + .unwrap(); + graph + } + + fn init_edges_graph(graph: Graph) -> Graph { + graph + .add_edge(1, 1, 2, [("p1", 1), ("p2", 2)], None) + .unwrap(); + graph.add_edge(2, 1, 2, [("p6", 6)], None).unwrap(); + graph.add_edge(2, 2, 3, [("p4", 5)], None).unwrap(); + graph + .add_edge(3, 3, 4, [("p2", 4), ("p3", 3)], None) + .unwrap(); + graph + } + + #[test] + fn test_if_bulk_load_create_graph_index_is_ok() { + let graph = Graph::new(); + let graph = init_nodes_graph(graph); + + assert_eq!(graph.count_nodes(), 3); + + graph.create_index_in_ram().unwrap(); + } + + #[test] + fn test_if_adding_nodes_to_existing_graph_index_is_ok() { + let graph = Graph::new(); + graph.create_index_in_ram().unwrap(); + + let graph = init_nodes_graph(graph); + + assert_eq!(graph.count_nodes(), 3); + } + + #[test] + fn test_if_adding_edges_to_existing_graph_index_is_ok() { + let graph = Graph::new(); + // Creates graph index + graph.create_index_in_ram().unwrap(); + + let graph = init_edges_graph(graph); + + assert_eq!(graph.count_edges(), 3); + } + + #[test] + fn test_node_metadata_graph_index_is_ok() { + let graph = Graph::new(); + let graph = init_nodes_graph(graph); + graph.create_index_in_ram().unwrap(); + graph.node(1).unwrap().add_metadata([("x", 1u64)]).unwrap(); + + let filter = NodeFilter.metadata("x").eq(1u64); + assert_eq!(search_nodes(&graph, filter.clone()), vec!["1"]); + + graph + .node(1) + .unwrap() + .update_metadata([("x", 2u64)]) + .unwrap(); + let filter = NodeFilter.metadata("x").eq(1u64); + assert_eq!(search_nodes(&graph, filter.clone()), Vec::<&str>::new()); + + graph + .node(1) + .unwrap() + .update_metadata([("x", 2u64)]) + .unwrap(); + let filter = NodeFilter.metadata("x").eq(2u64); + assert_eq!(search_nodes(&graph, filter.clone()), vec!["1"]); + } + + #[test] + fn test_edge_metadata_graph_index_is_ok() { + let graph = Graph::new(); + let graph = init_edges_graph(graph); + graph.create_index_in_ram().unwrap(); + graph + .edge(1, 2) + .unwrap() + .add_metadata([("x", 1u64)], None) + .unwrap(); + + let filter = EdgeFilter.metadata("x").eq(1u64); + assert_eq!(search_edges(&graph, filter.clone()), vec!["1->2"]); + + graph + .edge(1, 2) + .unwrap() + .update_metadata([("x", 2u64)], None) + .unwrap(); + let filter = EdgeFilter.metadata("x").eq(1u64); + assert_eq!(search_edges(&graph, filter.clone()), Vec::<&str>::new()); + + graph + .edge(1, 2) + .unwrap() + .update_metadata([("x", 2u64)], None) + .unwrap(); + let filter = EdgeFilter.metadata("x").eq(2u64); + assert_eq!(search_edges(&graph, filter.clone()), vec!["1->2"]); + } +} + +#[cfg(all(test, feature = "search"))] +mod test_index { + mod test_index_io { + use raphtory::{ + db::{ + api::view::{internal::InternalStorageOps, ResolvedIndexSpec}, + graph::views::filter::model::{ + node_filter::{ops::NodeFilterOps, NodeFilter}, + TryAsCompositeFilter, + }, + }, + errors::GraphError, + prelude::*, + serialise::GraphFolder, + }; + use raphtory_api::core::{ + entities::properties::prop::Prop, storage::arc_str::ArcStr, + utils::logging::global_info_logger, + }; + use tempfile::TempDir; + + fn temp_storage_path() -> std::path::PathBuf { + tempfile::tempdir().unwrap().path().to_path_buf() + } + + fn init_graph() -> Graph { + let graph = Graph::new(); + + graph + .add_node( + 1, + "Alice", + vec![("p1", Prop::U64(1000u64))], + Some("fire_nation"), + None, + ) + .unwrap(); + graph + } + + fn assert_search_results( + graph: &Graph, + filter: &T, + expected: Vec<&str>, + ) { + let res = graph + .search_nodes(filter.clone(), 2, 0) + .unwrap() + .into_iter() + .map(|n| n.name()) + .collect::>(); + assert_eq!(res, expected); + } + + #[test] + fn test_create_no_index_persist_no_index_on_encode_load_no_index_on_decode() { + // No index persisted since it was never created + let graph = init_graph(); + + let filter = NodeFilter::name().eq("Alice"); + assert_search_results(&graph, &filter, vec!["Alice"]); + + let binding = TempDir::new().unwrap(); + let path = binding.path(); + graph.encode(path).unwrap(); + + let graph = Graph::decode(path).unwrap(); + let is_indexed = graph.get_storage().unwrap().is_indexed(); + assert!(!is_indexed); + } + + #[test] + fn test_create_index_persist_index_on_encode_load_index_on_decode() { + let graph = init_graph(); + + // Created index + graph.create_index().unwrap(); + + let filter = NodeFilter::name().eq("Alice"); + assert_search_results(&graph, &filter, vec!["Alice"]); + + // Persisted both graph and index + let binding = TempDir::new().unwrap(); + let path = binding.path(); + graph.encode(path).unwrap(); + + // Loaded index that was persisted + let graph = Graph::decode(path).unwrap(); + let is_indexed = graph.get_storage().unwrap().is_indexed(); + assert!(is_indexed); + + assert_search_results(&graph, &filter, vec!["Alice"]); + } + + #[test] + fn test_encoding_graph_twice_to_same_storage_path_fails() { + let graph = init_graph(); + graph.create_index().unwrap(); + let binding = TempDir::new().unwrap(); + let path = binding.path(); + graph.encode(path).unwrap(); + let result = graph.encode(path); + + match result { + Err(GraphError::NonEmptyGraphFolder(err_path)) => { + assert_eq!(path, err_path); + } + Ok(_) => panic!("Expected error on second encode, got Ok"), + Err(e) => panic!("Unexpected error type: {:?}", e), + } + } + + #[test] + fn test_create_index_persist_index_on_encode_update_index_load_persisted_index_on_decode() { + let graph = init_graph(); + + // Created index + graph.create_index().unwrap(); + + let filter1 = NodeFilter::name().eq("Alice"); + assert_search_results(&graph, &filter1, vec!["Alice"]); + + // Persisted both graph and index + let binding = TempDir::new().unwrap(); + let path = binding.path(); + graph.encode(path).unwrap(); + + // Updated both graph and index + graph + .add_node( + 2, + "Tommy", + vec![("p1", Prop::U64(5u64))], + Some("water_tribe"), + None, + ) + .unwrap(); + let filter2 = NodeFilter::name().eq("Tommy"); + assert_search_results(&graph, &filter2, vec!["Tommy"]); + + // Loaded index that was persisted + let graph = Graph::decode(path).unwrap(); + let is_indexed = graph.get_storage().unwrap().is_indexed(); + assert!(is_indexed); + assert_search_results(&graph, &filter1, vec!["Alice"]); + assert_search_results(&graph, &filter2, Vec::<&str>::new()); + + // Updating and encode the graph and index should decode the updated the graph as well as index + // So far we have the index that was created and persisted for the first time + graph + .add_node( + 2, + "Tommy", + vec![("p1", Prop::U64(5u64))], + Some("water_tribe"), + None, + ) + .unwrap(); + let filter2 = NodeFilter::name().eq("Tommy"); + assert_search_results(&graph, &filter2, vec!["Tommy"]); + + // Should persist the updated graph and index + let binding = TempDir::new().unwrap(); + let path = binding.path(); + graph.encode(path).unwrap(); + + // Should load the updated graph and index + let storage_path = path.parent().unwrap().to_path_buf(); + let graph = Graph::decode(path).unwrap(); + let is_indexed = graph.get_storage().unwrap().is_indexed(); + assert!(is_indexed); + assert_search_results(&graph, &filter1, vec!["Alice"]); + assert_search_results(&graph, &filter2, vec!["Tommy"]); + } + + #[test] + fn test_zip_encode_decode_index() { + let graph = init_graph(); + graph.create_index().unwrap(); + let tmp_dir = TempDir::new().unwrap(); + let zip_path = tmp_dir.path().join("graph.zip"); + let folder = GraphFolder::new_as_zip(zip_path); + graph.encode(&folder).unwrap(); + + let storage_path = tmp_dir.path().to_path_buf(); + let graph = Graph::decode(&folder).unwrap(); + let node = graph.node("Alice").unwrap(); + let node_type = node.node_type(); + assert_eq!(node_type, Some(ArcStr::from("fire_nation"))); + + let filter = NodeFilter::name().eq("Alice"); + assert_search_results(&graph, &filter, vec!["Alice"]); + } + + #[test] + fn test_encoding_graph_twice_to_same_storage_path_fails_zip() { + let graph = init_graph(); + graph.create_index().unwrap(); + let tmp_dir = TempDir::new().unwrap(); + let zip_path = tmp_dir.path().join("graph.zip"); + let folder = GraphFolder::new_as_zip(&zip_path); + graph.encode(&folder).unwrap(); + graph + .add_node(1, "Ozai", [("prop", 1)], Some("fire_nation"), None) + .unwrap(); + let result = graph.encode(folder); + match result { + Err(GraphError::IOError { source, .. }) => { + assert!( + format!("{source}").to_lowercase().contains("file exists"), + "{}", + source + ); + } + Ok(_) => panic!("Expected error on second encode, got Ok"), + Err(e) => panic!("Unexpected error type: {:?}", e), + } + } + + #[test] + fn test_immutable_graph_index_persistence() { + let graph = init_graph(); + graph.create_index().unwrap(); + + let binding = TempDir::new().unwrap(); + let path = binding.path(); + graph.encode(path).unwrap(); + + // This gives us immutable index + let graph = Graph::decode(path).unwrap(); + + // This tests that we are able to persist the immutable index + let binding = TempDir::new().unwrap(); + let path = binding.path(); + graph.encode(path).unwrap(); + + let graph = Graph::decode(path).unwrap(); + let filter1 = NodeFilter::name().eq("Alice"); + assert_search_results(&graph, &filter1, vec!["Alice"]); + } + + #[test] + fn test_mutable_graph_index_persistence() { + let graph = init_graph(); + graph.create_index().unwrap(); + + let binding = TempDir::new().unwrap(); + let path = binding.path(); + graph.encode(path).unwrap(); + + // This gives us immutable index + let graph = Graph::decode(path).unwrap(); + + // This converts immutable index to mutable index + graph + .add_node(1, "Ozai", [("prop", 1)], Some("fire_nation"), None) + .unwrap(); + + // This tests that we are able to persist the mutable index + let binding = TempDir::new().unwrap(); + let path = binding.path(); + graph.encode(path).unwrap(); + + let graph = Graph::decode(path).unwrap(); + let filter = NodeFilter::name().eq("Ozai"); + assert_search_results(&graph, &filter, vec!["Ozai"]); + } + + #[test] + fn test_loading_zip_index_creates_mutable_index() { + let graph = init_graph(); + graph.create_index().unwrap(); + let tmp_dir = TempDir::new().unwrap(); + let zip_path = tmp_dir.path().join("graph.zip"); + let folder = GraphFolder::new_as_zip(&zip_path); + graph.encode(&folder).unwrap(); + + let graph = Graph::decode(&folder).unwrap(); + let immutable = graph + .get_storage() + .unwrap() + .index() + .read_recursive() + .is_immutable(); + assert! {!immutable}; + } + + #[test] + fn test_loading_index_creates_immutable_index() { + let graph = init_graph(); + graph.create_index().unwrap(); + let binding = TempDir::new().unwrap(); + let path = binding.path(); + graph.encode(path).unwrap(); + + let graph = Graph::decode(path).unwrap(); + let immutable = graph + .get_storage() + .unwrap() + .index() + .read_recursive() + .is_immutable(); + assert! {immutable}; + } + + #[test] + fn test_create_index_in_ram() { + global_info_logger(); + + let graph = init_graph(); + graph.create_index_in_ram().unwrap(); + + let filter = NodeFilter::name().eq("Alice"); + assert_search_results(&graph, &filter, vec!["Alice"]); + + let binding = TempDir::new().unwrap(); + let path = binding.path(); + graph.encode(path).unwrap(); + + let graph = Graph::decode(path).unwrap(); + let is_indexed = graph.get_storage().unwrap().is_indexed(); + assert!(!is_indexed); + + assert_search_results(&graph, &filter, vec!["Alice"]); + } + + #[test] + #[ignore] + fn test_too_many_open_files_graph_index() { + use TempDir; + + let mut graphs = vec![]; + + for i in 0..1000 { + let graph = init_graph(); + if let Err(e) = graph.create_index() { + match &e { + GraphError::IndexError { source } => { + panic!("Hit file descriptor limit after {} graphs. {:?}", 0, source); + } + other => { + panic!("Unexpected GraphError: {:?}", other); + } + } + } + graphs.push(graph); + } + } + + #[test] + fn test_graph_index_creation_with_too_many_properties() { + let graph = init_graph(); + let props: Vec<(String, Prop)> = (1..=100) + .map(|i| (format!("p{i}"), Prop::U64(i as u64))) + .collect(); + graph.node("Alice").unwrap().add_metadata(props).unwrap(); + + if let Err(e) = graph.create_index() { + match &e { + GraphError::IndexError { source } => { + panic!("Hit file descriptor limit after {} graphs. {:?}", 0, source); + } + other => { + panic!("Unexpected GraphError: {:?}", other); + } + } + } + } + + #[test] + // No new const prop index created because when index were created + // these properties did not exist. + fn test_graph_index_creation_for_incremental_node_update_no_new_prop_indexed() { + let graph = init_graph(); + graph.create_index().unwrap(); + let props: Vec<(String, Prop)> = (1..=100) + .map(|i| (format!("p{i}"), Prop::U64(i as u64))) + .collect(); + graph.node("Alice").unwrap().add_metadata(props).unwrap(); + + let tmp_dir = TempDir::new().unwrap(); + let path = tmp_dir.path().to_path_buf(); + graph.encode(&path).unwrap(); + let graph = Graph::decode(&path).unwrap(); + + let spec = graph.get_index_spec().unwrap().props(&graph); + assert_eq!( + spec, + ResolvedIndexSpec { + node_properties: vec!["p1".to_string()], + node_metadata: vec![], + edge_metadata: vec![], + edge_properties: vec![] + } + ); + } + } + + mod test_index_spec { + use raphtory::prelude::SearchableGraphOps; + use raphtory::{ + db::{ + api::view::{IndexSpec, IndexSpecBuilder}, + graph::views::filter::model::{ + edge_filter::EdgeFilter, node_filter::NodeFilter, + property_filter::ops::PropertyFilterOps, ComposableFilter, + PropertyFilterFactory, TemporalPropertyFilterFactory, + }, + }, + errors::GraphError, + prelude::{AdditionOps, Graph, IndexMutationOps, StableDecode}, + serialise::{GraphFolder, StableEncode}, + }; + use raphtory_test_utils::assertions::{search_edges, search_nodes}; + use tempfile::{tempdir, TempDir}; + + fn temp_storage_path() -> std::path::PathBuf { + tempfile::tempdir().unwrap().path().to_path_buf() + } + + fn init_graph() -> Graph { + let graph = Graph::new(); + + let nodes = vec![ + ( + 1, + "pometry", + [("p1", 5u64), ("p2", 50u64)], + Some("fire_nation"), + [("x", true)], + ), + ( + 1, + "raphtory", + [("p1", 10u64), ("p2", 100u64)], + Some("water_tribe"), + [("y", false)], + ), + ]; + + for (time, name, props, group, metadata) in nodes { + let node = graph.add_node(time, name, props, group, None).unwrap(); + node.add_metadata(metadata).unwrap(); + } + + let edges = vec![ + ( + 1, + "pometry", + "raphtory", + [("e_p1", 3.2f64), ("e_p2", 10f64)], + None, + [("e_x", true)], + ), + ( + 1, + "raphtory", + "pometry", + [("e_p1", 4.0f64), ("e_p2", 20f64)], + None, + [("e_y", false)], + ), + ]; + + for (time, src, dst, props, label, metadata) in edges { + let edge = graph.add_edge(time, src, dst, props, label).unwrap(); + edge.add_metadata(metadata, label).unwrap(); + } + + graph + } + + #[test] + #[ignore = "TODO: #2372"] + fn test_with_all_props_index_spec() { + let graph = init_graph(); + let index_spec = IndexSpecBuilder::new(graph.clone()) + .with_all_node_properties_and_metadata() + .with_all_edge_properties_and_metadata() + .build(); + assert_eq!( + index_spec.props(&graph).to_vec(), + vec![ + vec!["x", "y"], + vec!["p1", "p2"], + vec!["e_x", "e_y"], + vec!["e_p1", "e_p2"] + ] + ); + graph.create_index_in_ram_with_spec(index_spec).unwrap(); + + let filter = NodeFilter + .property("p1") + .eq(5u64) + .and(NodeFilter.metadata("x").eq(true)); + let results = search_nodes(&graph, filter); + assert_eq!(results, vec!["pometry"]); + + let filter = EdgeFilter + .property("e_p1") + .lt(5f64) + .and(EdgeFilter.metadata("e_y").eq(false)); + let results = search_edges(&graph, filter); + assert_eq!(results, vec!["raphtory->pometry"]); + } + + #[test] + #[ignore = "TODO: #2372"] + fn test_with_selected_props_index_spec() { + let graph = init_graph(); + let index_spec = IndexSpecBuilder::new(graph.clone()) + .with_node_metadata(vec!["y"]) + .unwrap() + .with_node_properties(vec!["p1"]) + .unwrap() + .with_edge_metadata(vec!["e_y"]) + .unwrap() + .with_edge_properties(vec!["e_p1"]) + .unwrap() + .build(); + assert_eq!( + index_spec.props(&graph).to_vec(), + vec![vec!["y"], vec!["p1"], vec!["e_y"], vec!["e_p1"]] + ); + graph.create_index_in_ram_with_spec(index_spec).unwrap(); + + let filter = NodeFilter + .property("p1") + .eq(5u64) + .or(NodeFilter.metadata("y").eq(false)); + let results = search_nodes(&graph, filter); + assert_eq!(results, vec!["pometry", "raphtory"]); + + let filter = NodeFilter.metadata("y").eq(false); + let results = search_nodes(&graph, filter); + assert_eq!(results, vec!["raphtory"]); + + let filter = EdgeFilter + .property("e_p1") + .lt(5f64) + .or(EdgeFilter.metadata("e_y").eq(false)); + let results = search_edges(&graph, filter); + assert_eq!(results, vec!["pometry->raphtory", "raphtory->pometry"]); + } + + #[test] + fn test_with_invalid_property_returns_error() { + let graph = init_graph(); + let result = IndexSpecBuilder::new(graph.clone()).with_node_metadata(["xyz"]); + + assert!(matches!(result, Err(GraphError::PropertyMissingError(p)) if p == "xyz")); + } + + #[test] + fn test_build_empty_spec_by_default() { + let graph = init_graph(); + let index_spec = IndexSpecBuilder::new(graph.clone()).build(); + + assert!(index_spec.node_metadata().is_empty()); + assert!(index_spec.node_properties().is_empty()); + assert!(index_spec.edge_metadata().is_empty()); + assert!(index_spec.edge_properties().is_empty()); + + graph.create_index_in_ram_with_spec(index_spec).unwrap(); + + let filter = NodeFilter + .property("p1") + .eq(5u64) + .and(NodeFilter.metadata("x").eq(true)); + let results = search_nodes(&graph, filter); + assert_eq!(results, vec!["pometry"]); + + let filter = EdgeFilter + .property("e_p1") + .lt(5f64) + .or(EdgeFilter.metadata("e_y").eq(false)); + let results = search_edges(&graph, filter); + assert_eq!(results, vec!["pometry->raphtory", "raphtory->pometry"]); + } + + #[test] + #[ignore = "TODO: #2372"] + fn test_mixed_node_and_edge_props_index_spec() { + let graph = init_graph(); + + let index_spec = IndexSpecBuilder::new(graph.clone()) + .with_node_metadata(vec!["x"]) + .unwrap() + .with_all_node_properties() + .with_all_edge_properties_and_metadata() + .build(); + assert_eq!( + index_spec.props(&graph).to_vec(), + vec![ + vec!["x"], + vec!["p1", "p2"], + vec!["e_x", "e_y"], + vec!["e_p1", "e_p2"] + ] + ); + + graph.create_index_in_ram_with_spec(index_spec).unwrap(); + + let filter = NodeFilter + .property("p1") + .eq(5u64) + .or(NodeFilter.metadata("y").eq(false)); + let results = search_nodes(&graph, filter); + assert_eq!(results, vec!["pometry", "raphtory"]); + + let filter = EdgeFilter + .property("e_p1") + .lt(5f64) + .or(EdgeFilter.metadata("e_y").eq(false)); + let results = search_edges(&graph, filter); + assert_eq!(results, vec!["pometry->raphtory", "raphtory->pometry"]); + } + + #[test] + fn test_get_index_spec_newly_created_index() { + let graph = init_graph(); + + let index_spec = IndexSpecBuilder::new(graph.clone()) + .with_node_metadata(vec!["x"]) + .unwrap() + .with_all_node_properties() + .with_all_edge_properties_and_metadata() + .build(); + + graph + .create_index_in_ram_with_spec(index_spec.clone()) + .unwrap(); + + assert_eq!(index_spec, graph.get_index_spec().unwrap()); + } + + #[test] + #[ignore = "TODO: #2372"] + fn test_get_index_spec_updated_index() { + let graph = init_graph(); + + let index_spec = IndexSpecBuilder::new(graph.clone()) + .with_edge_metadata(vec!["e_y"]) + .unwrap() + .build(); + graph.create_index_with_spec(index_spec.clone()).unwrap(); + + assert_eq!(index_spec, graph.get_index_spec().unwrap()); + let results = search_nodes(&graph, NodeFilter.metadata("y").eq(false)); + assert_eq!(results, vec!["raphtory"]); + let results = search_edges(&graph, EdgeFilter.metadata("e_y").eq(false)); + assert_eq!(results, vec!["raphtory->pometry"]); + + let index_spec = IndexSpecBuilder::new(graph.clone()) + .with_node_metadata(vec!["y"]) + .unwrap() + .with_node_properties(vec!["p2"]) + .unwrap() + .with_edge_metadata(vec!["e_y"]) + .unwrap() + .build(); + graph.create_index_with_spec(index_spec.clone()).unwrap(); + + assert_eq!(index_spec, graph.get_index_spec().unwrap()); + let results = search_nodes(&graph, NodeFilter.metadata("y").eq(false)); + assert_eq!(results, vec!["raphtory"]); + let results = search_edges(&graph, EdgeFilter.metadata("e_y").eq(false)); + assert_eq!(results, vec!["raphtory->pometry"]); + } + + #[test] + #[ignore = "TODO: #2372"] + fn test_get_index_spec_updated_index_persisted_and_loaded() { + let graph = init_graph(); + + let index_spec = IndexSpecBuilder::new(graph.clone()) + .with_edge_metadata(vec!["e_y"]) + .unwrap() + .build(); + graph.create_index_with_spec(index_spec.clone()).unwrap(); + + let tmp_graph_dir = tempdir().unwrap(); + let path = tmp_graph_dir.path().to_path_buf(); + graph.encode(&path).unwrap(); + let graph = Graph::decode(&path).unwrap(); + + assert_eq!(index_spec, graph.get_index_spec().unwrap()); + let results = search_nodes(&graph, NodeFilter.metadata("y").eq(false)); + assert_eq!(results, vec!["raphtory"]); + let results = search_edges(&graph, EdgeFilter.metadata("e_y").eq(false)); + assert_eq!(results, vec!["raphtory->pometry"]); + + let index_spec = IndexSpecBuilder::new(graph.clone()) + .with_node_metadata(vec!["y"]) + .unwrap() + .with_node_properties(vec!["p2"]) + .unwrap() + .with_edge_metadata(vec!["e_y"]) + .unwrap() + .build(); + graph.create_index_with_spec(index_spec.clone()).unwrap(); + let tmp_graph_dir = tempdir().unwrap(); + let path = tmp_graph_dir.path().to_path_buf(); + graph.encode(path.clone()).unwrap(); + let graph = Graph::decode(&path).unwrap(); + + assert_eq!(index_spec, graph.get_index_spec().unwrap()); + let results = search_nodes(&graph, NodeFilter.metadata("y").eq(false)); + assert_eq!(results, vec!["raphtory"]); + let results = search_edges(&graph, EdgeFilter.metadata("e_y").eq(false)); + assert_eq!(results, vec!["raphtory->pometry"]); + } + + #[test] + fn test_get_index_spec_loaded_index() { + let graph = init_graph(); + + let index_spec = IndexSpecBuilder::new(graph.clone()) + .with_node_metadata(vec!["y"]) + .unwrap() + .with_node_properties(vec!["p2"]) + .unwrap() + .with_edge_metadata(vec!["e_y"]) + .unwrap() + .with_edge_properties(vec!["e_p2"]) + .unwrap() + .build(); + + graph.create_index_with_spec(index_spec.clone()).unwrap(); + let tmp_graph_dir = tempdir().unwrap(); + let path = tmp_graph_dir.path().to_path_buf(); + graph.encode(path.clone()).unwrap(); + + let graph = Graph::decode(&path).unwrap(); + let index_spec2 = graph.get_index_spec().unwrap(); + + assert_eq!(index_spec, index_spec2); + } + + #[test] + fn test_get_index_spec_loaded_index_zip() { + let graph = init_graph(); + + let index_spec = IndexSpecBuilder::new(graph.clone()) + .with_node_metadata(vec!["y"]) + .unwrap() + .with_node_properties(vec!["p2"]) + .unwrap() + .with_edge_metadata(vec!["e_y"]) + .unwrap() + .build(); + graph.create_index_with_spec(index_spec.clone()).unwrap(); + + let binding = TempDir::new().unwrap(); + let path = binding.path(); + let folder = GraphFolder::new_as_zip(path); + graph.encode(folder).unwrap(); + + let graph = Graph::decode(path).unwrap(); + assert_eq!(index_spec, graph.get_index_spec().unwrap()); + } + + #[test] + fn test_no_new_node_prop_index_created_via_update_apis() { + run_node_index_test(|graph, index_spec| { + graph.create_index_with_spec(index_spec.clone()) + }); + + run_node_index_test(|graph, index_spec| { + graph.create_index_in_ram_with_spec(index_spec.clone()) + }); + } + + #[test] + fn test_no_new_edge_prop_index_created_via_update_apis() { + run_edge_index_test(|graph, index_spec| { + graph.create_index_with_spec(index_spec.clone()) + }); + + run_edge_index_test(|graph, index_spec| { + graph.create_index_in_ram_with_spec(index_spec.clone()) + }); + } + + fn run_node_index_test(create_index_fn: F) + where + F: Fn(&Graph, IndexSpec) -> Result<(), GraphError>, + { + let graph = init_graph(); + + let index_spec = IndexSpecBuilder::new(graph.clone()) + .with_node_metadata(vec!["y"]) + .unwrap() + .with_node_properties(vec!["p1"]) + .unwrap() + .build(); + create_index_fn(&graph, index_spec.clone()).unwrap(); + + let filter = NodeFilter.property("p2").temporal().last().eq(50u64); + assert_eq!(search_nodes(&graph, filter.clone()), vec!["pometry"]); + + let node = graph + .add_node(1, "shivam", [("p1", 100u64)], Some("fire_nation"), None) + .unwrap(); + assert_eq!(index_spec, graph.get_index_spec().unwrap()); + + let filter = NodeFilter.property("p1").temporal().last().eq(100u64); + assert_eq!(search_nodes(&graph, filter.clone()), vec!["shivam"]); + + node.add_metadata([("z", true)]).unwrap(); + assert_eq!(index_spec, graph.get_index_spec().unwrap()); + let filter = NodeFilter.metadata("z").eq(true); + assert_eq!(search_nodes(&graph, filter.clone()), vec!["shivam"]); + + node.update_metadata([("z", false)]).unwrap(); + assert_eq!(index_spec, graph.get_index_spec().unwrap()); + let filter = NodeFilter.metadata("z").eq(false); + assert_eq!(search_nodes(&graph, filter.clone()), vec!["shivam"]); + } + + fn run_edge_index_test(create_index_fn: F) + where + F: Fn(&Graph, IndexSpec) -> Result<(), GraphError>, + { + let graph = init_graph(); + + let index_spec = IndexSpecBuilder::new(graph.clone()) + .with_node_metadata(vec!["y"]) + .unwrap() + .with_node_properties(vec!["p2"]) + .unwrap() + .build(); + create_index_fn(&graph, index_spec.clone()).unwrap(); + + let edge = graph + .add_edge(1, "shivam", "kapoor", [("p1", 100u64)], None) + .unwrap(); + assert_eq!(index_spec, graph.get_index_spec().unwrap()); + let filter = EdgeFilter.property("p1").temporal().last().eq(100u64); + assert_eq!(search_edges(&graph, filter.clone()), vec!["shivam->kapoor"]); + + edge.add_metadata([("z", true)], None).unwrap(); + assert_eq!(index_spec, graph.get_index_spec().unwrap()); + let filter = EdgeFilter.metadata("z").eq(true); + assert_eq!(search_edges(&graph, filter.clone()), vec!["shivam->kapoor"]); + + edge.update_metadata([("z", false)], None).unwrap(); + assert_eq!(index_spec, graph.get_index_spec().unwrap()); + let filter = EdgeFilter.metadata("z").eq(false); + assert_eq!(search_edges(&graph, filter.clone()), vec!["shivam->kapoor"]); + } + } +} diff --git a/raphtory/tests/node_property_filter.rs b/raphtory-test-utils/tests/node_property_filter.rs similarity index 99% rename from raphtory/tests/node_property_filter.rs rename to raphtory-test-utils/tests/node_property_filter.rs index cb2385efd7..966f6e4a74 100644 --- a/raphtory/tests/node_property_filter.rs +++ b/raphtory-test-utils/tests/node_property_filter.rs @@ -4,7 +4,6 @@ use raphtory::{ db::{ api::view::filter_ops::{Filter, NodeSelect}, graph::{ - assertions::assert_ok_or_missing_nodes, graph::assert_edges_equal, views::filter::model::{ node_filter::{ops::NodeFilterOps, NodeFilter}, @@ -14,6 +13,9 @@ use raphtory::{ }, }, prelude::*, +}; +use raphtory_test_utils::{ + assertions::assert_ok_or_missing_nodes, test_utils::{ add_node_props, build_edge_list, build_graph_from_edge_list, build_node_props, node_filtered_graph, diff --git a/raphtory/tests/node_test.rs b/raphtory-test-utils/tests/node_test.rs similarity index 99% rename from raphtory/tests/node_test.rs rename to raphtory-test-utils/tests/node_test.rs index 2a2c607bc3..1dccdeccaa 100644 --- a/raphtory/tests/node_test.rs +++ b/raphtory-test-utils/tests/node_test.rs @@ -1,14 +1,13 @@ use raphtory::{ + core::storage::timeindex::AsTime, db::{ api::view::Filter, graph::views::filter::model::{NodeViewFilterOps, ViewWrapOps}, }, prelude::*, - test_storage, - test_utils::test_graph, }; use raphtory_api::core::storage::arc_str::ArcStr; -use raphtory_core::storage::timeindex::AsTime; +use raphtory_test_utils::{test_storage, test_utils::test_graph}; use std::collections::HashMap; #[test] diff --git a/raphtory/tests/proto_test.rs b/raphtory-test-utils/tests/proto_test.rs similarity index 99% rename from raphtory/tests/proto_test.rs rename to raphtory-test-utils/tests/proto_test.rs index 730d69b42e..55514816c4 100644 --- a/raphtory/tests/proto_test.rs +++ b/raphtory-test-utils/tests/proto_test.rs @@ -1,5 +1,4 @@ -#[cfg(test)] -#[cfg(feature = "proto")] +#[cfg(all(test, feature = "test-utils", feature = "proto"))] mod proto_test { use chrono::{DateTime, NaiveDateTime}; use itertools::Itertools; diff --git a/raphtory/tests/serialise_test.rs b/raphtory-test-utils/tests/serialise_test.rs similarity index 99% rename from raphtory/tests/serialise_test.rs rename to raphtory-test-utils/tests/serialise_test.rs index 13a4faea94..5e69d1fdc1 100644 --- a/raphtory/tests/serialise_test.rs +++ b/raphtory-test-utils/tests/serialise_test.rs @@ -1,5 +1,4 @@ -#[cfg(test)] -#[cfg(feature = "proto")] +#[cfg(all(test, feature = "test-utils", feature = "proto"))] mod serialise_test { use arrow::{array::types::Int32Type, datatypes::UInt8Type}; diff --git a/raphtory/tests/subgraph_tests.rs b/raphtory-test-utils/tests/subgraph_tests.rs similarity index 95% rename from raphtory/tests/subgraph_tests.rs rename to raphtory-test-utils/tests/subgraph_tests.rs index 40830931c2..85ac56011c 100644 --- a/raphtory/tests/subgraph_tests.rs +++ b/raphtory-test-utils/tests/subgraph_tests.rs @@ -5,10 +5,12 @@ use raphtory::{ algorithms::{components::weakly_connected_components, motifs::triangle_count::triangle_count}, db::graph::{graph::assert_graph_equal, views::deletion_graph::PersistentGraph}, prelude::*, +}; +use raphtory_storage::mutation::addition_ops::InternalAdditionOps; +use raphtory_test_utils::{ test_storage, test_utils::{build_graph, build_graph_strat}, }; -use raphtory_storage::mutation::addition_ops::InternalAdditionOps; use serde_json::json; use std::collections::BTreeSet; @@ -137,13 +139,11 @@ pub mod test_filters_node_subgraph { use raphtory::{ db::{ api::view::StaticGraphViewOps, - graph::{ - assertions::GraphTransformer, - views::{node_subgraph::NodeSubgraph, window_graph::WindowedGraph}, - }, + graph::views::{node_subgraph::NodeSubgraph, window_graph::WindowedGraph}, }, prelude::{GraphViewOps, NodeViewOps, TimeOps}, }; + use raphtory_test_utils::assertions::GraphTransformer; use std::ops::Range; struct NodeSubgraphTransformer(Option>); @@ -180,19 +180,17 @@ pub mod test_filters_node_subgraph { use raphtory::{ db::{ api::view::StaticGraphViewOps, - graph::{ - assertions::{ - assert_filter_nodes_results, assert_search_nodes_results, - TestGraphVariants, TestVariants, - }, - views::filter::model::{ - property_filter::ops::PropertyFilterOps, PropertyFilterFactory, - }, + graph::views::filter::model::{ + property_filter::ops::PropertyFilterOps, PropertyFilterFactory, }, }, prelude::{AdditionOps, NodeFilter}, }; use raphtory_api::core::entities::properties::prop::Prop; + use raphtory_test_utils::assertions::{ + assert_filter_nodes_results, assert_search_nodes_results, TestGraphVariants, + TestVariants, + }; fn init_graph(graph: G) -> G { let nodes = vec![ @@ -343,18 +341,16 @@ pub mod test_filters_node_subgraph { use raphtory::{ db::{ api::view::StaticGraphViewOps, - graph::{ - assertions::{ - assert_filter_edges_results, assert_search_edges_results, TestVariants, - }, - views::filter::model::{ - property_filter::ops::PropertyFilterOps, PropertyFilterFactory, - }, + graph::views::filter::model::{ + property_filter::ops::PropertyFilterOps, PropertyFilterFactory, }, }, prelude::{AdditionOps, EdgeFilter}, }; use raphtory_api::core::entities::properties::prop::Prop; + use raphtory_test_utils::assertions::{ + assert_filter_edges_results, assert_search_edges_results, TestVariants, + }; use crate::test_filters_node_subgraph::{ NodeSubgraphTransformer, WindowedNodeSubgraphTransformer, diff --git a/raphtory/tests/test_deletions.rs b/raphtory-test-utils/tests/test_deletions.rs similarity index 99% rename from raphtory/tests/test_deletions.rs rename to raphtory-test-utils/tests/test_deletions.rs index 58ff3ad0eb..f150de496a 100644 --- a/raphtory/tests/test_deletions.rs +++ b/raphtory-test-utils/tests/test_deletions.rs @@ -1,21 +1,22 @@ use itertools::Itertools; use proptest::{arbitrary::any, proptest, sample::subsequence}; -use raphtory::test_utils::{build_graph, build_graph_strat}; use raphtory::{ db::graph::{ edge::EdgeView, graph::{assert_graph_equal, assert_persistent_materialize_graph_equal}, views::deletion_graph::PersistentGraph, - // views::deletion_graph::{GraphTimeSemanticsOps, PersistentGraph}, }, prelude::*, - test_storage, }; use raphtory_api::core::{ entities::GID, storage::timeindex::{AsTime, EventTime}, }; use raphtory_storage::mutation::addition_ops::InternalAdditionOps; +use raphtory_test_utils::{ + test_storage, + test_utils::{build_graph, build_graph_strat}, +}; use rayon::ThreadPoolBuilder; use std::ops::Range; diff --git a/raphtory/tests/test_edge.rs b/raphtory-test-utils/tests/test_edge.rs similarity index 98% rename from raphtory/tests/test_edge.rs rename to raphtory-test-utils/tests/test_edge.rs index 1f22b24d6f..85a8499823 100644 --- a/raphtory/tests/test_edge.rs +++ b/raphtory-test-utils/tests/test_edge.rs @@ -5,13 +5,12 @@ use raphtory::{ graph::views::filter::model::{EdgeViewFilterOps, ViewWrapOps}, }, prelude::*, - test_storage, - test_utils::test_graph, }; use raphtory_api::core::{ entities::LayerId, storage::{arc_str::ArcStr, timeindex::AsTime}, }; +use raphtory_test_utils::{test_storage, test_utils::test_graph}; use std::collections::HashMap; #[test] diff --git a/raphtory/tests/test_edge_view.rs b/raphtory-test-utils/tests/test_edge_view.rs similarity index 98% rename from raphtory/tests/test_edge_view.rs rename to raphtory-test-utils/tests/test_edge_view.rs index 577fb6f8c3..b4a53a2e34 100644 --- a/raphtory/tests/test_edge_view.rs +++ b/raphtory-test-utils/tests/test_edge_view.rs @@ -1,4 +1,5 @@ -use raphtory::{prelude::*, test_storage, test_utils::test_graph}; +use raphtory::prelude::*; +use raphtory_test_utils::{test_storage, test_utils::test_graph}; #[test] fn test_exploded_edge_properties() { diff --git a/raphtory/tests/test_exploded_edges.rs b/raphtory-test-utils/tests/test_exploded_edges.rs similarity index 99% rename from raphtory/tests/test_exploded_edges.rs rename to raphtory-test-utils/tests/test_exploded_edges.rs index 820c91963b..32364f78ff 100644 --- a/raphtory/tests/test_exploded_edges.rs +++ b/raphtory-test-utils/tests/test_exploded_edges.rs @@ -1,5 +1,6 @@ use itertools::Itertools; -use raphtory::{prelude::*, test_storage}; +use raphtory::prelude::*; +use raphtory_test_utils::test_storage; #[test] fn test_add_node_properties_ordered_by_event_id() { diff --git a/raphtory/tests/test_filters.rs b/raphtory-test-utils/tests/test_filters.rs similarity index 98% rename from raphtory/tests/test_filters.rs rename to raphtory-test-utils/tests/test_filters.rs index 37038c2f0d..156c793d3e 100644 --- a/raphtory/tests/test_filters.rs +++ b/raphtory-test-utils/tests/test_filters.rs @@ -1,7 +1,4 @@ -use raphtory::{ - db::{api::view::StaticGraphViewOps, graph::assertions::GraphTransformer}, - prelude::*, -}; +use raphtory::{db::api::view::StaticGraphViewOps, prelude::*}; #[cfg(test)] mod test_composite_filters { @@ -108,6 +105,7 @@ use raphtory_api::core::entities::properties::prop::IntoProp; use raphtory_storage::mutation::{ addition_ops::InternalAdditionOps, property_addition_ops::InternalPropertyAdditionOps, }; +use raphtory_test_utils::assertions::GraphTransformer; struct IdentityGraphTransformer; @@ -126,14 +124,9 @@ mod test_property_semantics { use raphtory::{ db::{ api::view::{filter_ops::Filter, StaticGraphViewOps}, - graph::{ - assertions::{ - assert_filter_nodes_results, assert_search_nodes_results, TestVariants, - }, - views::filter::model::{ - node_filter::NodeFilter, property_filter::ops::PropertyFilterOps, - PropertyFilterFactory, TemporalPropertyFilterFactory, - }, + graph::views::filter::model::{ + node_filter::NodeFilter, property_filter::ops::PropertyFilterOps, + PropertyFilterFactory, TemporalPropertyFilterFactory, }, }, errors::GraphError, @@ -143,6 +136,9 @@ mod test_property_semantics { use raphtory_storage::mutation::{ addition_ops::InternalAdditionOps, property_addition_ops::InternalPropertyAdditionOps, }; + use raphtory_test_utils::assertions::{ + assert_filter_nodes_results, assert_search_nodes_results, TestVariants, + }; fn init_graph(graph: G) -> G { let nodes = [ @@ -473,18 +469,12 @@ mod test_property_semantics { use raphtory::{ db::{ api::view::{filter_ops::Filter, EdgeViewOps, StaticGraphViewOps}, - graph::{ - assertions::{ - assert_filter_edges_results, assert_search_edges_results, - TestGraphVariants, TestVariants, WindowGraphTransformer, - }, - views::filter::{ - model::{ - edge_filter::EdgeFilter, property_filter::ops::PropertyFilterOps, - PropertyFilterFactory, TemporalPropertyFilterFactory, - }, - CreateFilter, + graph::views::filter::{ + model::{ + edge_filter::EdgeFilter, property_filter::ops::PropertyFilterOps, + PropertyFilterFactory, TemporalPropertyFilterFactory, }, + CreateFilter, }, }, errors::GraphError, @@ -494,6 +484,10 @@ mod test_property_semantics { use raphtory_storage::mutation::{ addition_ops::InternalAdditionOps, property_addition_ops::InternalPropertyAdditionOps, }; + use raphtory_test_utils::assertions::{ + assert_filter_edges_results, assert_search_edges_results, TestGraphVariants, + TestVariants, WindowGraphTransformer, + }; fn init_graph< G: StaticGraphViewOps @@ -1655,7 +1649,6 @@ fn init_edges_graph_with_str_ids_del< graph } -#[cfg(test)] mod test_node_filter { use crate::{ init_nodes_graph, init_nodes_graph_with_num_ids, init_nodes_graph_with_str_ids, @@ -1665,16 +1658,10 @@ mod test_node_filter { algorithms::alternating_mask::alternating_mask, db::{ api::view::{filter_ops::NodeSelect, Filter}, - graph::{ - assertions::{ - assert_filter_nodes_results, assert_search_nodes_results, - assert_select_nodes_results, TestVariants, - }, - views::filter::model::{ - node_filter::ops::{NodeFilterOps, NodeIdFilterOps}, - ComposableFilter, CompositeNodeFilter, NodeViewFilterOps, TryAsCompositeFilter, - ViewWrapOps, - }, + graph::views::filter::model::{ + node_filter::ops::{NodeFilterOps, NodeIdFilterOps}, + ComposableFilter, CompositeNodeFilter, NodeViewFilterOps, TryAsCompositeFilter, + ViewWrapOps, }, }, prelude::{ @@ -1682,6 +1669,10 @@ mod test_node_filter { NO_PROPS, }, }; + use raphtory_test_utils::assertions::{ + assert_filter_nodes_results, assert_search_nodes_results, assert_select_nodes_results, + TestVariants, + }; #[test] fn test_node_list_is_preserved() { @@ -2534,18 +2525,18 @@ mod test_node_filter { #[cfg(test)] mod test_node_property_filter { use crate::{init_nodes_graph, IdentityGraphTransformer}; - use raphtory::db::graph::{ - assertions::{assert_filter_nodes_results, assert_search_nodes_results, TestVariants}, - views::filter::model::{ - graph_filter::GraphFilter, - node_filter::NodeFilter, - not_filter::NotFilter, - property_filter::ops::{ElemQualifierOps, ListAggOps, PropertyFilterOps}, - windowed_filter::Windowed, - ComposableFilter, PropertyFilterFactory, TemporalPropertyFilterFactory, ViewWrapOps, - }, + use raphtory::db::graph::views::filter::model::{ + graph_filter::GraphFilter, + node_filter::NodeFilter, + not_filter::NotFilter, + property_filter::ops::{ElemQualifierOps, ListAggOps, PropertyFilterOps}, + windowed_filter::Windowed, + ComposableFilter, PropertyFilterFactory, TemporalPropertyFilterFactory, ViewWrapOps, }; use raphtory_api::core::entities::properties::prop::Prop; + use raphtory_test_utils::assertions::{ + assert_filter_nodes_results, assert_search_nodes_results, TestVariants, + }; use std::vec; #[test] @@ -4606,18 +4597,16 @@ mod test_node_composite_filter { use crate::{init_edges_graph, init_nodes_graph, IdentityGraphTransformer}; use raphtory::{ - db::graph::{ - assertions::{ - assert_filter_neighbours_results, assert_filter_nodes_results, - assert_search_nodes_results, TestVariants, - }, - views::filter::model::{ - node_filter::ops::NodeFilterOps, property_filter::ops::PropertyFilterOps, - ComposableFilter, PropertyFilterFactory, TryAsCompositeFilter, - }, + db::graph::views::filter::model::{ + node_filter::ops::NodeFilterOps, property_filter::ops::PropertyFilterOps, + ComposableFilter, PropertyFilterFactory, TryAsCompositeFilter, }, prelude::NodeFilter, }; + use raphtory_test_utils::assertions::{ + assert_filter_neighbours_results, assert_filter_nodes_results, assert_search_nodes_results, + TestVariants, + }; #[test] fn test_filter_nodes_by_props_added_at_different_times() { @@ -5050,19 +5039,13 @@ mod test_node_property_filter_agg { use raphtory::{ db::{ api::view::StaticGraphViewOps, - graph::{ - assertions::{ - assert_filter_nodes_err, assert_filter_nodes_results, - assert_search_nodes_results, TestVariants::All, - }, - views::filter::{ - model::{ - node_filter::NodeFilter, - property_filter::ops::{ElemQualifierOps, ListAggOps, PropertyFilterOps}, - PropertyFilterFactory, TemporalPropertyFilterFactory, TryAsCompositeFilter, - }, - CreateFilter, + graph::views::filter::{ + model::{ + node_filter::NodeFilter, + property_filter::ops::{ElemQualifierOps, ListAggOps, PropertyFilterOps}, + PropertyFilterFactory, TemporalPropertyFilterFactory, TryAsCompositeFilter, }, + CreateFilter, }, }, prelude::{AdditionOps, GraphViewOps, PropertyAdditionOps}, @@ -5074,6 +5057,10 @@ mod test_node_property_filter_agg { use raphtory_storage::mutation::{ addition_ops::InternalAdditionOps, property_addition_ops::InternalPropertyAdditionOps, }; + use raphtory_test_utils::assertions::{ + assert_filter_nodes_err, assert_filter_nodes_results, assert_search_nodes_results, + TestVariants::All, + }; fn list_u8(xs: &[u8]) -> Prop { Prop::list(xs.iter().copied().map(Prop::U8)) @@ -8579,18 +8566,16 @@ mod test_edge_filter { init_edges_graph, init_edges_graph_with_num_ids, init_edges_graph_with_str_ids, init_edges_graph_with_str_ids_del, init_nodes_graph, IdentityGraphTransformer, }; - use raphtory::db::graph::{ - assertions::{ - assert_filter_edges_results, assert_search_edges_results, assert_select_edges_results, - TestGraphVariants, TestVariants, - }, - views::filter::model::{ - edge_filter::EdgeFilter, - node_filter::ops::{NodeFilterOps, NodeIdFilterOps}, - property_filter::ops::{ListAggOps, PropertyFilterOps}, - ComposableFilter, EdgeViewFilterOps, PropertyFilterFactory, - TemporalPropertyFilterFactory, ViewWrapOps, - }, + use raphtory::db::graph::views::filter::model::{ + edge_filter::EdgeFilter, + node_filter::ops::{NodeFilterOps, NodeIdFilterOps}, + property_filter::ops::{ListAggOps, PropertyFilterOps}, + ComposableFilter, EdgeViewFilterOps, PropertyFilterFactory, TemporalPropertyFilterFactory, + ViewWrapOps, + }; + use raphtory_test_utils::assertions::{ + assert_filter_edges_results, assert_search_edges_results, assert_select_edges_results, + TestGraphVariants, TestVariants, }; #[test] @@ -10061,18 +10046,16 @@ mod test_edge_filter { #[cfg(test)] mod test_edge_property_filter { use crate::{init_edges_graph, init_edges_graph2, IdentityGraphTransformer}; - use raphtory::db::graph::{ - assertions::{ - assert_filter_edges_results, assert_search_edges_results, TestGraphVariants, - TestVariants, - }, - views::filter::model::{ - edge_filter::EdgeFilter, - property_filter::ops::{ElemQualifierOps, ListAggOps, PropertyFilterOps}, - ComposableFilter, PropertyFilterFactory, TemporalPropertyFilterFactory, ViewWrapOps, - }, + use raphtory::db::graph::views::filter::model::{ + edge_filter::EdgeFilter, + property_filter::ops::{ElemQualifierOps, ListAggOps, PropertyFilterOps}, + ComposableFilter, PropertyFilterFactory, TemporalPropertyFilterFactory, ViewWrapOps, }; + use raphtory_api::core::entities::properties::prop::Prop; + use raphtory_test_utils::assertions::{ + assert_filter_edges_results, assert_search_edges_results, TestGraphVariants, TestVariants, + }; #[test] fn test_filter_edges_for_property_eq() { @@ -11907,19 +11890,17 @@ mod test_edge_property_filter { #[cfg(test)] mod test_edge_composite_filter { - use crate::{init_edges_graph, IdentityGraphTransformer}; - use raphtory::db::graph::{ - assertions::{ - assert_filter_edges_results, assert_search_edges_results, TestGraphVariants, - TestVariants, - }, - views::filter::model::{ - edge_filter::EdgeFilter, node_filter::ops::NodeFilterOps, - property_filter::ops::PropertyFilterOps, ComposableFilter, PropertyFilterFactory, - TryAsCompositeFilter, - }, + use raphtory::db::graph::views::filter::model::{ + edge_filter::EdgeFilter, node_filter::ops::NodeFilterOps, + property_filter::ops::PropertyFilterOps, ComposableFilter, PropertyFilterFactory, + TryAsCompositeFilter, + }; + use raphtory_test_utils::assertions::{ + assert_filter_edges_results, assert_search_edges_results, TestGraphVariants, TestVariants, }; + use crate::{init_edges_graph, IdentityGraphTransformer}; + #[test] fn test_filter_edge_for_src_dst() { // TODO: PropertyFilteringNotImplemented for variants persistent_graph, persistent_disk_graph for filter_edges. diff --git a/raphtory/tests/test_history.rs b/raphtory-test-utils/tests/test_history.rs similarity index 100% rename from raphtory/tests/test_history.rs rename to raphtory-test-utils/tests/test_history.rs diff --git a/raphtory/tests/test_layers.rs b/raphtory-test-utils/tests/test_layers.rs similarity index 97% rename from raphtory/tests/test_layers.rs rename to raphtory-test-utils/tests/test_layers.rs index 880b96f8cb..25709b56f8 100644 --- a/raphtory/tests/test_layers.rs +++ b/raphtory-test-utils/tests/test_layers.rs @@ -3,10 +3,12 @@ use proptest::proptest; use raphtory::{ db::graph::{graph::assert_graph_equal, views::deletion_graph::PersistentGraph}, prelude::*, +}; +use raphtory_api::core::entities::GID; +use raphtory_test_utils::{ test_storage, test_utils::{build_graph, build_graph_layer, build_graph_strat, GraphFixture}, }; -use raphtory_api::core::entities::GID; use serde_json::json; #[test] @@ -185,13 +187,11 @@ pub mod test_filters_layer_graph { use raphtory::{ db::{ api::view::StaticGraphViewOps, - graph::{ - assertions::GraphTransformer, - views::{layer_graph::LayeredGraph, window_graph::WindowedGraph}, - }, + graph::views::{layer_graph::LayeredGraph, window_graph::WindowedGraph}, }, prelude::{LayerOps, TimeOps}, }; + use raphtory_test_utils::assertions::GraphTransformer; use std::ops::Range; struct LayeredGraphTransformer(Vec); @@ -229,16 +229,13 @@ pub mod test_filters_layer_graph { LayeredGraphTransformer, LayeredGraphWindowTransformer, }; use raphtory::{ - db::graph::{ - assertions::{ - assert_filter_nodes_results, assert_search_nodes_results, TestGraphVariants, - TestVariants, - }, - views::filter::model::PropertyFilterFactory, - }, - prelude::NodeFilter, + db::graph::views::filter::model::PropertyFilterFactory, prelude::NodeFilter, }; + use raphtory_test_utils::assertions::{ + assert_filter_nodes_results, assert_search_nodes_results, TestGraphVariants, + TestVariants, + }; fn init_graph(graph: G) -> G { let edges = vec![ (6, "N1", "N2", vec![("p1", Prop::U64(2u64))], Some("layer1")), @@ -504,19 +501,17 @@ pub mod test_filters_layer_graph { use raphtory::{ db::{ api::view::StaticGraphViewOps, - graph::{ - assertions::{ - assert_filter_edges_results, assert_search_edges_results, - TestGraphVariants, TestVariants, - }, - views::filter::model::{ - property_filter::ops::PropertyFilterOps, PropertyFilterFactory, - }, + graph::views::filter::model::{ + property_filter::ops::PropertyFilterOps, PropertyFilterFactory, }, }, prelude::{AdditionOps, EdgeFilter}, }; use raphtory_api::core::entities::properties::prop::Prop; + use raphtory_test_utils::assertions::{ + assert_filter_edges_results, assert_search_edges_results, TestGraphVariants, + TestVariants, + }; use crate::test_filters_layer_graph::{ LayeredGraphTransformer, LayeredGraphWindowTransformer, diff --git a/raphtory/tests/test_materialize.rs b/raphtory-test-utils/tests/test_materialize.rs similarity index 98% rename from raphtory/tests/test_materialize.rs rename to raphtory-test-utils/tests/test_materialize.rs index 77a20b62b9..d6ca5e4683 100644 --- a/raphtory/tests/test_materialize.rs +++ b/raphtory-test-utils/tests/test_materialize.rs @@ -3,11 +3,15 @@ use proptest::{arbitrary::any, proptest}; use raphtory::{ db::graph::graph::{assert_graph_equal, assert_graph_equal_timestamps}, prelude::*, - test_storage, - test_utils::{build_edge_list, build_graph_from_edge_list}, + // test_storage, + // test_utils::{build_edge_list, build_graph_from_edge_list}, }; use raphtory_api::core::storage::arc_str::OptionAsStr; use raphtory_storage::core_ops::CoreGraphOps; +use raphtory_test_utils::{ + test_storage, + test_utils::{build_edge_list, build_graph_from_edge_list}, +}; use std::ops::Range; #[test] diff --git a/raphtory/tests/test_materialize_sf10.rs b/raphtory-test-utils/tests/test_materialize_sf10.rs similarity index 97% rename from raphtory/tests/test_materialize_sf10.rs rename to raphtory-test-utils/tests/test_materialize_sf10.rs index 89cc4874d4..c37e7c340a 100644 --- a/raphtory/tests/test_materialize_sf10.rs +++ b/raphtory-test-utils/tests/test_materialize_sf10.rs @@ -6,16 +6,15 @@ use raphtory::{ api::view::{materialize_impl, MaterializedGraph}, graph::graph::{assert_graph_equal_timestamps, graph_equal}, }, - prelude::{AdditionOps, DeletionOps, Graph, GraphViewOps, LayerOps, PropertyAdditionOps}, -}; -#[cfg(feature = "io")] -use raphtory::{ io::parquet_loaders::{ get_parquet_file_paths, load_edge_deletions_from_parquet, load_edge_metadata_from_parquet, load_edges_from_parquet, load_graph_props_from_parquet, load_node_metadata_from_parquet, load_nodes_from_parquet, }, - prelude::{ParquetDecoder, ParquetEncoder}, + prelude::{ + AdditionOps, DeletionOps, Graph, GraphViewOps, LayerOps, ParquetDecoder, ParquetEncoder, + PropertyAdditionOps, + }, }; use raphtory_storage::core_ops::CoreGraphOps; use std::{ @@ -25,26 +24,22 @@ use std::{ }; use storage::persist::strategy::PersistenceStrategy; -#[cfg(feature = "io")] fn default_sf10_graph_path() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("../../ldbc/data/social_network-sf10-CsvComposite-LongDateFormatter/graph") } -#[cfg(feature = "io")] fn default_sf10_parquet_path() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( "../../ldbc/data/social_network-sf10-CsvComposite-LongDateFormatter/parquet/data0/graph0", ) } -#[cfg(feature = "io")] fn default_sf1_graph_path() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("../../ldbc/data/social_network-sf1-CsvComposite-LongDateFormatter/graph") } -#[cfg(feature = "io")] fn default_sf1_parquet_path() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")).join( "../../ldbc/data/social_network-sf1-CsvComposite-LongDateFormatter/parquet/data0/graph0", @@ -83,7 +78,6 @@ fn test_materialize() { assert_graph_equal_timestamps(&expected, &g); } -#[cfg(feature = "io")] #[test] #[ignore = "requires a locally persisted SNB SF1 graph produced by ldbc/load_snb_sf10.py"] fn test_get_materialize_snb_sf1_time() { @@ -127,7 +121,6 @@ fn test_get_materialize_snb_sf1_time() { remove_dir_all_ignore_not_found(&rb_materialize_graph_path).unwrap(); } -#[cfg(feature = "io")] #[test] #[ignore = "requires a locally persisted SNB SF1 graph produced by ldbc/load_snb_sf10.py"] fn test_materialize_filtered_sf1_matches() { @@ -210,7 +203,6 @@ fn test_materialize_filtered_sf1_matches() { remove_dir_all_ignore_not_found(&rb_materialize_graph_path).unwrap(); } -#[cfg(feature = "io")] fn get_materialize_time(graph_path: &Path, materialize_graph_path: &Path) -> Duration { remove_dir_all_ignore_not_found(&materialize_graph_path).unwrap(); fs::create_dir_all(&materialize_graph_path).unwrap(); @@ -248,7 +240,6 @@ fn get_materialize_time(graph_path: &Path, materialize_graph_path: &Path) -> Dur materialize_elapsed } -#[cfg(feature = "io")] fn get_parquet_decode_time( graph_path: &Path, parquet_path: &Path, @@ -319,7 +310,6 @@ const EDGES_T_PARQUET_DIR: &str = "edges_t"; const EDGES_D_PARQUET_DIR: &str = "edges_d"; const EDGES_C_PARQUET_DIR: &str = "edges_c"; -#[cfg(feature = "io")] fn parquet_prop_columns(path: &Path, exclude: &[&str]) -> Vec { get_parquet_file_paths(path) .unwrap() @@ -338,7 +328,6 @@ fn parquet_prop_columns(path: &Path, exclude: &[&str]) -> Vec { .unwrap_or_default() } -#[cfg(feature = "io")] fn get_parquet_encode_time(graph_path: &Path, parquet_graph_path: &Path) -> Duration { remove_dir_all_ignore_not_found(&parquet_graph_path).unwrap(); fs::create_dir_all(&parquet_graph_path).unwrap(); @@ -368,7 +357,6 @@ fn get_parquet_encode_time(graph_path: &Path, parquet_graph_path: &Path) -> Dura parquet_dump_elapsed } -#[cfg(feature = "io")] fn get_parquet_df_loader_time( graph_path: &Path, parquet_path: &Path, @@ -656,7 +644,6 @@ fn get_parquet_df_loader_time( parquet_load_elapsed } -#[cfg(feature = "io")] #[test] #[ignore = "requires locally persisted SNB SF10 graphs and parquet export"] fn test_current() { diff --git a/raphtory/tests/tests_node_type_filtered_subgraph.rs b/raphtory-test-utils/tests/tests_node_type_filtered_subgraph.rs similarity index 96% rename from raphtory/tests/tests_node_type_filtered_subgraph.rs rename to raphtory-test-utils/tests/tests_node_type_filtered_subgraph.rs index 7105129b7d..3c7265e9f2 100644 --- a/raphtory/tests/tests_node_type_filtered_subgraph.rs +++ b/raphtory-test-utils/tests/tests_node_type_filtered_subgraph.rs @@ -10,9 +10,11 @@ use raphtory::{ }, }, prelude::*, - test_utils::{build_graph, build_graph_strat, make_node_types, GraphFixture}, }; use raphtory_storage::mutation::addition_ops::InternalAdditionOps; +use raphtory_test_utils::test_utils::{ + build_graph, build_graph_strat, make_node_types, GraphFixture, +}; use serde_json::json; use std::ops::Range; @@ -135,16 +137,14 @@ mod test_filters_node_type_filtered_subgraph { use raphtory::{ db::{ api::{state::ops::filter::NodeTypeFilterOp, view::StaticGraphViewOps}, - graph::{ - assertions::GraphTransformer, - views::{ - filter::node_filtered_graph::NodeFilteredGraph, layer_graph::LayeredGraph, - window_graph::WindowedGraph, - }, + graph::views::{ + filter::node_filtered_graph::NodeFilteredGraph, layer_graph::LayeredGraph, + window_graph::WindowedGraph, }, }, prelude::{GraphViewOps, LayerOps, NodeViewOps, TimeOps}, }; + use raphtory_test_utils::assertions::GraphTransformer; use std::ops::Range; fn get_all_node_types(graph: &G) -> Vec { @@ -253,14 +253,11 @@ mod test_filters_node_type_filtered_subgraph { NodeTypeGraphTransformer, WindowedNodeTypeGraphTransformer, }; use raphtory::{ - db::graph::{ - assertions::{ - assert_filter_nodes_results, assert_search_nodes_results, TestGraphVariants, - TestVariants, - }, - views::filter::model::PropertyFilterFactory, - }, - prelude::NodeFilter, + db::graph::views::filter::model::PropertyFilterFactory, prelude::NodeFilter, + }; + use raphtory_test_utils::assertions::{ + assert_filter_nodes_results, assert_search_nodes_results, TestGraphVariants, + TestVariants, }; #[test] fn test_nodes_filters() { @@ -500,15 +497,12 @@ mod test_filters_node_type_filtered_subgraph { NodeTypeGraphTransformer, WindowedNodeTypeGraphTransformer, }; use raphtory::{ - db::graph::{ - assertions::{ - assert_filter_edges_results, assert_search_edges_results, TestVariants, - }, - views::filter::model::PropertyFilterFactory, - }, - prelude::EdgeFilter, + db::graph::views::filter::model::PropertyFilterFactory, prelude::EdgeFilter, }; + use raphtory_test_utils::assertions::{ + assert_filter_edges_results, assert_search_edges_results, TestVariants, + }; #[test] fn test_edges_filters() { let filter = EdgeFilter.property("p1").eq(1u64); diff --git a/raphtory/tests/time_tests.rs b/raphtory-test-utils/tests/time_tests.rs similarity index 99% rename from raphtory/tests/time_tests.rs rename to raphtory-test-utils/tests/time_tests.rs index 633967d91b..7ea852659d 100644 --- a/raphtory/tests/time_tests.rs +++ b/raphtory-test-utils/tests/time_tests.rs @@ -8,12 +8,12 @@ use raphtory::{ }, }, prelude::{DeletionOps, GraphViewOps, LayerOps, TimeOps, NO_PROPS}, - test_storage, }; use raphtory_api::core::{ storage::timeindex::AsTime, utils::time::{ParseTimeError, TryIntoTime}, }; +use raphtory_test_utils::test_storage; // start inclusive, end exclusive fn graph_with_timeline(start: i64, end: i64) -> Graph { diff --git a/raphtory/tests/valid_graph.rs b/raphtory-test-utils/tests/valid_graph.rs similarity index 98% rename from raphtory/tests/valid_graph.rs rename to raphtory-test-utils/tests/valid_graph.rs index d3dd971db2..f759a6995d 100644 --- a/raphtory/tests/valid_graph.rs +++ b/raphtory-test-utils/tests/valid_graph.rs @@ -7,10 +7,10 @@ use raphtory::{ }, errors::GraphError, prelude::*, - test_utils::{build_graph, build_graph_strat}, }; use raphtory_api::core::storage::timeindex::AsTime; use raphtory_storage::mutation::addition_ops::InternalAdditionOps; +use raphtory_test_utils::test_utils::{build_graph, build_graph_strat}; use std::ops::Range; #[test] @@ -334,7 +334,8 @@ fn broken_degree() { assert_eq!(n4.out_degree(), 2); assert_eq!(n4.in_degree(), 1); - assert_persistent_materialize_graph_equal(&gv, &gv.materialize().unwrap()); // PersistentGraph ignores the earliest time's event id + assert_persistent_materialize_graph_equal(&gv, &gv.materialize().unwrap()); + // PersistentGraph ignores the earliest time's event id } #[test] diff --git a/raphtory/tests/views_test.rs b/raphtory-test-utils/tests/views_test.rs similarity index 99% rename from raphtory/tests/views_test.rs rename to raphtory-test-utils/tests/views_test.rs index fdf70f6a0f..abfa51166a 100644 --- a/raphtory/tests/views_test.rs +++ b/raphtory-test-utils/tests/views_test.rs @@ -5,14 +5,13 @@ use raphtory::{ algorithms::centrality::degree_centrality::degree_centrality, db::graph::{graph::assert_graph_equal, views::window_graph::WindowedGraph}, prelude::*, - test_storage, - test_utils::test_graph, }; use raphtory_api::core::{ entities::GID, storage::timeindex::AsTime, utils::{logging::global_info_logger, time::IntoTime}, }; +use raphtory_test_utils::{test_storage, test_utils::test_graph}; use rayon::prelude::*; use std::ops::Range; use tracing::{error, info}; @@ -536,38 +535,31 @@ mod test_filters_window_graph { mod test_nodes_filters_window_graph { use raphtory::{ - db::{ - api::view::StaticGraphViewOps, - graph::{ - assertions::{ - assert_filter_nodes_results, assert_search_nodes_results, - TestGraphVariants, TestVariants, - }, - views::filter::model::ComposableFilter, - }, - }, + db::{api::view::StaticGraphViewOps, graph::views::filter::model::ComposableFilter}, prelude::{AdditionOps, PropertyAdditionOps}, }; use raphtory_api::core::{entities::properties::prop::Prop, storage::arc_str::ArcStr}; use raphtory_storage::mutation::{ addition_ops::InternalAdditionOps, property_addition_ops::InternalPropertyAdditionOps, }; + use raphtory_test_utils::assertions::{ + assert_filter_nodes_results, assert_search_nodes_results, TestGraphVariants, + TestVariants, + }; use raphtory::{ db::{ api::view::filter_ops::Filter, - graph::{ - assertions::WindowGraphTransformer, - views::filter::model::{ - node_filter::{ops::NodeFilterOps, NodeFilter}, - property_filter::ops::PropertyFilterOps, - PropertyFilterFactory, - }, + graph::views::filter::model::{ + node_filter::{ops::NodeFilterOps, NodeFilter}, + property_filter::ops::PropertyFilterOps, + PropertyFilterFactory, }, }, errors::GraphError, prelude::{Graph, GraphViewOps, TimeOps}, }; + use raphtory_test_utils::assertions::WindowGraphTransformer; fn init_graph(graph: G) -> G { let nodes = vec![ @@ -2682,22 +2674,20 @@ mod test_filters_window_graph { use raphtory::{ db::{ api::view::{filter_ops::Filter, StaticGraphViewOps}, - graph::{ - assertions::{ - assert_filter_edges_results, assert_search_edges_results, - TestGraphVariants, TestVariants, WindowGraphTransformer, - }, - views::filter::model::{ - edge_filter::EdgeFilter, node_filter::ops::NodeFilterOps, - property_filter::ops::PropertyFilterOps, ComposableFilter, - PropertyFilterFactory, - }, + graph::views::filter::model::{ + edge_filter::EdgeFilter, node_filter::ops::NodeFilterOps, + property_filter::ops::PropertyFilterOps, ComposableFilter, + PropertyFilterFactory, }, }, errors::GraphError, prelude::{AdditionOps, Graph, GraphViewOps, PropertyAdditionOps, TimeOps, NO_PROPS}, }; use raphtory_api::core::{entities::properties::prop::Prop, storage::arc_str::ArcStr}; + use raphtory_test_utils::assertions::{ + assert_filter_edges_results, assert_search_edges_results, TestGraphVariants, + TestVariants, WindowGraphTransformer, + }; fn init_graph(graph: G) -> G { let edges = vec![ diff --git a/raphtory/Cargo.toml b/raphtory/Cargo.toml index 1629456088..47d877aecd 100644 --- a/raphtory/Cargo.toml +++ b/raphtory/Cargo.toml @@ -116,7 +116,8 @@ tokio = { workspace = true } # for vector testing dotenv = { workspace = true } # for vector testing streaming-stats = { workspace = true } indoc = { workspace = true } -raphtory = { workspace = true, features = ["test-utils"] } # enable test-utils for integration tests +proptest.workspace = true +proptest-derive.workspace = true [target.'cfg(target_os = "macos")'.dependencies] tikv-jemallocator = "0.6.1" @@ -125,7 +126,7 @@ tikv-jemallocator = "0.6.1" prost-build = { workspace = true, optional = true } [features] -default = [] +default = [] # we can't depend on ourselves but we want to share test-utils # enables progress bars progress = ["dep:kdam"] @@ -184,8 +185,3 @@ proto = [ "dep:memmap2", "io", ] - -test-utils = [ - "dep:proptest", - "dep:proptest-derive" -] diff --git a/raphtory/examples/eth_loader.rs b/raphtory/examples/eth_loader.rs index b8944fda4f..565454eeb6 100644 --- a/raphtory/examples/eth_loader.rs +++ b/raphtory/examples/eth_loader.rs @@ -1,5 +1,6 @@ #[cfg(feature = "io")] use raphtory::io::parquet_loaders::load_edges_from_parquet; +#[cfg(feature = "io")] use raphtory::{arrow_loader::df_loaders::edges::ColumnNames, errors::GraphError, prelude::*}; use std::path::{Path, PathBuf}; diff --git a/raphtory/src/db/api/storage/graph/storage_ops/property_schema.rs b/raphtory/src/db/api/storage/graph/storage_ops/property_schema.rs index 99e88ecc4e..6ea4ff7f75 100644 --- a/raphtory/src/db/api/storage/graph/storage_ops/property_schema.rs +++ b/raphtory/src/db/api/storage/graph/storage_ops/property_schema.rs @@ -3,7 +3,6 @@ use crate::db::api::{ view::BoxedLIter, }; use raphtory_api::{core::storage::arc_str::ArcStr, iter::IntoDynBoxed}; -use raphtory_storage::core_ops::CoreGraphOps; use super::GraphStorage; diff --git a/raphtory/src/db/api/storage/storage.rs b/raphtory/src/db/api/storage/storage.rs index 273acd3589..92bc69d083 100644 --- a/raphtory/src/db/api/storage/storage.rs +++ b/raphtory/src/db/api/storage/storage.rs @@ -102,6 +102,10 @@ impl Storage { } } + #[cfg(feature = "search")] + pub fn index(&self) -> &RwLock { + &self.index + } pub(crate) fn new_at_path(path: impl AsRef) -> Result { let config = Config::default(); let ext = Extension::new(config, Some(path.as_ref()))?; @@ -326,7 +330,7 @@ impl Storage { &self.index } - pub(crate) fn is_indexed(&self) -> bool { + pub fn is_indexed(&self) -> bool { self.index.read_recursive().is_indexed() } diff --git a/raphtory/src/db/api/view/graph.rs b/raphtory/src/db/api/view/graph.rs index 196c3c7a8b..666079e3cc 100644 --- a/raphtory/src/db/api/view/graph.rs +++ b/raphtory/src/db/api/view/graph.rs @@ -936,6 +936,20 @@ impl IndexSpec { } } + pub fn node_metadata(&self) -> &HashSet { + &self.node_metadata + } + + pub fn node_properties(&self) -> &HashSet { + &self.node_properties + } + pub fn edge_metadata(&self) -> &HashSet { + &self.edge_metadata + } + pub fn edge_properties(&self) -> &HashSet { + &self.edge_properties + } + pub fn union(existing: &IndexSpec, other: &IndexSpec) -> IndexSpec { fn union_props(a: &HashSet, b: &HashSet) -> HashSet { a.union(b).copied().collect() diff --git a/raphtory/src/db/graph/mod.rs b/raphtory/src/db/graph/mod.rs index 04b339d461..d354d97f93 100644 --- a/raphtory/src/db/graph/mod.rs +++ b/raphtory/src/db/graph/mod.rs @@ -1,8 +1,6 @@ use raphtory_api::core::storage::dict_mapper::DictMapper; use std::sync::Arc; -#[cfg(any(test, feature = "test-utils"))] -pub mod assertions; pub mod edge; pub mod edges; pub mod graph; diff --git a/raphtory/src/db/graph/views/deletion_graph.rs b/raphtory/src/db/graph/views/deletion_graph.rs index 075a39163f..695b643075 100644 --- a/raphtory/src/db/graph/views/deletion_graph.rs +++ b/raphtory/src/db/graph/views/deletion_graph.rs @@ -30,9 +30,12 @@ use std::{ }; use storage::{ api::graph_props::{GraphPropEntryOps, GraphPropRefOps}, - Config, Extension, + Config, }; +#[cfg(feature = "io")] +use storage::Extension; + /// A graph view where an edge remains active from the time it is added until it is explicitly marked as deleted. /// /// Note that the graph will give you access to all edges that were added at any point in time, even those that are marked as deleted. diff --git a/raphtory/src/db/graph/views/filter/model/node_filter/mod.rs b/raphtory/src/db/graph/views/filter/model/node_filter/mod.rs index 9ccfa9a0e3..6064c5e102 100644 --- a/raphtory/src/db/graph/views/filter/model/node_filter/mod.rs +++ b/raphtory/src/db/graph/views/filter/model/node_filter/mod.rs @@ -80,7 +80,7 @@ impl NodeFilter { V: NodeStateValue + 'graph, T: Clone + Send + Sync + 'graph, { - Ok(state.bool_col_filter(col)?) + state.bool_col_filter(col) } } diff --git a/raphtory/src/db/graph/views/property_redacted_graph.rs b/raphtory/src/db/graph/views/property_redacted_graph.rs index e6eace9e47..c06aa001d7 100644 --- a/raphtory/src/db/graph/views/property_redacted_graph.rs +++ b/raphtory/src/db/graph/views/property_redacted_graph.rs @@ -1,8 +1,7 @@ use crate::db::api::{ properties::internal::{ - EdgePropertySchemaOps, InheritEdgePropertySchemaOps, InheritNodePropertySchemaOps, - InheritTemporalPropertyViewOps, InternalMetadataOps, InternalTemporalPropertiesOps, - NodePropertySchemaOps, + EdgePropertySchemaOps, InheritTemporalPropertyViewOps, InternalMetadataOps, + InternalTemporalPropertiesOps, NodePropertySchemaOps, }, view::{ internal::{ diff --git a/raphtory/src/lib.rs b/raphtory/src/lib.rs index 2cfcd05fd1..980816865f 100644 --- a/raphtory/src/lib.rs +++ b/raphtory/src/lib.rs @@ -114,9 +114,6 @@ pub mod parquet_encoder; pub mod serialise; pub mod storage; -#[cfg(any(test, feature = "test-utils"))] -pub mod test_utils; - /// Return Raphtory crate version. /// /// Returns: diff --git a/raphtory/src/search/entity_index.rs b/raphtory/src/search/entity_index.rs index 506c035ccd..c0fd817251 100644 --- a/raphtory/src/search/entity_index.rs +++ b/raphtory/src/search/entity_index.rs @@ -167,21 +167,20 @@ impl EntityIndex { let indexes = self.temporal_property_indexes.read_recursive(); if let Some(prop_index) = &indexes[prop_id] { let mut writer = prop_index.index.writer(50_000_000)?; - (0..graph.count_nodes()) - .into_par_iter() - .try_for_each(|v_id| { - let node = graph.core_node(VID(v_id)); - for (t, prop_value) in node.tprop(prop_id).iter() { - let prop_doc = prop_index.create_node_temporal_property_document( - t.into(), - v_id as u64, - &prop_value, - )?; - writer.add_document(prop_doc)?; - } + graph.nodes().par_iter().try_for_each(|node| { + let v_id = node.vid(); + let node = graph.core_node(v_id); + for (t, prop_value) in node.tprop(prop_id).iter() { + let prop_doc = prop_index.create_node_temporal_property_document( + t, + v_id.0 as u64, + &prop_value, + )?; + writer.add_document(prop_doc)?; + } - Ok::<(), GraphError>(()) - })?; + Ok::<(), GraphError>(()) + })?; writer.commit()?; } diff --git a/raphtory/src/search/graph_index.rs b/raphtory/src/search/graph_index.rs index 630886490b..c0a76e71c2 100644 --- a/raphtory/src/search/graph_index.rs +++ b/raphtory/src/search/graph_index.rs @@ -58,7 +58,7 @@ impl MutableGraphIndex { pub fn update(&self, graph: &GraphStorage, index_spec: IndexSpec) -> Result<(), GraphError> { let mut existing_spec = self.index_spec.write(); - if let Some(diff_spec) = IndexSpec::diff(&*existing_spec, &index_spec) { + if let Some(diff_spec) = IndexSpec::diff(&existing_spec, &index_spec) { let path = get_node_index_path(&self.path); self.index .node_index @@ -71,7 +71,7 @@ impl MutableGraphIndex { // self.index.print()?; - *existing_spec = IndexSpec::union(&*existing_spec, &diff_spec); + *existing_spec = IndexSpec::union(&existing_spec, &diff_spec); } Ok(()) @@ -447,135 +447,3 @@ fn load_indexes(index_path: &Path) -> Result<(Index, IndexSpec), GraphError> { index_spec, )) } - -#[cfg(test)] -mod graph_index_test { - use crate::prelude::{AdditionOps, Graph, GraphViewOps}; - - use crate::db::graph::views::filter::model::{ - edge_filter::EdgeFilter, node_filter::NodeFilter, property_filter::ops::PropertyFilterOps, - PropertyFilterFactory, - }; - #[cfg(feature = "search")] - use crate::{ - db::graph::assertions::{search_edges, search_nodes}, - prelude::IndexMutationOps, - }; - - fn init_nodes_graph(graph: Graph) -> Graph { - graph - .add_node(1, 1, [("p1", 1), ("p2", 2)], Some("fire_nation"), None) - .unwrap(); - graph - .add_node(2, 1, [("p6", 6)], Some("fire_nation"), None) - .unwrap(); - graph - .add_node(2, 2, [("p4", 5)], Some("fire_nation"), None) - .unwrap(); - graph - .add_node(3, 3, [("p2", 4), ("p3", 3)], Some("water_tribe"), None) - .unwrap(); - graph - } - - fn init_edges_graph(graph: Graph) -> Graph { - graph - .add_edge(1, 1, 2, [("p1", 1), ("p2", 2)], None) - .unwrap(); - graph.add_edge(2, 1, 2, [("p6", 6)], None).unwrap(); - graph.add_edge(2, 2, 3, [("p4", 5)], None).unwrap(); - graph - .add_edge(3, 3, 4, [("p2", 4), ("p3", 3)], None) - .unwrap(); - graph - } - - #[test] - fn test_if_bulk_load_create_graph_index_is_ok() { - let graph = Graph::new(); - let graph = init_nodes_graph(graph); - - assert_eq!(graph.count_nodes(), 3); - - graph.create_index_in_ram().unwrap(); - } - - #[test] - fn test_if_adding_nodes_to_existing_graph_index_is_ok() { - let graph = Graph::new(); - graph.create_index_in_ram().unwrap(); - - let graph = init_nodes_graph(graph); - - assert_eq!(graph.count_nodes(), 3); - } - - #[test] - fn test_if_adding_edges_to_existing_graph_index_is_ok() { - let graph = Graph::new(); - // Creates graph index - graph.create_index_in_ram().unwrap(); - - let graph = init_edges_graph(graph); - - assert_eq!(graph.count_edges(), 3); - } - - #[test] - fn test_node_metadata_graph_index_is_ok() { - let graph = Graph::new(); - let graph = init_nodes_graph(graph); - graph.create_index_in_ram().unwrap(); - graph.node(1).unwrap().add_metadata([("x", 1u64)]).unwrap(); - - let filter = NodeFilter.metadata("x").eq(1u64); - assert_eq!(search_nodes(&graph, filter.clone()), vec!["1"]); - - graph - .node(1) - .unwrap() - .update_metadata([("x", 2u64)]) - .unwrap(); - let filter = NodeFilter.metadata("x").eq(1u64); - assert_eq!(search_nodes(&graph, filter.clone()), Vec::<&str>::new()); - - graph - .node(1) - .unwrap() - .update_metadata([("x", 2u64)]) - .unwrap(); - let filter = NodeFilter.metadata("x").eq(2u64); - assert_eq!(search_nodes(&graph, filter.clone()), vec!["1"]); - } - - #[test] - fn test_edge_metadata_graph_index_is_ok() { - let graph = Graph::new(); - let graph = init_edges_graph(graph); - graph.create_index_in_ram().unwrap(); - graph - .edge(1, 2) - .unwrap() - .add_metadata([("x", 1u64)], None) - .unwrap(); - - let filter = EdgeFilter.metadata("x").eq(1u64); - assert_eq!(search_edges(&graph, filter.clone()), vec!["1->2"]); - - graph - .edge(1, 2) - .unwrap() - .update_metadata([("x", 2u64)], None) - .unwrap(); - let filter = EdgeFilter.metadata("x").eq(1u64); - assert_eq!(search_edges(&graph, filter.clone()), Vec::<&str>::new()); - - graph - .edge(1, 2) - .unwrap() - .update_metadata([("x", 2u64)], None) - .unwrap(); - let filter = EdgeFilter.metadata("x").eq(2u64); - assert_eq!(search_edges(&graph, filter.clone()), vec!["1->2"]); - } -} diff --git a/raphtory/src/search/mod.rs b/raphtory/src/search/mod.rs index 5611df1392..1f8e1e649a 100644 --- a/raphtory/src/search/mod.rs +++ b/raphtory/src/search/mod.rs @@ -180,859 +180,4 @@ pub(crate) fn fallback_filter_exploded_edges( } #[cfg(test)] -mod test_index { - #[cfg(feature = "search")] - mod test_index_io { - use crate::{ - db::{ - api::view::{internal::InternalStorageOps, ResolvedIndexSpec}, - graph::views::filter::model::{ - node_filter::{ops::NodeFilterOps, NodeFilter}, - TryAsCompositeFilter, - }, - }, - errors::GraphError, - prelude::*, - serialise::GraphFolder, - }; - use raphtory_api::core::{ - entities::properties::prop::Prop, storage::arc_str::ArcStr, - utils::logging::global_info_logger, - }; - use tempfile::TempDir; - - fn temp_storage_path() -> std::path::PathBuf { - tempfile::tempdir().unwrap().path().to_path_buf() - } - - fn init_graph() -> Graph { - let graph = Graph::new(); - - graph - .add_node( - 1, - "Alice", - vec![("p1", Prop::U64(1000u64))], - Some("fire_nation"), - None, - ) - .unwrap(); - graph - } - - fn assert_search_results( - graph: &Graph, - filter: &T, - expected: Vec<&str>, - ) { - let res = graph - .search_nodes(filter.clone(), 2, 0) - .unwrap() - .into_iter() - .map(|n| n.name()) - .collect::>(); - assert_eq!(res, expected); - } - - #[test] - fn test_create_no_index_persist_no_index_on_encode_load_no_index_on_decode() { - // No index persisted since it was never created - let graph = init_graph(); - - let filter = NodeFilter::name().eq("Alice"); - assert_search_results(&graph, &filter, vec!["Alice"]); - - let binding = TempDir::new().unwrap(); - let path = binding.path(); - graph.encode(path).unwrap(); - - let graph = Graph::decode(path).unwrap(); - let is_indexed = graph.get_storage().unwrap().is_indexed(); - assert!(!is_indexed); - } - - #[test] - fn test_create_index_persist_index_on_encode_load_index_on_decode() { - let graph = init_graph(); - - // Created index - graph.create_index().unwrap(); - - let filter = NodeFilter::name().eq("Alice"); - assert_search_results(&graph, &filter, vec!["Alice"]); - - // Persisted both graph and index - let binding = TempDir::new().unwrap(); - let path = binding.path(); - graph.encode(path).unwrap(); - - // Loaded index that was persisted - let graph = Graph::decode(path).unwrap(); - let is_indexed = graph.get_storage().unwrap().is_indexed(); - assert!(is_indexed); - - assert_search_results(&graph, &filter, vec!["Alice"]); - } - - #[test] - fn test_encoding_graph_twice_to_same_storage_path_fails() { - let graph = init_graph(); - graph.create_index().unwrap(); - let binding = TempDir::new().unwrap(); - let path = binding.path(); - graph.encode(path).unwrap(); - let result = graph.encode(path); - - match result { - Err(GraphError::NonEmptyGraphFolder(err_path)) => { - assert_eq!(path, err_path); - } - Ok(_) => panic!("Expected error on second encode, got Ok"), - Err(e) => panic!("Unexpected error type: {:?}", e), - } - } - - #[test] - fn test_create_index_persist_index_on_encode_update_index_load_persisted_index_on_decode() { - let graph = init_graph(); - - // Created index - graph.create_index().unwrap(); - - let filter1 = NodeFilter::name().eq("Alice"); - assert_search_results(&graph, &filter1, vec!["Alice"]); - - // Persisted both graph and index - let binding = TempDir::new().unwrap(); - let path = binding.path(); - graph.encode(path).unwrap(); - - // Updated both graph and index - graph - .add_node( - 2, - "Tommy", - vec![("p1", Prop::U64(5u64))], - Some("water_tribe"), - None, - ) - .unwrap(); - let filter2 = NodeFilter::name().eq("Tommy"); - assert_search_results(&graph, &filter2, vec!["Tommy"]); - - // Loaded index that was persisted - let graph = Graph::decode(path).unwrap(); - let is_indexed = graph.get_storage().unwrap().is_indexed(); - assert!(is_indexed); - assert_search_results(&graph, &filter1, vec!["Alice"]); - assert_search_results(&graph, &filter2, Vec::<&str>::new()); - - // Updating and encode the graph and index should decode the updated the graph as well as index - // So far we have the index that was created and persisted for the first time - graph - .add_node( - 2, - "Tommy", - vec![("p1", Prop::U64(5u64))], - Some("water_tribe"), - None, - ) - .unwrap(); - let filter2 = NodeFilter::name().eq("Tommy"); - assert_search_results(&graph, &filter2, vec!["Tommy"]); - - // Should persist the updated graph and index - let binding = TempDir::new().unwrap(); - let path = binding.path(); - graph.encode(path).unwrap(); - - // Should load the updated graph and index - let storage_path = path.parent().unwrap().to_path_buf(); - let graph = Graph::decode(path).unwrap(); - let is_indexed = graph.get_storage().unwrap().is_indexed(); - assert!(is_indexed); - assert_search_results(&graph, &filter1, vec!["Alice"]); - assert_search_results(&graph, &filter2, vec!["Tommy"]); - } - - #[test] - fn test_zip_encode_decode_index() { - let graph = init_graph(); - graph.create_index().unwrap(); - let tmp_dir = TempDir::new().unwrap(); - let zip_path = tmp_dir.path().join("graph.zip"); - let folder = GraphFolder::new_as_zip(zip_path); - graph.encode(&folder).unwrap(); - - let storage_path = tmp_dir.path().to_path_buf(); - let graph = Graph::decode(&folder).unwrap(); - let node = graph.node("Alice").unwrap(); - let node_type = node.node_type(); - assert_eq!(node_type, Some(ArcStr::from("fire_nation"))); - - let filter = NodeFilter::name().eq("Alice"); - assert_search_results(&graph, &filter, vec!["Alice"]); - } - - #[test] - fn test_encoding_graph_twice_to_same_storage_path_fails_zip() { - let graph = init_graph(); - graph.create_index().unwrap(); - let tmp_dir = TempDir::new().unwrap(); - let zip_path = tmp_dir.path().join("graph.zip"); - let folder = GraphFolder::new_as_zip(&zip_path); - graph.encode(&folder).unwrap(); - graph - .add_node(1, "Ozai", [("prop", 1)], Some("fire_nation"), None) - .unwrap(); - let result = graph.encode(folder); - match result { - Err(GraphError::IOError { source, .. }) => { - assert!( - format!("{source}").to_lowercase().contains("file exists"), - "{}", - source - ); - } - Ok(_) => panic!("Expected error on second encode, got Ok"), - Err(e) => panic!("Unexpected error type: {:?}", e), - } - } - - #[test] - fn test_immutable_graph_index_persistence() { - let graph = init_graph(); - graph.create_index().unwrap(); - - let binding = TempDir::new().unwrap(); - let path = binding.path(); - graph.encode(path).unwrap(); - - // This gives us immutable index - let graph = Graph::decode(path).unwrap(); - - // This tests that we are able to persist the immutable index - let binding = TempDir::new().unwrap(); - let path = binding.path(); - graph.encode(path).unwrap(); - - let graph = Graph::decode(path).unwrap(); - let filter1 = NodeFilter::name().eq("Alice"); - assert_search_results(&graph, &filter1, vec!["Alice"]); - } - - #[test] - fn test_mutable_graph_index_persistence() { - let graph = init_graph(); - graph.create_index().unwrap(); - - let binding = TempDir::new().unwrap(); - let path = binding.path(); - graph.encode(path).unwrap(); - - // This gives us immutable index - let graph = Graph::decode(path).unwrap(); - - // This converts immutable index to mutable index - graph - .add_node(1, "Ozai", [("prop", 1)], Some("fire_nation"), None) - .unwrap(); - - // This tests that we are able to persist the mutable index - let binding = TempDir::new().unwrap(); - let path = binding.path(); - graph.encode(path).unwrap(); - - let graph = Graph::decode(path).unwrap(); - let filter = NodeFilter::name().eq("Ozai"); - assert_search_results(&graph, &filter, vec!["Ozai"]); - } - - #[test] - fn test_loading_zip_index_creates_mutable_index() { - let graph = init_graph(); - graph.create_index().unwrap(); - let tmp_dir = TempDir::new().unwrap(); - let zip_path = tmp_dir.path().join("graph.zip"); - let folder = GraphFolder::new_as_zip(&zip_path); - graph.encode(&folder).unwrap(); - - let graph = Graph::decode(&folder).unwrap(); - let immutable = graph - .get_storage() - .unwrap() - .index - .read_recursive() - .is_immutable(); - assert! {!immutable}; - } - - #[test] - fn test_loading_index_creates_immutable_index() { - let graph = init_graph(); - graph.create_index().unwrap(); - let binding = TempDir::new().unwrap(); - let path = binding.path(); - graph.encode(path).unwrap(); - - let graph = Graph::decode(path).unwrap(); - let immutable = graph - .get_storage() - .unwrap() - .index - .read_recursive() - .is_immutable(); - assert! {immutable}; - } - - #[test] - fn test_create_index_in_ram() { - global_info_logger(); - - let graph = init_graph(); - graph.create_index_in_ram().unwrap(); - - let filter = NodeFilter::name().eq("Alice"); - assert_search_results(&graph, &filter, vec!["Alice"]); - - let binding = TempDir::new().unwrap(); - let path = binding.path(); - graph.encode(path).unwrap(); - - let graph = Graph::decode(path).unwrap(); - let is_indexed = graph.get_storage().unwrap().is_indexed(); - assert!(!is_indexed); - - assert_search_results(&graph, &filter, vec!["Alice"]); - } - - #[test] - #[ignore] - fn test_too_many_open_files_graph_index() { - use TempDir; - - let mut graphs = vec![]; - - for i in 0..1000 { - let graph = init_graph(); - if let Err(e) = graph.create_index() { - match &e { - GraphError::IndexError { source } => { - panic!("Hit file descriptor limit after {} graphs. {:?}", 0, source); - } - other => { - panic!("Unexpected GraphError: {:?}", other); - } - } - } - graphs.push(graph); - } - } - - #[test] - fn test_graph_index_creation_with_too_many_properties() { - let graph = init_graph(); - let props: Vec<(String, Prop)> = (1..=100) - .map(|i| (format!("p{i}"), Prop::U64(i as u64))) - .collect(); - graph.node("Alice").unwrap().add_metadata(props).unwrap(); - - if let Err(e) = graph.create_index() { - match &e { - GraphError::IndexError { source } => { - panic!("Hit file descriptor limit after {} graphs. {:?}", 0, source); - } - other => { - panic!("Unexpected GraphError: {:?}", other); - } - } - } - } - - #[test] - // No new const prop index created because when index were created - // these properties did not exist. - fn test_graph_index_creation_for_incremental_node_update_no_new_prop_indexed() { - let graph = init_graph(); - graph.create_index().unwrap(); - let props: Vec<(String, Prop)> = (1..=100) - .map(|i| (format!("p{i}"), Prop::U64(i as u64))) - .collect(); - graph.node("Alice").unwrap().add_metadata(props).unwrap(); - - let tmp_dir = TempDir::new().unwrap(); - let path = tmp_dir.path().to_path_buf(); - graph.encode(&path).unwrap(); - let graph = Graph::decode(&path).unwrap(); - - let spec = graph.get_index_spec().unwrap().props(&graph); - assert_eq!( - spec, - ResolvedIndexSpec { - node_properties: vec!["p1".to_string()], - node_metadata: vec![], - edge_metadata: vec![], - edge_properties: vec![] - } - ); - } - } - - mod test_index_spec { - #[cfg(feature = "search")] - use crate::prelude::SearchableGraphOps; - use crate::{ - db::{ - api::view::{IndexSpec, IndexSpecBuilder}, - graph::{ - assertions::{search_edges, search_nodes}, - views::filter::model::{ - edge_filter::EdgeFilter, node_filter::NodeFilter, - property_filter::ops::PropertyFilterOps, ComposableFilter, - PropertyFilterFactory, TemporalPropertyFilterFactory, - }, - }, - }, - errors::GraphError, - prelude::{AdditionOps, Graph, IndexMutationOps, StableDecode}, - serialise::{GraphFolder, StableEncode}, - }; - use tempfile::{tempdir, TempDir}; - - fn temp_storage_path() -> std::path::PathBuf { - tempfile::tempdir().unwrap().path().to_path_buf() - } - - fn init_graph() -> Graph { - let graph = Graph::new(); - - let nodes = vec![ - ( - 1, - "pometry", - [("p1", 5u64), ("p2", 50u64)], - Some("fire_nation"), - [("x", true)], - ), - ( - 1, - "raphtory", - [("p1", 10u64), ("p2", 100u64)], - Some("water_tribe"), - [("y", false)], - ), - ]; - - for (time, name, props, group, metadata) in nodes { - let node = graph.add_node(time, name, props, group, None).unwrap(); - node.add_metadata(metadata).unwrap(); - } - - let edges = vec![ - ( - 1, - "pometry", - "raphtory", - [("e_p1", 3.2f64), ("e_p2", 10f64)], - None, - [("e_x", true)], - ), - ( - 1, - "raphtory", - "pometry", - [("e_p1", 4.0f64), ("e_p2", 20f64)], - None, - [("e_y", false)], - ), - ]; - - for (time, src, dst, props, label, metadata) in edges { - let edge = graph.add_edge(time, src, dst, props, label).unwrap(); - edge.add_metadata(metadata, label).unwrap(); - } - - graph - } - - #[test] - #[ignore = "TODO: #2372"] - fn test_with_all_props_index_spec() { - let graph = init_graph(); - let index_spec = IndexSpecBuilder::new(graph.clone()) - .with_all_node_properties_and_metadata() - .with_all_edge_properties_and_metadata() - .build(); - assert_eq!( - index_spec.props(&graph).to_vec(), - vec![ - vec!["x", "y"], - vec!["p1", "p2"], - vec!["e_x", "e_y"], - vec!["e_p1", "e_p2"] - ] - ); - graph.create_index_in_ram_with_spec(index_spec).unwrap(); - - let filter = NodeFilter - .property("p1") - .eq(5u64) - .and(NodeFilter.metadata("x").eq(true)); - let results = search_nodes(&graph, filter); - assert_eq!(results, vec!["pometry"]); - - let filter = EdgeFilter - .property("e_p1") - .lt(5f64) - .and(EdgeFilter.metadata("e_y").eq(false)); - let results = search_edges(&graph, filter); - assert_eq!(results, vec!["raphtory->pometry"]); - } - - #[test] - #[ignore = "TODO: #2372"] - fn test_with_selected_props_index_spec() { - let graph = init_graph(); - let index_spec = IndexSpecBuilder::new(graph.clone()) - .with_node_metadata(vec!["y"]) - .unwrap() - .with_node_properties(vec!["p1"]) - .unwrap() - .with_edge_metadata(vec!["e_y"]) - .unwrap() - .with_edge_properties(vec!["e_p1"]) - .unwrap() - .build(); - assert_eq!( - index_spec.props(&graph).to_vec(), - vec![vec!["y"], vec!["p1"], vec!["e_y"], vec!["e_p1"]] - ); - graph.create_index_in_ram_with_spec(index_spec).unwrap(); - - let filter = NodeFilter - .property("p1") - .eq(5u64) - .or(NodeFilter.metadata("y").eq(false)); - let results = search_nodes(&graph, filter); - assert_eq!(results, vec!["pometry", "raphtory"]); - - let filter = NodeFilter.metadata("y").eq(false); - let results = search_nodes(&graph, filter); - assert_eq!(results, vec!["raphtory"]); - - let filter = EdgeFilter - .property("e_p1") - .lt(5f64) - .or(EdgeFilter.metadata("e_y").eq(false)); - let results = search_edges(&graph, filter); - assert_eq!(results, vec!["pometry->raphtory", "raphtory->pometry"]); - } - - #[test] - fn test_with_invalid_property_returns_error() { - let graph = init_graph(); - let result = IndexSpecBuilder::new(graph.clone()).with_node_metadata(["xyz"]); - - assert!(matches!(result, Err(GraphError::PropertyMissingError(p)) if p == "xyz")); - } - - #[test] - fn test_build_empty_spec_by_default() { - let graph = init_graph(); - let index_spec = IndexSpecBuilder::new(graph.clone()).build(); - - assert!(index_spec.node_metadata.is_empty()); - assert!(index_spec.node_properties.is_empty()); - assert!(index_spec.edge_metadata.is_empty()); - assert!(index_spec.edge_properties.is_empty()); - - graph.create_index_in_ram_with_spec(index_spec).unwrap(); - - let filter = NodeFilter - .property("p1") - .eq(5u64) - .and(NodeFilter.metadata("x").eq(true)); - let results = search_nodes(&graph, filter); - assert_eq!(results, vec!["pometry"]); - - let filter = EdgeFilter - .property("e_p1") - .lt(5f64) - .or(EdgeFilter.metadata("e_y").eq(false)); - let results = search_edges(&graph, filter); - assert_eq!(results, vec!["pometry->raphtory", "raphtory->pometry"]); - } - - #[test] - #[ignore = "TODO: #2372"] - fn test_mixed_node_and_edge_props_index_spec() { - let graph = init_graph(); - - let index_spec = IndexSpecBuilder::new(graph.clone()) - .with_node_metadata(vec!["x"]) - .unwrap() - .with_all_node_properties() - .with_all_edge_properties_and_metadata() - .build(); - assert_eq!( - index_spec.props(&graph).to_vec(), - vec![ - vec!["x"], - vec!["p1", "p2"], - vec!["e_x", "e_y"], - vec!["e_p1", "e_p2"] - ] - ); - - graph.create_index_in_ram_with_spec(index_spec).unwrap(); - - let filter = NodeFilter - .property("p1") - .eq(5u64) - .or(NodeFilter.metadata("y").eq(false)); - let results = search_nodes(&graph, filter); - assert_eq!(results, vec!["pometry", "raphtory"]); - - let filter = EdgeFilter - .property("e_p1") - .lt(5f64) - .or(EdgeFilter.metadata("e_y").eq(false)); - let results = search_edges(&graph, filter); - assert_eq!(results, vec!["pometry->raphtory", "raphtory->pometry"]); - } - - #[test] - fn test_get_index_spec_newly_created_index() { - let graph = init_graph(); - - let index_spec = IndexSpecBuilder::new(graph.clone()) - .with_node_metadata(vec!["x"]) - .unwrap() - .with_all_node_properties() - .with_all_edge_properties_and_metadata() - .build(); - - graph - .create_index_in_ram_with_spec(index_spec.clone()) - .unwrap(); - - assert_eq!(index_spec, graph.get_index_spec().unwrap()); - } - - #[test] - #[ignore = "TODO: #2372"] - fn test_get_index_spec_updated_index() { - let graph = init_graph(); - - let index_spec = IndexSpecBuilder::new(graph.clone()) - .with_edge_metadata(vec!["e_y"]) - .unwrap() - .build(); - graph.create_index_with_spec(index_spec.clone()).unwrap(); - - assert_eq!(index_spec, graph.get_index_spec().unwrap()); - let results = search_nodes(&graph, NodeFilter.metadata("y").eq(false)); - assert_eq!(results, vec!["raphtory"]); - let results = search_edges(&graph, EdgeFilter.metadata("e_y").eq(false)); - assert_eq!(results, vec!["raphtory->pometry"]); - - let index_spec = IndexSpecBuilder::new(graph.clone()) - .with_node_metadata(vec!["y"]) - .unwrap() - .with_node_properties(vec!["p2"]) - .unwrap() - .with_edge_metadata(vec!["e_y"]) - .unwrap() - .build(); - graph.create_index_with_spec(index_spec.clone()).unwrap(); - - assert_eq!(index_spec, graph.get_index_spec().unwrap()); - let results = search_nodes(&graph, NodeFilter.metadata("y").eq(false)); - assert_eq!(results, vec!["raphtory"]); - let results = search_edges(&graph, EdgeFilter.metadata("e_y").eq(false)); - assert_eq!(results, vec!["raphtory->pometry"]); - } - - #[test] - #[ignore = "TODO: #2372"] - fn test_get_index_spec_updated_index_persisted_and_loaded() { - let graph = init_graph(); - - let index_spec = IndexSpecBuilder::new(graph.clone()) - .with_edge_metadata(vec!["e_y"]) - .unwrap() - .build(); - graph.create_index_with_spec(index_spec.clone()).unwrap(); - - let tmp_graph_dir = tempdir().unwrap(); - let path = tmp_graph_dir.path().to_path_buf(); - graph.encode(&path).unwrap(); - let graph = Graph::decode(&path).unwrap(); - - assert_eq!(index_spec, graph.get_index_spec().unwrap()); - let results = search_nodes(&graph, NodeFilter.metadata("y").eq(false)); - assert_eq!(results, vec!["raphtory"]); - let results = search_edges(&graph, EdgeFilter.metadata("e_y").eq(false)); - assert_eq!(results, vec!["raphtory->pometry"]); - - let index_spec = IndexSpecBuilder::new(graph.clone()) - .with_node_metadata(vec!["y"]) - .unwrap() - .with_node_properties(vec!["p2"]) - .unwrap() - .with_edge_metadata(vec!["e_y"]) - .unwrap() - .build(); - graph.create_index_with_spec(index_spec.clone()).unwrap(); - let tmp_graph_dir = tempdir().unwrap(); - let path = tmp_graph_dir.path().to_path_buf(); - graph.encode(path.clone()).unwrap(); - let graph = Graph::decode(&path).unwrap(); - - assert_eq!(index_spec, graph.get_index_spec().unwrap()); - let results = search_nodes(&graph, NodeFilter.metadata("y").eq(false)); - assert_eq!(results, vec!["raphtory"]); - let results = search_edges(&graph, EdgeFilter.metadata("e_y").eq(false)); - assert_eq!(results, vec!["raphtory->pometry"]); - } - - #[test] - fn test_get_index_spec_loaded_index() { - let graph = init_graph(); - - let index_spec = IndexSpecBuilder::new(graph.clone()) - .with_node_metadata(vec!["y"]) - .unwrap() - .with_node_properties(vec!["p2"]) - .unwrap() - .with_edge_metadata(vec!["e_y"]) - .unwrap() - .with_edge_properties(vec!["e_p2"]) - .unwrap() - .build(); - - graph.create_index_with_spec(index_spec.clone()).unwrap(); - let tmp_graph_dir = tempdir().unwrap(); - let path = tmp_graph_dir.path().to_path_buf(); - graph.encode(path.clone()).unwrap(); - - let graph = Graph::decode(&path).unwrap(); - let index_spec2 = graph.get_index_spec().unwrap(); - - assert_eq!(index_spec, index_spec2); - } - - #[test] - fn test_get_index_spec_loaded_index_zip() { - let graph = init_graph(); - - let index_spec = IndexSpecBuilder::new(graph.clone()) - .with_node_metadata(vec!["y"]) - .unwrap() - .with_node_properties(vec!["p2"]) - .unwrap() - .with_edge_metadata(vec!["e_y"]) - .unwrap() - .build(); - graph.create_index_with_spec(index_spec.clone()).unwrap(); - - let binding = TempDir::new().unwrap(); - let path = binding.path(); - let folder = GraphFolder::new_as_zip(path); - graph.encode(folder).unwrap(); - - let graph = Graph::decode(path).unwrap(); - assert_eq!(index_spec, graph.get_index_spec().unwrap()); - } - - #[test] - fn test_no_new_node_prop_index_created_via_update_apis() { - run_node_index_test(|graph, index_spec| { - graph.create_index_with_spec(index_spec.clone()) - }); - - run_node_index_test(|graph, index_spec| { - graph.create_index_in_ram_with_spec(index_spec.clone()) - }); - } - - #[test] - fn test_no_new_edge_prop_index_created_via_update_apis() { - run_edge_index_test(|graph, index_spec| { - graph.create_index_with_spec(index_spec.clone()) - }); - - run_edge_index_test(|graph, index_spec| { - graph.create_index_in_ram_with_spec(index_spec.clone()) - }); - } - - fn run_node_index_test(create_index_fn: F) - where - F: Fn(&Graph, IndexSpec) -> Result<(), GraphError>, - { - let graph = init_graph(); - - let index_spec = IndexSpecBuilder::new(graph.clone()) - .with_node_metadata(vec!["y"]) - .unwrap() - .with_node_properties(vec!["p1"]) - .unwrap() - .build(); - create_index_fn(&graph, index_spec.clone()).unwrap(); - - let filter = NodeFilter.property("p2").temporal().last().eq(50u64); - assert_eq!(search_nodes(&graph, filter.clone()), vec!["pometry"]); - - let node = graph - .add_node(1, "shivam", [("p1", 100u64)], Some("fire_nation"), None) - .unwrap(); - assert_eq!(index_spec, graph.get_index_spec().unwrap()); - - let filter = NodeFilter.property("p1").temporal().last().eq(100u64); - assert_eq!(search_nodes(&graph, filter.clone()), vec!["shivam"]); - - node.add_metadata([("z", true)]).unwrap(); - assert_eq!(index_spec, graph.get_index_spec().unwrap()); - let filter = NodeFilter.metadata("z").eq(true); - assert_eq!(search_nodes(&graph, filter.clone()), vec!["shivam"]); - - node.update_metadata([("z", false)]).unwrap(); - assert_eq!(index_spec, graph.get_index_spec().unwrap()); - let filter = NodeFilter.metadata("z").eq(false); - assert_eq!(search_nodes(&graph, filter.clone()), vec!["shivam"]); - } - - fn run_edge_index_test(create_index_fn: F) - where - F: Fn(&Graph, IndexSpec) -> Result<(), GraphError>, - { - let graph = init_graph(); - - let index_spec = IndexSpecBuilder::new(graph.clone()) - .with_node_metadata(vec!["y"]) - .unwrap() - .with_node_properties(vec!["p2"]) - .unwrap() - .build(); - create_index_fn(&graph, index_spec.clone()).unwrap(); - - let edge = graph - .add_edge(1, "shivam", "kapoor", [("p1", 100u64)], None) - .unwrap(); - assert_eq!(index_spec, graph.get_index_spec().unwrap()); - let filter = EdgeFilter.property("p1").temporal().last().eq(100u64); - assert_eq!(search_edges(&graph, filter.clone()), vec!["shivam->kapoor"]); - - edge.add_metadata([("z", true)], None).unwrap(); - assert_eq!(index_spec, graph.get_index_spec().unwrap()); - let filter = EdgeFilter.metadata("z").eq(true); - assert_eq!(search_edges(&graph, filter.clone()), vec!["shivam->kapoor"]); - - edge.update_metadata([("z", false)], None).unwrap(); - assert_eq!(index_spec, graph.get_index_spec().unwrap()); - let filter = EdgeFilter.metadata("z").eq(false); - assert_eq!(search_edges(&graph, filter.clone()), vec!["shivam->kapoor"]); - } - } -} +mod test_index {} diff --git a/raphtory/src/search/node_index.rs b/raphtory/src/search/node_index.rs index 986fdd77b2..4986ee8a13 100644 --- a/raphtory/src/search/node_index.rs +++ b/raphtory/src/search/node_index.rs @@ -10,7 +10,7 @@ use crate::{ }, }; use ahash::HashSet; -use raphtory_storage::graph::graph::GraphStorage; +use raphtory_storage::graph::{graph::GraphStorage, nodes::node_storage_ops::NodeStorageOps}; use rayon::{iter::IntoParallelIterator, prelude::ParallelIterator}; use std::{ fmt::{Debug, Formatter}, @@ -222,13 +222,11 @@ impl NodeIndex { // Index nodes fields let mut writer = self.entity_index.index.writer(100_000_000)?; - (0..graph.count_nodes()) - .into_par_iter() - .try_for_each(|v_id| { - let node = NodeView::new_internal(graph, VID(v_id)); - self.index_node(node, &writer)?; - Ok::<(), GraphError>(()) - })?; + graph.nodes().par_iter().try_for_each(|node| { + let node = NodeView::new_internal(graph, node.vid()); + self.index_node(node, &writer)?; + Ok::<(), GraphError>(()) + })?; writer.commit()?; Ok(()) diff --git a/raphtory/tests/history_filter.rs b/raphtory/tests/history_filter.rs deleted file mode 100644 index 8b13789179..0000000000 --- a/raphtory/tests/history_filter.rs +++ /dev/null @@ -1 +0,0 @@ -