diff --git a/src/paths_serialization.rs b/src/paths_serialization.rs index 088fc50..6ab63eb 100644 --- a/src/paths_serialization.rs +++ b/src/paths_serialization.rs @@ -5,7 +5,6 @@ //! `.paths` data does not contain values, so the `_auxdata` functions allow values to //! be associated with path indices. -// GOAT both functions should be tested on long paths (larger than chunk size) use libz_ng_sys::*; use crate::PathMap; use crate::TrieValue; @@ -392,4 +391,39 @@ mod test { Err(e) => { println!("ser e {}", e) } } } + + #[cfg(not(miri))] // miri really hates the zlib-ng-sys C API + #[test] + fn path_serialize_deserialize_long_paths_cross_chunk_boundary() { + // Paths chosen around the internal 4096-byte compression CHUNK: one just + // over half, one just past one chunk, one spanning several. Deterministic + // xorshift contents so failures reproduce. + let mut state = 0x9e3779b97f4a7c15_u64; + let mut next = move || { state ^= state << 13; state ^= state >> 7; state ^= state << 17; state }; + let lengths = [7usize, 2049, 4097, 24000]; + let mut btm = PathMap::new(); + let mut paths = vec![]; + for &len in &lengths { + let path: Vec = (0..len).map(|_| (next() >> 33) as u8).collect(); + btm.set_val_at(&path[..], ()); + paths.push(path); + } + + let mut v = vec![]; + let ser = serialize_paths(btm.read_zipper(), &mut v).unwrap(); + assert_eq!(ser.path_count, lengths.len()); + assert!(ser.bytes_in > 4096, "the input must span the compression chunk size"); + + let mut restored = PathMap::new(); + let de = deserialize_paths(restored.write_zipper(), v.as_slice(), ()).unwrap(); + assert_eq!(de.path_count, lengths.len()); + + for path in &paths { + assert!(restored.contains(&path[..]), "path of len {} lost in round-trip", path.len()); + } + let mut rz = restored.read_zipper(); + let mut restored_count = 0; + while rz.to_next_val() { restored_count += 1; } + assert_eq!(restored_count, lengths.len(), "no extra paths may appear"); + } } diff --git a/src/ring.rs b/src/ring.rs index cb3370a..c5fee27 100644 --- a/src/ring.rs +++ b/src/ring.rs @@ -1193,9 +1193,201 @@ set_dist_lattice!(HashSet); #[cfg(test)] mod tests { - use std::collections::{HashSet, HashMap}; - use crate::ring::Lattice; - use super::{AlgebraicResult, SetLattice, SELF_IDENT, COUNTER_IDENT}; + use super::{AlgebraicResult, AlgebraicStatus, SetLattice, COUNTER_IDENT, SELF_IDENT}; + use crate::ring::{DistributiveLattice, Lattice}; + use std::collections::{HashMap, HashSet}; + use std::fmt::Debug; + + type NestedSetMap = HashMap>; + + fn assert_binary_result( + result: AlgebraicResult, + self_value: &T, + counter_value: &T, + expected: &T, + allow_counter_identity: bool, + context: &str, + ) where + T: Clone + Default + Eq + Debug, + { + match &result { + AlgebraicResult::None => { + assert_eq!(expected, &T::default(), "{context}: None result"); + } + AlgebraicResult::Identity(mask) => { + assert_ne!(*mask, 0, "{context}: zero identity mask"); + assert_eq!( + *mask & !(SELF_IDENT | COUNTER_IDENT), + 0, + "{context}: identity mask sets an out-of-arity bit" + ); + if !allow_counter_identity { + assert_eq!( + *mask & COUNTER_IDENT, + 0, + "{context}: non-commutative operation returned counter identity" + ); + } + if *mask & SELF_IDENT != 0 { + assert_eq!(self_value, expected, "{context}: self identity mismatch"); + } + if *mask & COUNTER_IDENT != 0 { + assert_eq!( + counter_value, expected, + "{context}: counter identity mismatch" + ); + } + } + AlgebraicResult::Element(_) => {} + } + + let actual = result.unwrap_or([self_value, counter_value], T::default()); + assert_eq!(actual, *expected, "{context}: materialized result"); + } + + fn normalize_nested_map(map: &NestedSetMap) -> NestedSetMap { + map.iter() + .filter(|(_, values)| !values.is_empty()) + .map(|(key, values)| (*key, values.clone())) + .collect() + } + + fn assert_nested_result( + result: AlgebraicResult, + self_value: &NestedSetMap, + counter_value: &NestedSetMap, + expected: &NestedSetMap, + allow_counter_identity: bool, + context: &str, + ) { + match &result { + AlgebraicResult::None => { + assert!(expected.is_empty(), "{context}: None result"); + } + AlgebraicResult::Identity(mask) => { + assert_ne!(*mask, 0, "{context}: zero identity mask"); + assert_eq!( + *mask & !(SELF_IDENT | COUNTER_IDENT), + 0, + "{context}: identity mask sets an out-of-arity bit" + ); + if !allow_counter_identity { + assert_eq!( + *mask & COUNTER_IDENT, + 0, + "{context}: non-commutative operation returned counter identity" + ); + } + if *mask & SELF_IDENT != 0 { + assert_eq!( + normalize_nested_map(self_value), + *expected, + "{context}: self identity mismatch" + ); + } + if *mask & COUNTER_IDENT != 0 { + assert_eq!( + normalize_nested_map(counter_value), + *expected, + "{context}: counter identity mismatch" + ); + } + } + AlgebraicResult::Element(_) => {} + } + + let actual = result.unwrap_or([self_value, counter_value], NestedSetMap::new()); + assert_eq!( + normalize_nested_map(&actual), + *expected, + "{context}: materialized result" + ); + } + + fn mixed(seed: u64) -> u64 { + let mut x = seed.wrapping_add(0x9e37_79b9_7f4a_7c15); + x = (x ^ (x >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + x = (x ^ (x >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + x ^ (x >> 31) + } + + fn generated_set(seed: u64, salt: u64) -> HashSet { + let mut set = HashSet::new(); + for value in 0..48 { + if mixed(seed ^ salt ^ ((value as u64) << 32)) % 5 < 2 { + set.insert(value); + } + } + set + } + + fn generated_nested_map(seed: u64, salt: u64) -> NestedSetMap { + let mut map = NestedSetMap::new(); + for key in 0..8 { + let key_seed = mixed(seed ^ salt ^ ((key as u64) << 24)); + if key_seed % 4 == 0 { + continue; + } + + let mut values = HashSet::new(); + for value in 0..16 { + if mixed(key_seed ^ ((value as u64) << 32)) % 5 < 2 { + values.insert(value); + } + } + if key_seed % 31 == 0 { + values.clear(); + } + map.insert(key, values); + } + map + } + + fn nested_join(a: &NestedSetMap, b: &NestedSetMap) -> NestedSetMap { + let mut result = normalize_nested_map(a); + for (key, values) in b { + if values.is_empty() { + continue; + } + result + .entry(*key) + .or_default() + .extend(values.iter().copied()); + } + result + } + + fn nested_meet(a: &NestedSetMap, b: &NestedSetMap) -> NestedSetMap { + let mut result = NestedSetMap::new(); + for (key, a_values) in a { + let Some(b_values) = b.get(key) else { + continue; + }; + let values = a_values + .intersection(b_values) + .copied() + .collect::>(); + if !values.is_empty() { + result.insert(*key, values); + } + } + result + } + + fn nested_subtract(a: &NestedSetMap, b: &NestedSetMap) -> NestedSetMap { + let mut result = NestedSetMap::new(); + for (key, a_values) in a { + let values = if let Some(b_values) = b.get(key) { + a_values.difference(b_values).copied().collect() + } else { + a_values.clone() + }; + if !values.is_empty() { + result.insert(*key, values); + } + } + result + } #[test] fn set_lattice_join_test1() { @@ -1298,6 +1490,105 @@ mod tests { assert_eq!(meet_result, AlgebraicResult::Identity(SELF_IDENT | COUNTER_IDENT)); } + #[test] + fn seeded_hash_set_operations_match_set_oracle() { + for seed in 0..256 { + let a = generated_set(seed, 0x243f_6a88_85a3_08d3); + let b = generated_set(seed, 0x1319_8a2e_0370_7344); + + let expected_join = a.union(&b).copied().collect::>(); + assert_binary_result( + a.pjoin(&b), + &a, + &b, + &expected_join, + true, + &format!("HashSet join seed {seed}"), + ); + let mut join_in_place = a.clone(); + join_in_place.join_into(b.clone()); + assert_eq!( + join_in_place, expected_join, + "HashSet join_into seed {seed}" + ); + + let expected_meet = a.intersection(&b).copied().collect::>(); + assert_binary_result( + a.pmeet(&b), + &a, + &b, + &expected_meet, + true, + &format!("HashSet meet seed {seed}"), + ); + let expected_subtract = a.difference(&b).copied().collect::>(); + assert_binary_result( + a.psubtract(&b), + &a, + &b, + &expected_subtract, + false, + &format!("HashSet subtract seed {seed}"), + ); + } + } + + #[test] + fn seeded_hash_map_operations_match_nested_set_oracle() { + for seed in 0..256 { + let a = generated_nested_map(seed, 0x243f_6a88_85a3_08d3); + let b = generated_nested_map(seed, 0x1319_8a2e_0370_7344); + let c = generated_nested_map(seed, 0xa409_3822_299f_31d0); + + let expected_join = nested_join(&a, &b); + assert_nested_result( + a.pjoin(&b), + &a, + &b, + &expected_join, + true, + &format!("HashMap join seed {seed}"), + ); + let mut join_in_place = a.clone(); + join_in_place.join_into(b.clone()); + assert_eq!( + normalize_nested_map(&join_in_place), + expected_join, + "HashMap join_into seed {seed}" + ); + + let expected_meet = nested_meet(&a, &b); + assert_nested_result( + a.pmeet(&b), + &a, + &b, + &expected_meet, + true, + &format!("HashMap meet seed {seed}"), + ); + let expected_subtract = nested_subtract(&a, &b); + assert_nested_result( + a.psubtract(&b), + &a, + &b, + &expected_subtract, + false, + &format!("HashMap subtract seed {seed}"), + ); + + let ab_join = a.pjoin(&b).unwrap_or([&a, &b], NestedSetMap::new()); + let expected_chain = nested_subtract(&nested_join(&a, &b), &c); + assert_nested_result( + ab_join.psubtract(&c), + &ab_join, + &c, + &expected_chain, + false, + &format!("HashMap chained join/subtract seed {seed}"), + ); + } + } + /// Used in [set_lattice_join_test2] and [set_lattice_meet_test2] #[derive(Clone, Debug)] struct Map<'a>(HashMap::<&'a str, HashMap<&'a str, ()>>);// TODO, should be struct Map<'a>(HashMap::<&'a str, Map<'a>>); see comment above about chalk @@ -1332,7 +1623,8 @@ mod tests { inner_map_1.insert("1", ()); a.0.insert("A", inner_map_1.clone()); b.0.insert("B", inner_map_1); - // b.0.insert("C", HashMap::new()); TODO: We might want to test collapse of empty items using the is_bottom() method + a.0.insert("C", HashMap::new()); + b.0.insert("C", HashMap::new()); let joined_result = a.pjoin(&b); assert!(joined_result.is_element()); let joined = joined_result.unwrap([&a, &b]); @@ -1340,6 +1632,8 @@ mod tests { assert!(joined.get(&"A").is_some()); assert!(joined.get(&"B").is_some()); assert!(joined.get(&"C").is_none()); //Empty sub-sets should not be merged + a.0.remove("C"); + b.0.remove("C"); // Two level join, results should be Element even though the key existed in both args, because the values joined let mut inner_map_2 = HashMap::with_capacity(1); @@ -1419,8 +1713,6 @@ mod tests { assert_eq!(meet_result.identity_mask().unwrap(), COUNTER_IDENT); } } - -//GOAT, do a test for the HashMap impl of psubtract //GOAT, do an impl of SetLattice for Vec as an indexed set @@ -1428,20 +1720,3 @@ mod tests { // BitfieldLattice should be implemented on bool // Make monad types that can implement these traits on all prim types // Make a "convertable_to" trait across all prim types - -// GOAT, TEST TODO: A fuzz test for some of the algebraic operations (join, meet, subtract) across all -// different path configurations and operation orderings. - -// Envisioned Implementation: -// 1. Create `set_a` of N values (e.g. integers 0..100) and assign a pseudorandom path to each element -// 2. Create `set_b` of M values and assign a different pseudorandom path to each element -// 3. Compose pseudorandom subsets of `set_a` and `set_b` and put them into HashSets -// 4. Put corresponding concatenated paths (Cartesian product) into PathMaps. -// 5. Select an operation to perform, and do the same operation to both the HashSets contining simple -// indices and to the PathMaps. And validate the results match -// 6. Loop back to 3, continuing to choose additional subsets to perform additional operations. - -// The reason behind the cartesian product (concatenated paths) is because the chances of getting overlap -// beyond the first couple bytes of a random path are very slim. So the Cartesian product appraoch -// means we are likely to get large common prefixes followed by splits deep in the trie, which will -// exercise the code more thoroughly. diff --git a/src/trie_map.rs b/src/trie_map.rs index 21a679c..15b9c6f 100644 --- a/src/trie_map.rs +++ b/src/trie_map.rs @@ -123,6 +123,25 @@ impl PathMap { } impl PathMap { + /// Whether `self` and `other` share the same root node by pointer (O(1)). + /// PathMap is copy-on-write, so a map and a clone of it stay `same_trie` + /// until one is mutated: any insert or removal replaces the root node with a + /// fresh allocation. A `true` result therefore proves the trie's node + /// structure has not changed relative to `other`. Because `other` (typically + /// a cheap COW clone kept as a snapshot) holds the old root alive, its + /// address cannot be reused while the snapshot exists, so this is free of the + /// ABA pointer-reuse hazard. Root VALUES (the empty-key entry) are not + /// compared; callers that key a cache on subtree contents do not touch it. + #[inline] + pub fn same_trie(&self, other: &Self) -> bool { + let a = unsafe { &*self.root.get() }; + let b = unsafe { &*other.root.get() }; + match (a, b) { + (None, None) => true, + (Some(x), Some(y)) => x.ptr_eq(y), + _ => false, + } + } #[inline] pub(crate) fn root(&self) -> Option<&TrieNodeODRc> { unsafe{ &*self.root.get() }.as_ref() @@ -1454,3 +1473,30 @@ mod tests { //GOAT, Consider refactor of zipper traits. `WriteZipper` -> `PathWriter`. Zipper is split into the zipper // movement traits and a `PathReader` trait. Then `PathWriter` and `PathReader` can both be implemented on // the map, and we can get rid of duplicate methods like `graft_map` + +#[cfg(test)] +mod same_trie_tests { + use crate::PathMap; + #[test] + fn same_trie_tracks_cow_mutation() { + let mut m: PathMap<()> = PathMap::new(); + m.insert(b"(edge a b)", ()); + let snap = m.clone(); // COW clone shares the root + assert!(m.same_trie(&snap), "a clone shares the root"); + m.insert(b"(edge b c)", ()); // mutation CoW-replaces the root + assert!(!m.same_trie(&snap), "a write must change the root identity"); + let snap2 = m.clone(); + assert!(m.same_trie(&snap2)); + m.remove(b"(edge b c)"); + assert!(!m.same_trie(&snap2), "a removal must change the root identity"); + // two independently-built equal maps are NOT the same trie (different roots) + let mut n: PathMap<()> = PathMap::new(); + n.insert(b"(edge a b)", ()); + // n and snap have equal CONTENT but distinct roots + assert!(!n.same_trie(&snap)); + // empty vs empty + let e1: PathMap<()> = PathMap::new(); + let e2: PathMap<()> = PathMap::new(); + assert!(e1.same_trie(&e2), "two empty maps share the None root"); + } +}