ksparse recovery test cases

Author: Gillgamesh

CMakeLists.txt

@@ -131,6 +131,7 @@ if (BUILD_EXE)
     test/test_runner.cpp
     test/cc_alg_test.cpp
     test/sketch_test.cpp
+    test/recovery_test.cpp
     test/dsu_test.cpp
     test/util_test.cpp
     test/util/graph_verifier_test.cpp)

include/recovery.h

@@ -20,7 +20,7 @@ class SparseRecovery {
         size_t cleanup_sketch_support;
         // 1 - 1/2e. TODO - can do better. closer to 1-1/e. for the power-of-two-rounding, 
         // I'm gonna propose 0.69 (comfortably below sqrt(2) so we decrease the size every two levels)
-        static constexpr double reduction_factor = 0.82;
+        // static constexpr double reduction_factor = 0.82;
         static constexpr double reduction_factor = 0.69;
         uint64_t _checksum_seed;
         uint64_t seed;
@@ -30,10 +30,10 @@ class SparseRecovery {
         // should just be a single array, maybe with a lookup set of pointers for the start of each
         std::vector<Bucket> recovery_buckets;
         std::vector<size_t> starter_indices;        
-        Sketch cleanup_sketch;
         // TODO - see if we want to continue maintaining the deterministic bucket
         Bucket deterministic_bucket;
     public:
+        Sketch cleanup_sketch;
         SparseRecovery(size_t universe_size, size_t max_recovery_size, double cleanup_sketch_support_factor, uint64_t seed):
             // TODO - ugly constructor
         cleanup_sketch(universe_size, seed, ceil(cleanup_sketch_support_factor * log2(universe_size)) * 2, 1)
@@ -59,6 +59,7 @@ class SparseRecovery {
             auto full_storage_size = starter_indices.back();
             // starter_indices.pop_back();
             recovery_buckets.resize(full_storage_size);
+            reset();
         };
     private:
         size_t num_levels() const {
@@ -89,7 +90,6 @@ class SparseRecovery {
             vec_hash_t checksum = Bucket_Boruvka::get_index_hash(update, checksum_seed());
             deterministic_bucket ^= {update, checksum};
             for (size_t cfr_idx=0; cfr_idx < num_levels(); cfr_idx++) {
-                auto cfr_size = get_cfr_size(cfr_idx);
                 size_t bucket_idx = get_level_placement(update, cfr_idx);
                 Bucket &bucket = get_cfr_bucket(cfr_idx, bucket_idx);
                 bucket ^= {update, checksum};
@@ -98,6 +98,7 @@ class SparseRecovery {
         }
         void reset() {
             // zero contents of the CFRs
+            deterministic_bucket = {0, 0};
             for (size_t i=0; i < recovery_buckets.size(); i++) {
                 recovery_buckets[i] = {0, 0};
             }
@@ -112,6 +113,7 @@ class SparseRecovery {
             Bucket working_det_bucket = {0, 0};
             for (size_t cfr_idx=0; cfr_idx < num_levels(); cfr_idx++) {
                 auto cfr_size = get_cfr_size(cfr_idx);
+                std::cout << "level " << cfr_idx << " size " << cfr_size << std::endl;
                 // temporarily zero out already recovvered things:
                 size_t previously_recovered = recovered_indices.size();
                 for (size_t i=0; i < previously_recovered; i++) {
@@ -152,16 +154,11 @@ class SparseRecovery {
                     return {SUCCESS, recovered_return_vals};
                 }
                 for (auto idx: sample.idxs) {
-                    // todo - checksum stuff. tihs is bad code writing but whatever, anything
-                    // to get out of writing psuedocode...
                     recovered_return_vals.push_back(idx);
-                    // todo - this is inefficient. we are recalculating the bucket hash
-                    // for literally no reason
-                    // but doing things this way is important for undoing our recovery!
-                    // otherwise, we're stuck with a bunch of extra bookkeeping 
                     this->update(idx);
                 }
             }
+            // undo the removals for everything
             for (auto idx: recovered_return_vals) {
                 this->update(idx);
             }
@@ -174,5 +171,7 @@ class SparseRecovery {
             }
             cleanup_sketch.merge(other.cleanup_sketch);
         };
-        ~SparseRecovery();
+        ~SparseRecovery() {
+
+        };
 };

src/sketch.cpp

@@ -291,11 +291,14 @@ void Sketch::zero_contents() {
 }
 
 SketchSample Sketch::sample() {
+  // TODO - this is bugged
+  // inject buffer buckets no longer guarantees compactness
 
   if (sample_idx >= num_samples) {
     throw OutOfSamplesException(seed, num_samples, sample_idx);
   }
   // TODO - fix this so this isnt required
+  bucket_buffer.sort_and_compact();
   inject_buffer_buckets();
 
 
@@ -338,6 +341,9 @@ SketchSample Sketch::sample() {
 }
 
 ExhaustiveSketchSample Sketch::exhaustive_sample() {
+  // TODO - fix this so this isnt required
+  bucket_buffer.sort_and_compact();
+  inject_buffer_buckets();
   if (sample_idx >= num_samples) {
     throw OutOfSamplesException(seed, num_samples, sample_idx);
   }
@@ -367,6 +373,21 @@ ExhaustiveSketchSample Sketch::exhaustive_sample() {
     }
   }
 
+  // finally, check the deep buffer
+  for (size_t i = 0; i < bucket_buffer.size(); i++) {
+    const BufferEntry &entry = bucket_buffer[i];
+    // TODO - optimize this check. THIS IS GONNA CAUSE REALLY POOR
+    // PERFORMANCE UNTIL WE DO SOMETHING ABOUT IT
+    if (entry.col_idx >= first_column &&
+        entry.col_idx < first_column + cols_per_sample) {
+      if (Bucket_Boruvka::is_good(entry.value, checksum_seed())) {
+        // std::cout << "Found a bucket in the buffer" << std::endl;
+        assert(entry.row_idx >= bkt_per_col);
+        // return {entry.value.alpha, GOOD};
+        ret.insert(entry.value.alpha);
+      }
+    }
+  }
   unlikely_if (ret.size() == 0)
     return {ret, FAIL};
   return {ret, GOOD};

test/recovery_test.cpp

@@ -0,0 +1,79 @@
+#include "sketch.h"
+#include "recovery.h"
+#include "bucket.h"
+#include <chrono>
+#include <gtest/gtest.h>
+#include <random>
+#include "testing_vector.h"
+
+static size_t get_seed() {
+  auto now = std::chrono::high_resolution_clock::now();
+  return std::chrono::duration_cast<std::chrono::nanoseconds>(now.time_since_epoch()).count();
+}
+
+static const int num_columns = 1;
+TEST(RecoveryTestSuite, RecoveryZeroOrOne) {
+    SparseRecovery recovery(1 << 20, 1 << 10, 1, get_seed());
+    auto result = recovery.recover();
+    ASSERT_EQ(result.recovered_indices.size(), 0);
+    ASSERT_EQ(result.result, SUCCESS);
+    recovery.update(5);
+    ASSERT_EQ(recovery.recover().recovered_indices.size(), 1);  
+    ASSERT_EQ(recovery.recover().recovered_indices[0], 5);
+    recovery.update(5);
+    ASSERT_EQ(result.recovered_indices.size(), 0);
+    ASSERT_EQ(result.result, SUCCESS);
+}
+
+TEST(RecoveryTestSuite, RecoveryMediumSize) {
+    SparseRecovery recovery(1 << 20, 1 << 10, 1, get_seed());
+    auto result = recovery.recover();
+    ASSERT_EQ(result.recovered_indices.size(), 0);
+    ASSERT_EQ(result.result, SUCCESS);
+    recovery.update(5);
+    ASSERT_EQ(recovery.recover().recovered_indices.size(), 1);  
+    ASSERT_EQ(recovery.recover().recovered_indices[0], 5);
+    std::unordered_set<vec_t> inserted;
+    recovery.update(5);
+    for (vec_t i = 0; i < 1 << 10; i++) {
+        recovery.update(i);
+        inserted.insert(i);
+    }
+    auto result2 = recovery.recover();
+    std::unordered_set<vec_t> recovered2(result2.recovered_indices.begin(), result2.recovered_indices.end());
+    ASSERT_EQ(recovered2, inserted);
+    auto result3 = recovery.recover();
+    std::unordered_set<vec_t> recovered3(result3.recovered_indices.begin(), result3.recovered_indices.end());
+    ASSERT_EQ(recovered3, inserted);
+    
+    // REPEAT TO MAKE SURE NON-DESTRUCTIVE
+}
+
+TEST(RecoveryTestSuite, RecoveryFailureCondition) {
+    SparseRecovery recovery(1 << 20, 1 << 10, 1, get_seed());
+    std::unordered_set<vec_t> inserted;
+    for (vec_t i = 0; i < 1 << 14; i++) {
+        recovery.update(i);
+        inserted.insert(i);
+    }
+    auto result = recovery.recover();
+    ASSERT_EQ(result.result, FAILURE);
+    std::cout << "size: " << result.recovered_indices.size() << std::endl;
+    // make sure all returned things were in there:
+    for (auto idx: result.recovered_indices) {
+      ASSERT_TRUE(inserted.find(idx) != inserted.end());
+    }
+    // inserted.clear();
+    // remove all but the final few elements
+    // TODO - figure out the right place to put sketch clearing
+    recovery.cleanup_sketch.reset_sample_state();
+    for (vec_t i = 0; i < (1 << 14) - 1027; i++) {
+        recovery.update(i);
+        inserted.erase(i);
+    }
+    // TODO - WRITE A HELPER FUNCTION FOR TIHS STYLE OF TEST CASE
+    auto result3 = recovery.recover();
+    std::unordered_set<vec_t> recovered3(result3.recovered_indices.begin(), result3.recovered_indices.end());
+    ASSERT_EQ(result3.result, SUCCESS);
+    ASSERT_EQ(recovered3, inserted);
+}