more progress, still not complete prototype

Author: etwest

CMakeLists.txt

@@ -3,7 +3,7 @@ project(GraphZeppelin)
 
 include (FetchContent)
 
-set(CMAKE_CXX_STANDARD 14)
+set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS ON)
 set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib)
@@ -59,6 +59,22 @@ FetchContent_Declare(
   GIT_TAG         main
 )
 
+# Get VieCut
+FetchContent_Declare(
+  VieCut
+
+  GIT_REPOSITORY  https://github.com/etwest/VieCut.git
+  GIT_TAG         master
+)
+
+# Get tlx
+FetchContent_Declare(
+  tlx
+
+  GIT_REPOSITORY  https://github.com/tlx/tlx.git
+  GIT_TAG         master
+)
+
 if (BUILD_BENCH)
   # Get Google Benchmark
   FetchContent_Declare(
@@ -72,7 +88,7 @@ if (BUILD_BENCH)
   FetchContent_MakeAvailable(benchmark)
 endif()
 
-FetchContent_MakeAvailable(GutterTree StreamingUtilities)
+FetchContent_MakeAvailable(GutterTree StreamingUtilities VieCut tlx)
 
 # AVAILABLE COMPILATION DEFINITIONS:
 # VERIFY_SAMPLES_F   Use a deterministic connected-components
@@ -95,8 +111,8 @@ add_library(GraphZeppelin
   src/cc_alg_configuration.cpp
   src/sketch.cpp
   src/util.cpp)
-add_dependencies(GraphZeppelin GutterTree StreamingUtilities)
-target_link_libraries(GraphZeppelin PUBLIC xxhash GutterTree StreamingUtilities)
+add_dependencies(GraphZeppelin GutterTree StreamingUtilities VieCut tlx)
+target_link_libraries(GraphZeppelin PUBLIC xxhash GutterTree StreamingUtilities VieCut tlx)
 target_include_directories(GraphZeppelin PUBLIC include/)
 target_compile_options(GraphZeppelin PUBLIC -fopenmp)
 target_link_options(GraphZeppelin PUBLIC -fopenmp)
@@ -112,8 +128,8 @@ add_library(GraphZeppelinVerifyCC
   src/sketch.cpp
   src/util.cpp
   test/util/graph_verifier.cpp)
-add_dependencies(GraphZeppelinVerifyCC GutterTree StreamingUtilities)
-target_link_libraries(GraphZeppelinVerifyCC PUBLIC xxhash GutterTree StreamingUtilities)
+add_dependencies(GraphZeppelinVerifyCC GutterTree StreamingUtilities VieCut tlx)
+target_link_libraries(GraphZeppelinVerifyCC PUBLIC xxhash GutterTree StreamingUtilities VieCut tlx)
 target_include_directories(GraphZeppelinVerifyCC PUBLIC include/ include/test/)
 target_compile_options(GraphZeppelinVerifyCC PUBLIC -fopenmp)
 target_link_options(GraphZeppelinVerifyCC PUBLIC -fopenmp)

include/cc_sketch_alg.h

@@ -83,6 +83,7 @@ class CCSketchAlg {
   // for accessing if the DSU is valid from threads that do not perform updates
   std::atomic<bool> shared_dsu_valid;
 
+  // Most recent spanning forest computed by algorithm and associated locks
   std::unordered_set<node_id_t> *spanning_forest;
   std::mutex *spanning_forest_mtx;
 
@@ -104,8 +105,8 @@ class CCSketchAlg {
   /**
    * Sample a single supernode represented by a single sketch containing one or more vertices.
    * Updates the dsu and spanning forest with query results if edge contains new connectivity info.
-   * @param skt   sketch to sample
-   * @return      [bool] true if the query result indicates we should run an additional round.
+   * param: skt   sketch to sample
+   * return:      [bool] true if the query result indicates we should run an additional round.
    */
   bool sample_supernode(Sketch &skt);
 
@@ -115,8 +116,8 @@ class CCSketchAlg {
   void create_merge_instructions(std::vector<MergeInstr> &merge_instr);
 
   /**
-   * @param reps         set containing the roots of each supernode
-   * @param merge_instr  a list of lists of supernodes to be merged
+   * param: reps         set containing the roots of each supernode
+   * param: merge_instr  a list of lists of supernodes to be merged
    */
   bool perform_boruvka_round(const size_t cur_round, const std::vector<MergeInstr> &merge_instr,
                              std::vector<GlobalMergeData> &global_merges);
@@ -127,6 +128,8 @@ class CCSketchAlg {
    */
   void boruvka_emulation();
 
+  void filter_sf_edges(SpanningForest &sf);
+
   // constructor for use when reading from a serialized file
   CCSketchAlg(node_id_t num_vertices, size_t seed, std::ifstream &binary_stream,
               CCAlgConfiguration config);
@@ -169,9 +172,9 @@ class CCSketchAlg {
 
   /**
    * Update all the sketches for a node, given a batch of updates.
-   * @param thr_id         The id of the thread performing the update [0, num_threads)
-   * @param src_vertex     The vertex where the edges originate.
-   * @param dst_vertices   A vector of destinations.
+   * param: thr_id         The id of the thread performing the update [0, num_threads)
+   * param: src_vertex     The vertex where the edges originate.
+   * param: dst_vertices   A vector of destinations.
    */
   void apply_update_batch(int thr_id, node_id_t src_vertex,
                           const std::vector<node_id_t> &dst_vertices);
@@ -198,8 +201,8 @@ class CCSketchAlg {
   /**
    * Apply a batch of updates that have already been processed into a sketch delta.
    * Specifically, the delta is in the form of a pointer to raw bucket data.
-   * @param src_vertex   The vertex where the all edges originate.
-   * @param raw_buckets  Pointer to the array of buckets from the delta sketch
+   * param: src_vertex   The vertex where the all edges originate.
+   * param: raw_buckets  Pointer to the array of buckets from the delta sketch
    */
   void apply_raw_buckets_update(node_id_t src_vertex, Bucket *raw_buckets);
 
@@ -213,26 +216,34 @@ class CCSketchAlg {
 
   /**
    * Main parallel query algorithm utilizing Boruvka and L_0 sampling.
-   * @return  the connected components in the graph.
+   * return:  the connected components in the graph.
    */
   ConnectedComponents connected_components();
 
   /**
    * Point query algorithm utilizing Boruvka and L_0 sampling.
    * Allows for additional updates when done.
-   * @param a, b
-   * @return true if a and b are in the same connected component, false otherwise.
+   * param:  a, b vertices of the graph. Check if these are connected.
+   * return: true if a and b are in the same connected component, false otherwise.
    */
   bool point_query(node_id_t a, node_id_t b);
 
   /**
    * Return a spanning forest of the graph utilizing Boruvka and L_0 sampling
    * IMPORTANT: The updates to this algorithm MUST NOT be a function of the output of this query
    * that is, unless you really know what you're doing.
-   * @return  the spanning forest of the graph
+   * return:  the spanning forest of the graph
    */
   SpanningForest calc_spanning_forest();
 
+  /**
+   * Return k edge-disjoint spanning forests of the graph.
+   * IMPORTANT: The updates to this algorithm MUST NOT be a function of the output of this query
+   * that is, unless you really know what you're doing.
+   * return:  k edge-disjoint spanning forests of the graph.
+   */
+  std::vector<SpanningForest> calc_disjoint_spanning_forests(size_t k);
+
 #ifdef VERIFY_SAMPLES_F
   void set_verifier(std::unique_ptr<GraphVerifier> verifier) {
     this->verifier = std::move(verifier);
@@ -241,7 +252,7 @@ class CCSketchAlg {
 
   /**
    * Serialize the graph data to a binary file.
-   * @param filename the name of the file to (over)write data to.
+   * param: filename the name of the file to (over)write data to.
    */
   void write_binary(const std::string &filename);
 

include/edge_store.h

@@ -59,7 +59,7 @@ class EdgeStore {
 
   // this function is called when there are some sketch subgraphs.
   TaggedUpdateBatch insert_adj_edges(node_id_t src, node_id_t caller_first_es_subgraph,
-                                     SubgraphTaggedUpdate* dst_data, size_t dst_data_size);
+                                     std::vector<SubgraphTaggedUpdate> &dst_data);
 
   // contract vertex data by removing all updates bound for lower subgraphs than the store
   // is responsible for

include/mc_configuration.h

@@ -0,0 +1,53 @@
+#pragma once
+#include <iostream>
+
+// Configuration options for the minimum cut sketch algorithm
+class MCAlgConfiguration {
+ private:
+  // How large to make update batches as factor of sketch size
+  double _batch_factor = 1;
+  
+  // Returned min-cut guaranteed to be a +/- epsilon multiplicative approx of the true min cut.
+  double _epsilon = 0.5;
+
+  // Number of subgraphs for which we use a delta sketch
+  // When applying sketch updates to other subgraphs, apply updates directly to sketch
+  size_t _num_subgraphs_use_delta = 2;
+
+  friend class MinCutSketchAlg;
+ public:
+  // setters
+  MCAlgConfiguration& batch_factor(double batch_factor) {
+    if (batch_factor <= 0) {
+      std::cerr << "WARNING: Batch factor in MCAlgConfiguration must be > 0." << std::endl;
+      std::cerr << "         Setting to default value: " << _batch_factor << std::endl;
+    } else {
+      _batch_factor = batch_factor;
+    }
+    return *this;
+  }
+  MCAlgConfiguration& epsilon(double epsilon) {
+    if (epsilon <= 0 || epsilon > 1) {
+      std::cerr << "WARNING: MCAlgConfiguration epsilon must be in range (0, 1]." << std::endl;
+      std::cerr << "         Setting to default value: " << _epsilon << std::endl;
+    } else {
+      _epsilon = epsilon;
+    }
+    return *this;
+  }
+  MCAlgConfiguration& num_subgraphs_use_delta(size_t num_subgraphs) {
+    _num_subgraphs_use_delta = num_subgraphs;
+    return *this;
+  }
+
+  // getters
+  double get_batch_factor() { return _batch_factor; }
+  double get_epsilon() { return _epsilon; }
+  size_t get_num_subgraphs_use_delta() { return _num_subgraphs_use_delta; }
+
+  friend std::ostream& operator<< (std::ostream &out, const MCAlgConfiguration &conf) {
+    out << "Minimum Cut Algorithm Configuration:" << std::endl;
+    out << "  batch_factor = " << conf._batch_factor << std::endl;
+    return out;
+  }
+};

include/min_cut_sketch_alg.h

@@ -5,62 +5,16 @@
 
 #include "cc_sketch_alg.h"
 #include "edge_store.h"
-
-
-// Configuration options for the minimum cut sketch algorithm
-class MCAlgConfiguration {
- private:
-  // How large to make update batches as factor of sketch size
-  double _batch_factor = 1;
-  
-  // Returned min-cut guaranteed to be a +/- epsilon multiplicative approx of the true min cut.
-  double _epsilon = 0.5;
-
-  // Number of subgraphs for which we use a delta sketch
-  // When applying sketch updates to other subgraphs, apply updates directly to sketch
-  size_t _num_subgraphs_use_delta = 2;
-
-  friend class MinCutSketchAlg;
- public:
-  // setters
-  MCAlgConfiguration& batch_factor(double batch_factor) {
-    if (batch_factor <= 0) {
-      std::cerr << "WARNING: Batch factor in MCAlgConfiguration must be > 0." << std::endl;
-      std::cerr << "         Setting to default value: " << _batch_factor << std::endl;
-    } else {
-      _batch_factor = batch_factor;
-    }
-    return *this;
-  }
-  MCAlgConfiguration& epsilon(double epsilon) {
-    if (epsilon <= 0 || epsilon > 1) {
-      std::cerr << "WARNING: MCAlgConfiguration epsilon must be in range (0, 1]." << std::endl;
-      std::cerr << "         Setting to default value: " << _epsilon << std::endl;
-    } else {
-      _epsilon = epsilon;
-    }
-    return *this;
-  }
-  MCAlgConfiguration& num_subgraphs_use_delta(size_t num_subgraphs) {
-    _num_subgraphs_use_delta = num_subgraphs;
-    return *this;
-  }
-
-  // getters
-  double get_batch_factor() { return _batch_factor; }
-  double get_epsilon() { return _epsilon; }
-  size_t get_num_subgraphs_use_delta() { return _num_subgraphs_use_delta; }
-
-  friend std::ostream& operator<< (std::ostream &out, const MCAlgConfiguration &conf) {
-    out << "Minimum Cut Algorithm Configuration:" << std::endl;
-    out << "  batch_factor = " << conf._batch_factor << std::endl;
-    return out;
-  }
-};
+#include "mc_configuration.h"
 
 // Minimum cut sketch algorithm class
 class MinCutSketchAlg {
  private:
+  struct ThreadData {
+    std::vector<std::vector<node_id_t>> cc_buffers;
+    std::vector<SubgraphTaggedUpdate> edge_store_buffer;
+  };
+
   const node_id_t num_vertices;
   const size_t seed;
   MCAlgConfiguration config;
@@ -69,14 +23,15 @@ class MinCutSketchAlg {
 
   const double sketch_factor;
   const size_t sketch_samples;
+  const size_t buffer_elms;
 
   CCSketchAlg **cc_sketches;
   EdgeStore edge_store;
 
   Sketch *delta_sketches = nullptr;
-  node_id_t **update_buffers = nullptr;
+  ThreadData *thread_data = nullptr;
   size_t num_delta_sketches = 0;
-  size_t num_upd_buffers = 0;
+  size_t num_workers;
 
 #ifdef VERIFY_SAMPLES_F
   std::unique_ptr<GraphVerifier> verifier;
@@ -139,7 +94,7 @@ class MinCutSketchAlg {
    * seen thus far. This approximation is guaranteed to be within 1 +/- epsilon of the true
    * minimum cut.
    */
-  size_t calc_minimum_cut();
+  MinCut calc_minimum_cut();
 
   /**
    * Return if we have cached an answer to query.

include/return_types.h

@@ -28,10 +28,19 @@ class ConnectedComponents {
 // This class defines a spanning forest of a graph
 class SpanningForest {
  private:
-  std::vector<Edge> edges;
   node_id_t num_vertices;
+  std::vector<Edge> edges;
+  std::vector<Edge> sorted_adjacency;
+  bool has_adjacency = false;
  public:
   SpanningForest(node_id_t num_vertices, const std::unordered_set<node_id_t> *spanning_forest);
 
   const std::vector<Edge>& get_edges() const { return edges; }
+  const std::vector<Edge>& get_sorted_adjacency();
+};
+
+struct MinCut {
+  std::set<node_id_t> left_vertices;
+  std::set<node_id_t> right_vertices;
+  size_t value;
 };

include/sketch.h

@@ -117,7 +117,7 @@ class Sketch {
    * Update a sketch based on information about one of its indices.
    * param update   the point update.
    */
-  void update(const vec_t update);
+  void update(const vec_t update_idx);
 
   /**
    * Function to sample from the sketch.