Migrate to new hints file format

README.md

@@ -5,5 +5,5 @@
 This repository is a collection of crates related to a SwiftSync node implementation. Some crates will be SwiftSync-specific, while others may have broader use cases.
 
 - `accumulator`: A hash-based SwiftSync accumulator used to add and subtrack elements from a set.
-- `hintfile`: Generate a SwiftSync hintfile as the role of a server.
+- `hintfile`: Read a hints file from a server.
 - `node`: Perform fast IBD using a SwiftSync hints file.

hintfile/Cargo.toml

@@ -3,16 +3,4 @@ name = "hintfile"
 version = "0.1.0"
 edition = "2021"
 
-[dependencies]
-kernel = { workspace = true }
 
-configure_me = "0.4.0"
-
-[build-dependencies]
-configure_me_codegen = "0.4.0"
-
-[package.metadata.configure_me]
-spec = "config_spec.toml"
-
-[[bin]]
-name = "construct"

hintfile/src/lib.rs

@@ -1,10 +1,11 @@
 use std::{
     collections::BTreeMap,
-    io::{self, Read, Write},
+    fs::File,
+    io::{self, Read, Seek, SeekFrom, Write},
 };
 
-type BlockHash = [u8; 32];
 type BlockHeight = u32;
+type FilePos = u64;
 
 pub fn write_compact_size<W: Write>(value: u64, writer: &mut W) -> Result<(), io::Error> {
     match value {
@@ -45,43 +46,46 @@ pub fn read_compact_size<R: Read>(reader: &mut R) -> Result<u64, io::Error> {
 
 #[derive(Debug)]
 pub struct Hints {
-    map: BTreeMap<BlockHeight, Vec<u64>>,
-    assume_valid: BlockHash,
+    map: BTreeMap<BlockHeight, FilePos>,
+    file: File,
     stop_height: BlockHeight,
 }
 
 impl Hints {
     // # Panics
     //
     // Panics when expected data is not present, or the hintfile overflows the maximum blockheight
-    pub fn from_file<R: Read>(reader: &mut R) -> Self {
+    pub fn from_file(mut file: File) -> Self {
         let mut map = BTreeMap::new();
-        let mut height = 1;
+        let mut magic = [0; 4];
+        file.read_exact(&mut magic).unwrap();
+        assert_eq!(magic, [0x55, 0x54, 0x58, 0x4f]);
+        let mut ver = [0; 1];
+        file.read_exact(&mut ver).unwrap();
+        if u8::from_le_bytes(ver) != 0x00 {
+            panic!("Unsupported file version.");
+        }
         let mut stop_height = [0; 4];
-        reader.read_exact(&mut stop_height).expect("empty file");
-        let mut assume_valid = [0; 32];
-        reader.read_exact(&mut assume_valid).expect("empty file");
-        while let Ok(count) = read_compact_size(reader) {
-            // panics on 32 bit machines
-            let mut offsets = Vec::with_capacity(count as usize);
-            for _ in 0..count {
-                offsets.push(read_compact_size(reader).expect("unexpected end of hintfile"));
-            }
-            map.insert(height, offsets);
-            height += 1;
+        file.read_exact(&mut stop_height).expect("empty file");
+        let stop_height = BlockHeight::from_le_bytes(stop_height);
+        for _ in 1..=stop_height {
+            let mut height = [0; 4];
+            file.read_exact(&mut height)
+                .expect("expected kv pair does not exist.");
+            let height = BlockHeight::from_le_bytes(height);
+            let mut file_pos = [0; 8];
+            file.read_exact(&mut file_pos)
+                .expect("expected kv pair does not exist.");
+            let file_pos = FilePos::from_le_bytes(file_pos);
+            map.insert(height, file_pos);
         }
         Self {
             map,
-            assume_valid,
-            stop_height: BlockHeight::from_le_bytes(stop_height),
+            file,
+            stop_height,
         }
     }
 
-    /// Get the last hash encoded in the hintfile.
-    pub fn stop_hash(&self) -> BlockHash {
-        self.assume_valid
-    }
-
     /// Get the stop height of the hint file.
     pub fn stop_height(&self) -> BlockHeight {
         self.stop_height
@@ -91,15 +95,20 @@ impl Hints {
     ///
     /// If there are no offset present at that height, aka an overflow, or the entry has already
     /// been fetched.
-    pub fn get_indexes(&self, height: BlockHeight) -> Vec<u64> {
-        let offsets = self
+    pub fn get_indexes(&mut self, height: BlockHeight) -> Vec<u64> {
+        let file_pos = self
             .map
             .get(&height)
             .cloned()
             .expect("block height overflow");
-        let mut indexes = Vec::with_capacity(offsets.len());
+        self.file
+            .seek(SeekFrom::Start(file_pos))
+            .expect("missing file position.");
+        let num_unspents = read_compact_size(&mut self.file).expect("unexpected missing hints.");
+        let mut indexes = Vec::new();
         let mut prev = 0;
-        for offset in offsets {
+        for _ in 0..num_unspents {
+            let offset = read_compact_size(&mut self.file).expect("unexpected missing hints.");
             let next = prev + offset;
             indexes.push(next);
             prev = next;

node/README.md

@@ -2,19 +2,23 @@
 
 This binary implements a SwiftSync client that downloads blocks in parallel from multiple peers, references a hint file, and updates an accumulator. Once the client has reached the stop hash specified in the hint file, the accumulator state is reported as verified or false. For more information, read the [SwiftSync specification](https://gist.github.com/RubenSomsen/a61a37d14182ccd78760e477c78133cd).
 
-You will need a `.hints` file locally to run this binary. There is one committed to by the repository as a `zip` file. You may uncompress it with the `unzip` tool.
+You will need a `.hints` file locally to run this binary. You may download one from a server:
 
-```
-sudo apt-get install unzip
-```
+For Signet:
 
+```bash
+curl -o signet.hints http://utxohints.store/hints/signet
 ```
-unzip bitcoin.hints.zip
+
+Or for Bitcoin:
+
+```bash
+curl -o bitcoin.hints http://utxohints.store/hints/bitcoin
 ```
 
 To build the Bitcoin kernel, you will need the following on Ubuntu:
 
-```
+```bash
 sudo apt-get install build-essential cmake pkgconf python3 libevent-dev libboost-dev
 ```
 
@@ -24,7 +28,7 @@ Finally, you will need Rust and cargo installed, you may download them from [her
 
 To start fast IBD:
 
-```
+```bash
 cargo run --bin ibd --release -- <args>
 ```
 

node/src/bin/ibd.rs

@@ -5,7 +5,7 @@ use std::{
     time::{Duration, Instant},
 };
 
-use bitcoin::{consensus, BlockHash, Network};
+use bitcoin::Network;
 use hintfile::Hints;
 use kernel::{ChainstateManager, ChainstateManagerOptions, ContextBuilder};
 
@@ -41,17 +41,16 @@ fn main() {
     tracing::subscriber::set_global_default(subscriber).unwrap();
     let hintfile_start_time = Instant::now();
     tracing::info!("Reading in {hint_path}");
-    let mut hintfile = File::open(hint_path).expect("invalid hintfile path");
-    let hints = Arc::new(Hints::from_file(&mut hintfile));
+    let hintfile = File::open(hint_path).expect("invalid hintfile path");
+    let hints = Hints::from_file(hintfile);
+    let stop_height = hints.stop_height();
     elapsed_time(hintfile_start_time);
     tracing::info!("Syncing to height {}", hints.stop_height());
     let block_file_path = blocks_dir.map(PathBuf::from);
     if let Some(block_file_path) = block_file_path.as_ref() {
         std::fs::create_dir(block_file_path).expect("could not create block file directory");
     }
-    let stop_hash =
-        consensus::deserialize::<BlockHash>(&hints.stop_hash()).expect("stop hash is not valid");
-    tracing::info!("Assume valid hash: {stop_hash}");
+    tracing::info!("Assume valid height: {stop_height}");
     tracing::info!("Finding peers with DNS");
     let dns_start_time = Instant::now();
     let peers = bootstrap_dns(network);
@@ -68,7 +67,13 @@ fn main() {
     let tip = chainman.best_header().height();
     tracing::info!("Kernel best header: {tip}");
     let chain = Arc::new(chainman);
-    sync_block_headers(stop_hash, &peers, Arc::clone(&chain), network, timeout_conf);
+    sync_block_headers(
+        stop_height,
+        &peers,
+        Arc::clone(&chain),
+        network,
+        timeout_conf,
+    );
     tracing::info!("Assume valid height: {}", chain.best_header().height());
     let (tx, rx) = channel();
     let main_routine_time = Instant::now();
@@ -78,6 +83,7 @@ fn main() {
     let mut tasks = Vec::new();
     let hashes = hashes_from_chain(Arc::clone(&chain));
     let arc_hashes = Arc::new(Mutex::new(hashes));
+    let hints = Arc::new(Mutex::new(hints));
     for task_id in 0..task_num {
         let chain = Arc::clone(&chain);
         let tx = tx.clone();
@@ -93,7 +99,7 @@ fn main() {
                 network,
                 block_file_path,
                 chain,
-                &hints,
+                hints,
                 peers,
                 tx,
                 hashes,

node/src/lib.rs

@@ -74,7 +74,7 @@ pub fn bootstrap_dns(network: Network) -> Vec<SocketAddr> {
 }
 
 pub fn sync_block_headers(
-    stop_hash: BlockHash,
+    stop_height: u32,
     hosts: &[SocketAddr],
     chainman: Arc<ChainstateManager>,
     network: Network,
@@ -99,9 +99,10 @@ pub fn sync_block_headers(
             Err(_) => continue,
         };
         tracing::info!("Connection established");
-        let curr = chainman.best_header().block_hash().hash;
-        let locator = BlockHash::from_byte_array(curr);
-        if locator.eq(&stop_hash) {
+        let curr = chainman.best_header();
+        let locator = BlockHash::from_byte_array(curr.block_hash().hash);
+        let curr_height = curr.height() as u32;
+        if curr_height.eq(&stop_height) {
             tracing::info!("Using existing header state");
             return;
         }
@@ -124,7 +125,8 @@ pub fn sync_block_headers(
                         chainman
                             .process_new_block_headers(&consensus::serialize(&header), true)
                             .expect("process headers failed");
-                        if header.block_hash().eq(&stop_hash) {
+                        let curr_height = chainman.best_header().height() as u32;
+                        if curr_height.eq(&stop_height) {
                             tracing::info!("Done syncing block headers");
                             if let Some(message_rate) =
                                 metrics.message_rate(p2p::TimedMessage::BlockHeaders)
@@ -175,7 +177,7 @@ pub fn get_blocks_for_range(
     network: Network,
     block_dir: Option<PathBuf>,
     chain: Arc<ChainstateManager>,
-    hints: &Hints,
+    hints: Arc<Mutex<Hints>>,
     peers: Arc<Mutex<Vec<SocketAddr>>>,
     updater: Sender<AccumulatorUpdate>,
     hashes: Arc<Mutex<Vec<Vec<BlockHash>>>>,
@@ -231,8 +233,10 @@ pub fn get_blocks_for_range(
                         .block_index_by_hash(kernal_hash)
                         .expect("header is in best chain.");
                     let block_height = block_index.height().unsigned_abs();
-                    let unspent_indexes: HashSet<u64> =
-                        hints.get_indexes(block_height).into_iter().collect();
+                    let unspent_indexes: HashSet<u64> = {
+                        let mut hint_ref = hints.lock().unwrap();
+                        hint_ref.get_indexes(block_height).into_iter().collect()
+                    };
                     if let Some(block_dir) = block_dir.as_ref() {
                         let file_path = block_dir.join(format!("{hash}.block"));
                         let file = File::create_new(file_path);