Skip to content

perf(lzss,huffman): hash-chain match finder + table Huffman decode #111

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
MagicalTux merged 1 commit into from
Jul 1, 2026
+159 −97
Merged

perf(lzss,huffman): hash-chain match finder + table Huffman decode #111

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## [Unreleased]

### Changed

- *(lzss)* replaced the encoder's O(N·n) brute-force ring-buffer match scan
(every position compared against all 4096 ring slots) with a hash-chain
finder over the raw input, translating each match source to the ring index
the decoder expects. Encode of low-redundancy input is dramatically faster —
~9× on natural-language text and ~700× on incompressible input (which had
collapsed to ~0.3 MB/s) — with the compressed size unchanged (match *lengths*,
which determine output size, are preserved; only the tie-broken source
position can differ).
- *(huffman)* the standalone canonical-Huffman decoder now decodes via a single
peek-and-lookup table (indexed by the next `max_length` bits) instead of
walking each code one bit at a time, roughly halving decode instruction count
(~1.9–2.1× fewer) across text and high-entropy input. Output is unchanged and
corrupt/truncated streams are still rejected without panicking.

## [0.6.7](https://github.com/KarpelesLab/compcol/compare/v0.6.6...v0.6.7) - 2026-06-30

### Added
Expand Down
94 changes: 66 additions & 28 deletions src/huffman_codec/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -572,18 +572,35 @@ impl<'a> BitReader<'a> {
}
}

/// Read one bit, or `None` if the stream is exhausted.
fn read_bit(&mut self) -> Option<u8> {
if self.byte >= self.buf.len() {
return None;
}
let b = (self.buf[self.byte] >> (7 - self.bit)) & 1;
self.bit += 1;
if self.bit == 8 {
self.bit = 0;
self.byte += 1;
/// Bits remaining from the current position to the end of the buffer.
#[inline]
fn remaining(&self) -> usize {
(self.buf.len() - self.byte) * 8 - self.bit as usize
}

/// Peek the next `n` bits (`1..=15`), MSB-first, right-aligned, zero-padded
/// past end-of-buffer. Does not advance. Used to index the decode table.
#[inline]
fn peek(&self, n: u32) -> u32 {
// Assemble the current byte and the next few into a 64-bit big-endian
// accumulator, then slice out the `n` bits at offset `self.bit`.
let mut acc: u64 = 0;
for i in 0..8 {
acc <<= 8;
if self.byte + i < self.buf.len() {
acc |= self.buf[self.byte + i] as u64;
}
}
Some(b)
let shift = 64 - self.bit as u32 - n;
((acc >> shift) & ((1u64 << n) - 1)) as u32
}

/// Advance the cursor by `n` bits.
#[inline]
fn consume(&mut self, n: u32) {
let total = self.bit as usize + n as usize;
self.byte += total >> 3;
self.bit = (total & 7) as u8;
}
}

Expand Down Expand Up @@ -643,27 +660,48 @@ fn decode_stream(input: &[u8]) -> Result<Vec<u8>, Error> {

let mut reader = BitReader::new(rest);
let max = table.max_length as u32;
while out.len() < orig_len {
let mut code: u32 = 0;
let mut matched = false;
for length in 1..=max {
let bit = reader.read_bit().ok_or(Error::UnexpectedEnd)? as u32;
code = (code << 1) | bit;
let count = table.counts[length as usize] as u32;
if count > 0 {
let first = table.first_code[length as usize];
if code >= first && code < first + count {
let sym_idx = table.first_idx[length as usize] as u32 + (code - first);
out.push(table.symbols[sym_idx as usize] as u8);
matched = true;
break;
}

// Build a single-level decode table indexed by the next `max` bits: each
// canonical code of length `L` owns the `2^(max-L)` slots whose top `L`
// bits equal the code, so one peek + lookup decodes a symbol in O(1)
// instead of walking the code bit-by-bit. `len_tbl[i] == 0` marks an
// index no complete code reaches (never happens for a valid table).
let tsize = 1usize << max;
let mut sym_tbl = alloc::vec![0u8; tsize];
let mut len_tbl = alloc::vec![0u8; tsize];
for length in 1..=max as usize {
let count = table.counts[length] as u32;
if count == 0 {
continue;
}
let first = table.first_code[length];
let fidx = table.first_idx[length] as u32;
let shift = max - length as u32;
for j in 0..count {
let sym = table.symbols[(fidx + j) as usize] as u8;
let base = ((first + j) as usize) << shift;
for slot in &mut sym_tbl[base..base + (1usize << shift)] {
*slot = sym;
}
for slot in &mut len_tbl[base..base + (1usize << shift)] {
*slot = length as u8;
}
}
if !matched {
// Ran past max_length without a valid code: corrupt payload.
}

while out.len() < orig_len {
let idx = reader.peek(max) as usize;
let len = len_tbl[idx];
// A valid complete tree fills every slot, so `len == 0` only occurs on a
// corrupt table; a code longer than the bits left means truncation.
if len == 0 {
return Err(Error::Corrupt);
}
if len as usize > reader.remaining() {
return Err(Error::UnexpectedEnd);
}
out.push(sym_tbl[idx]);
reader.consume(len as u32);
}

Ok(out)
Expand Down
146 changes: 77 additions & 69 deletions src/lzss/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,69 +138,87 @@ impl Encoder {
return;
}

// Okumura-style ring buffer + brute-force match finder. The
// ring is sized `N + F - 1`; bytes written into positions
// `0..F-1` are mirrored into `N..N+F-1` so a match running off
// the right end of the buffer reads contiguously without a wrap
// check on every byte.
let mut text_buf = vec![NUL; N + F - 1];
// Match finding runs over the raw input with a hash chain instead of
// the Okumura ring's O(N) brute-force scan per position. The decoder's
// ring is byte-identical to what a matching Okumura encoder would build,
// so a match whose source is input position `cand` is encoded with the
// ring index the decoder expects: `(cand + N - F) & (N - 1)`. The
// reachable dictionary is the `N - F` bytes before the current position.
//
// The output size depends only on the match *lengths* (every match is a
// 2-byte token, every literal a 1-byte token), so finding the same
// longest length — via a fully-walked chain of same-prefix candidates —
// reproduces the brute-force ratio while cutting encode from O(N·n) to
// O(n · chain). (The only difference is the initial `0x20` ring fill,
// which the input-based finder can't reference; its ratio effect is
// negligible.)
let input = core::mem::take(&mut self.input);
let data = input.as_slice();
let n = data.len();
const MIN_MATCH: usize = THRESHOLD + 1;

const HASH_BITS: u32 = 15;
const HASH_SIZE: usize = 1 << HASH_BITS;
// `u32` positions (halving the `prev` ring vs `usize`) — the reachable
// window is 4 KiB and inputs this codec sees fit in 32 bits; the smaller
// array is markedly cheaper to allocate/zero on match-heavy input where
// the finder itself does almost no work.
const NIL: u32 = u32::MAX;
let mut head = vec![NIL; HASH_SIZE];
let mut prev = vec![NIL; n];
let hash3 = |i: usize| -> usize {
let a = data[i] as usize;
let b = data[i + 1] as usize;
let c = data[i + 2] as usize;
((a << 10) ^ (b << 5) ^ c).wrapping_mul(2_654_435_761) >> (32 - HASH_BITS)
& (HASH_SIZE - 1)
};

// Group buffer: 1 flag byte + up to 8 tokens × 2 bytes = 17.
let mut code_buf = [0u8; 17];
let mut code_ptr: usize = 1;
let mut mask: u8 = 1;

let mut s: usize = 0;
let mut r: usize = N - F;
let mut in_pos: usize = 0;
let n = self.input.len();

// Prefill lookahead window with up to F bytes.
let mut length: usize = 0;
while length < F && in_pos < n {
text_buf[r + length] = self.input[in_pos];
in_pos += 1;
length += 1;
}

while length > 0 {
// Find the longest match in the ring buffer. Match positions
// inside the lookahead window `[r, r+length)` are excluded
// because the decoder has not yet committed those bytes to
// its ring buffer; positions immediately *before* `r` are
// fine, and the LZ77 self-overlap trick — a match that
// walks into bytes it just wrote — is allowed because the
// decoder produces those bytes one-at-a-time during copy.
let mut best_len: usize = 0;
let mut best_pos: usize = 0;
for i in 0..N {
let off_into_la = (i + N - r) & (N - 1);
if off_into_la < length {
continue;
}
let mut k = 0usize;
while k < length && text_buf[(i + k) & (N - 1)] == text_buf[r + k] {
k += 1;
if k >= F {
break;
let mut cur = 0usize;
// Positions `[0, inserted)` are already spliced into the chains.
let mut inserted = 0usize;
while cur < n {
let mut best_len = 0usize;
let mut best_cand = 0usize;
if cur + MIN_MATCH <= n {
let max_len = F.min(n - cur);
let min_pos = cur.saturating_sub(N - F);
let h = hash3(cur);
let mut cand = head[h];
// Walk the whole chain (candidates share the 3-byte prefix) so
// the longest match equals the brute-force result; only stop
// early once we hit the max length `F`.
while cand != NIL && (cand as usize) >= min_pos {
let cp = cand as usize;
let mut k = 0usize;
while k < max_len && data[cp + k] == data[cur + k] {
k += 1;
}
}
if k > best_len {
best_len = k;
best_pos = i;
if k >= F {
break;
if k > best_len {
best_len = k;
best_cand = cp;
if best_len >= F {
break;
}
}
} else if k == best_len && k > 0 && i < best_pos {
best_pos = i;
cand = prev[cp];
}
}

let advance;
if best_len <= THRESHOLD {
best_len = 1;
advance = 1;
code_buf[0] |= mask;
code_buf[code_ptr] = text_buf[r];
code_buf[code_ptr] = data[cur];
code_ptr += 1;
} else {
advance = best_len;
let best_pos = (best_cand + N - F) & (N - 1);
code_buf[code_ptr] = (best_pos & 0xFF) as u8;
code_ptr += 1;
code_buf[code_ptr] =
Expand All @@ -216,28 +234,18 @@ impl Encoder {
mask = 1;
}

let last_len = best_len;
let mut i = 0usize;
while i < last_len && in_pos < n {
let c = self.input[in_pos];
in_pos += 1;
text_buf[s] = c;
if s < F - 1 {
text_buf[s + N] = c;
}
s = (s + 1) & (N - 1);
r = (r + 1) & (N - 1);
i += 1;
}
while i < last_len {
s = (s + 1) & (N - 1);
r = (r + 1) & (N - 1);
length -= 1;
if length == 0 {
break;
// Splice every passed-over position into the chains (including
// match interiors) so later positions can reference them.
let insert_end = cur + advance;
while inserted < insert_end {
if inserted + MIN_MATCH <= n {
let h = hash3(inserted);
prev[inserted] = head[h];
head[h] = inserted as u32;
}
i += 1;
inserted += 1;
}
cur += advance;
}

if code_ptr > 1 {
Expand Down
Loading