diff --git a/CHANGELOG.md b/CHANGELOG.md index fb88303..05f4b82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **HTTP/3 QPACK header compression** (RFC 9204) behind the new `qpack` + feature. `compcol::qpack::{QpackEncoder, QpackDecoder}` — full decoder + (static table, dynamic table built from the encoder-stream instructions, and + all field-line representations) validated byte-for-byte against the RFC 9204 + Appendix B examples; the encoder uses the static table + literals (Required + Insert Count = 0). Reuses the HPACK string Huffman code and `HeaderField`. +- **Standalone primitives / transforms**, each a first-class codec reachable + through the factory: + - `huffman` — a self-delimiting canonical (length-limited, order-0) Huffman + codec, `compcol::huffman_codec::Huffman` (name `"huffman"`). + - `rangecoder` — an adaptive order-0 binary range coder, + `compcol::rangecoder::RangeCoder` (name `"range"`). + - `mtf` — the Move-To-Front reversible transform, `compcol::mtf::Mtf` + (name `"mtf"`), a streaming length-preserving filter. + - `bwt` — a standalone block Burrows-Wheeler Transform, `compcol::bwt::Bwt` + (name `"bwt"`), with a per-block primary index. Pairs with `mtf` + an + entropy coder to build a bzip2-style pipeline from parts. + ## [0.6.2](https://github.com/KarpelesLab/compcol/compare/v0.6.1...v0.6.2) - 2026-06-12 ### Other diff --git a/Cargo.toml b/Cargo.toml index 0adde08..1a9733b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,7 +50,8 @@ all = [ "lha", "bcj", "bcj2", "delta", "arc_crunch", "arc_squeeze", "arc_squash", - "hpack", + "hpack", "qpack", + "huffman", "rangecoder", "mtf", "bwt", ] # Enables `alloc`-backed conveniences (e.g. the `factory` module, the # `compcol::vec` one-shot helpers). Pulled in automatically by features @@ -246,6 +247,25 @@ arc_squash = ["alloc"] # (name `"h2-huffman"`). The full header codec lives behind its own # `compcol::hpack` API (it is stateful over header lists, not a byte stream). hpack = ["alloc"] +# HTTP/3 QPACK header compression (RFC 9204): static table (Appendix A), +# dynamic table built from the encoder-stream instructions, and all field-line +# representations. Reuses the HPACK string Huffman code, so it pulls `hpack`. +# Lives behind its own `compcol::qpack` API (stateful header codec, not a byte +# stream). Decoder is full; the encoder uses the static table + literals. +qpack = ["alloc", "hpack"] +# Standalone canonical (length-limited) Huffman codec — `compcol::huffman_codec` +# (name `"huffman"`). Self-delimiting order-0 Huffman over bytes. +huffman = ["alloc"] +# Standalone adaptive order-0 binary range coder — `compcol::rangecoder` +# (name `"range"`). A self-contained entropy codec. +rangecoder = ["alloc"] +# Move-To-Front reversible transform — `compcol::mtf` (name `"mtf"`). A +# length-preserving, streaming byte filter (as used inside bzip2). +mtf = ["alloc"] +# Standalone block Burrows-Wheeler Transform — `compcol::bwt` (name `"bwt"`). +# Reversible block transform with a per-block primary index; pairs with `mtf` +# and an entropy coder. +bwt = ["alloc"] # `compcol::tokio_io` — async mirrors of compcol::io for the tokio # runtime. Pulls the tokio dependency for its AsyncRead/AsyncWrite # trait definitions; the rest of the crate stays dep-free. diff --git a/README.md b/README.md index d853c3f..3bee9b0 100644 --- a/README.md +++ b/README.md @@ -77,12 +77,20 @@ flag, and a `compcol` binary turns the library into a Unix-style filter. | RAR 3.x | `rar3` | `.rar` | `Unsupported` (license) | full LZ77+Huffman + E8 filter; PPMd & VM filters refused | libarchive RAR3 fixtures | | RAR 5.x | `rar5` | `.rar` | `Unsupported` (license) | full LZ77+Huffman + x86 filter; Delta/ARM refused | RARLAB-CLI fixtures | | HTTP/2 HPACK (RFC 7541) | `hpack` | — | full (header codec + `h2-huffman` string codec) | full (static+dynamic tables, integer/string coding) | RFC 7541 Appendix C vectors | - -HPACK is HTTP/2's header-compression codec, not a byte-stream codec: it -operates on `(name, value)` header lists with per-connection dynamic-table -state, so it lives behind its own `compcol::hpack` API (`HpackEncoder` / -`HpackDecoder`). The §5.2 string Huffman primitive is also exposed as the -`Http2Huffman` codec (name `h2-huffman`) through the uniform trait surface. +| HTTP/3 QPACK (RFC 9204) | `qpack` | — | static-table + literal encoder; full decoder | full (static+dynamic tables via encoder stream, all field representations) | RFC 9204 Appendix B vectors | +| Canonical Huffman (standalone) | `huffman` | `.huff` | full (length-limited, self-delimiting) | full | own round-trip | +| Range coder (adaptive order-0) | `rangecoder` | `.range` | full | full | own round-trip | +| Move-To-Front transform | `mtf` | `mtf` | full (reversible filter) | full | round-trip identity | +| Burrows-Wheeler Transform (standalone) | `bwt` | `bwt` | full (block BWT + primary index) | full | round-trip identity | + +HPACK and QPACK are HTTP header-compression codecs, not byte-stream codecs: +they operate on `(name, value)` header lists with per-connection dynamic-table +state, so they live behind their own `compcol::hpack` / `compcol::qpack` APIs. +The §5.2 string Huffman primitive is also exposed as the `Http2Huffman` codec +(name `h2-huffman`) through the uniform trait surface; QPACK reuses it. The +`huffman` / `rangecoder` / `mtf` / `bwt` features expose standalone +building-block codecs (entropy coding and reversible transforms) that can be +composed into a custom pipeline. The RAR encoders are permanently `Unsupported` per RARLAB's unRAR license terms (every clean-room RAR reader — libarchive, The diff --git a/src/bwt/mod.rs b/src/bwt/mod.rs new file mode 100644 index 0000000..2a51789 --- /dev/null +++ b/src/bwt/mod.rs @@ -0,0 +1,340 @@ +//! Burrows–Wheeler Transform (BWT) — a standalone, reversible block codec. +//! +//! The BWT is a *permutation* of the input, not a compressor: its output is +//! exactly as long as its input (plus a small per-block header). What it buys +//! you is **local clustering** — runs of bytes that share a following context +//! end up adjacent, which makes the output far more compressible by a +//! downstream entropy stage (move-to-front + RLE + Huffman, as in bzip2). It +//! is shipped here as a first-class codec alongside the other transform-only +//! filters in this crate (`delta`, `bcj`). +//! +//! The crate already computes a BWT *inside* the bzip2 pipeline, but never +//! exposes it on its own. This module is an independent, clean-room +//! implementation (it shares no code with `src/bzip2/bwt.rs`) that round-trips +//! arbitrary data on its own. +//! +//! ## Framing — block stream +//! +//! The input is split into fixed-size blocks (the last block may be shorter). +//! Each block is emitted as: +//! +//! ```text +//! ┌────────────┬───────────────┬─────────────────────────────────┐ +//! │ len u32LE │ primary u32LE │ L (BWT last column), len bytes │ +//! └────────────┴───────────────┴─────────────────────────────────┘ +//! ``` +//! +//! * `len` — number of bytes in this block (`1..=block_size`). A `len` of 0 is +//! never emitted; empty input produces zero blocks (an empty stream). +//! * `primary` — the row index, in the sorted rotation matrix, of the row that +//! is the original block (the BWT "primary index"). Must be `< len`. +//! * `L` — the last column of the sorted rotation matrix: `len` bytes. +//! +//! The stream is **self-delimiting**: the decoder reads blocks back-to-back +//! until the input is exhausted. There is no overall header or trailer, so the +//! codec composes cleanly in front of an entropy coder. +//! +//! ## Forward transform +//! +//! For each block we build the order of its cyclic rotations with a +//! prefix-doubling (Manber–Myers) sort — `O(n log n)` ranking rounds, each an +//! `O(n)` counting sort on the `(rank[i], rank[i + k])` pairs. From the sorted +//! rotation order `sa` (where `sa[r]` is the starting offset of the rotation +//! ranked `r`) the BWT last column is `L[r] = block[(sa[r] + n - 1) mod n]`, +//! and the primary index is the rank of the rotation that starts at offset 0. +//! +//! The prefix-doubling sort is `O(n log n)` and handles the pathological cases +//! (all-equal bytes, long repeats) without degrading to `O(n² log n)`. +//! +//! ## Inverse transform +//! +//! Standard LF-mapping reconstruction: counting-sort the last column to get +//! each symbol's starting offset in the first column, build the `next` vector +//! that links each row to its predecessor in the original order, then walk +//! `len` steps from the primary index, emitting one byte per step. +//! +//! ## Edge cases +//! +//! Empty input → zero blocks. One-byte block → `primary = 0`, `L = [b]`. +//! All-equal bytes and highly repetitive blocks are handled by the stable +//! rank-based sort. The decoder rejects a `primary >= len`, a `len` of zero, +//! or a truncated block with [`Error::Corrupt`] / [`Error::UnexpectedEnd`] and +//! never panics. +//! +//! ## Licensing +//! +//! Clean-room from the published BWT algorithm description (Burrows & Wheeler, +//! 1994) and the textbook prefix-doubling rotation sort. No code was copied +//! from `src/bzip2/` or any third-party source. + +#![cfg_attr(docsrs, doc(cfg(feature = "bwt")))] + +extern crate alloc; +use alloc::vec::Vec; + +use crate::error::Error; +use crate::traits::{Algorithm, RawDecoder, RawEncoder, RawProgress}; + +mod transform; + +#[cfg(test)] +mod tests; + +/// Default block size: 256 KiB. Large enough to give the transform good +/// context for typical text/binary inputs, small enough that the `u32` +/// length/primary fields and the `O(n log n)` sort stay comfortable. +pub const DEFAULT_BLOCK_SIZE: usize = 256 * 1024; + +/// Minimum permitted block size. A block must hold at least one byte. +pub const MIN_BLOCK_SIZE: usize = 1; + +/// Maximum permitted block size. Bounded so the per-block `u32` length and +/// primary-index fields cannot overflow, with margin to spare. +pub const MAX_BLOCK_SIZE: usize = 64 * 1024 * 1024; + +/// Zero-sized marker type implementing [`Algorithm`] for the BWT codec. +#[derive(Debug, Clone, Copy, Default)] +pub struct Bwt; + +/// Encoder configuration: the block size in bytes. +/// +/// `#[non_exhaustive]`: construct via [`EncoderConfig::default`] and the +/// `with_*` builders rather than a struct literal, so new tuning knobs can be +/// added later without breaking downstream code. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub struct EncoderConfig { + /// Block size in bytes. Clamped to `MIN_BLOCK_SIZE..=MAX_BLOCK_SIZE` at + /// encoder construction time. Default is [`DEFAULT_BLOCK_SIZE`]. + pub block_size: usize, +} + +impl Default for EncoderConfig { + fn default() -> Self { + Self { + block_size: DEFAULT_BLOCK_SIZE, + } + } +} + +impl EncoderConfig { + /// Default configuration (256 KiB blocks). + pub fn new() -> Self { + Self::default() + } + + /// Set the block size in bytes (clamped to + /// `MIN_BLOCK_SIZE..=MAX_BLOCK_SIZE` at encoder build time). + #[must_use] + pub fn with_block_size(mut self, block_size: usize) -> Self { + self.block_size = block_size; + self + } +} + +impl Algorithm for Bwt { + const NAME: &'static str = "bwt"; + type Encoder = Encoder; + type Decoder = Decoder; + type EncoderConfig = EncoderConfig; + type DecoderConfig = (); + + fn encoder_with(cfg: EncoderConfig) -> Encoder { + Encoder::new(cfg.block_size) + } + fn decoder_with(_: ()) -> Decoder { + Decoder::new() + } +} + +// ─── encoder ───────────────────────────────────────────────────────────── + +/// Streaming BWT encoder. +/// +/// Buffers all input, then on `finish` splits it into blocks and emits the +/// block stream described in the [module docs](crate::bwt). The whole input is +/// buffered because the transform operates on complete blocks; memory cost is +/// `O(input)`. +#[derive(Debug)] +pub struct Encoder { + block_size: usize, + input: Vec, + output: Vec, + out_cursor: usize, + finalized: bool, +} + +impl Encoder { + /// Construct an encoder with the given block size (clamped to + /// `MIN_BLOCK_SIZE..=MAX_BLOCK_SIZE`). + pub fn new(block_size: usize) -> Self { + Self { + block_size: block_size.clamp(MIN_BLOCK_SIZE, MAX_BLOCK_SIZE), + input: Vec::new(), + output: Vec::new(), + out_cursor: 0, + finalized: false, + } + } + + fn finalize(&mut self) { + for block in self.input.chunks(self.block_size) { + // `block` is never empty: chunks() of a non-empty slice yields + // non-empty chunks, and an empty input yields no chunks at all. + let (last_col, primary) = transform::forward(block); + let len = block.len() as u32; + self.output.extend_from_slice(&len.to_le_bytes()); + self.output + .extend_from_slice(&(primary as u32).to_le_bytes()); + self.output.extend_from_slice(&last_col); + } + } +} + +impl RawEncoder for Encoder { + fn raw_encode(&mut self, input: &[u8], _output: &mut [u8]) -> Result { + self.input.extend_from_slice(input); + Ok(RawProgress { + consumed: input.len(), + written: 0, + done: false, + }) + } + + fn raw_finish(&mut self, output: &mut [u8]) -> Result { + if !self.finalized { + self.finalize(); + self.finalized = true; + } + let remaining = self.output.len() - self.out_cursor; + let take = remaining.min(output.len()); + output[..take].copy_from_slice(&self.output[self.out_cursor..self.out_cursor + take]); + self.out_cursor += take; + Ok(RawProgress { + consumed: 0, + written: take, + done: self.out_cursor >= self.output.len(), + }) + } + + fn raw_reset(&mut self) { + self.input.clear(); + self.output.clear(); + self.out_cursor = 0; + self.finalized = false; + } +} + +// ─── decoder ───────────────────────────────────────────────────────────── + +/// Streaming BWT decoder (inverse of [`Encoder`]). +/// +/// Buffers the whole encoded stream, then decodes every block in one pass on +/// `finish` and drains the reconstructed bytes into the caller's output across +/// calls. Output size per block is bounded by that block's declared `len`, so +/// a crafted small input cannot expand without limit. +#[derive(Debug)] +pub struct Decoder { + input: Vec, + output: Vec, + out_cursor: usize, + decoded: bool, +} + +impl Default for Decoder { + fn default() -> Self { + Self::new() + } +} + +impl Decoder { + /// Construct a fresh decoder. + pub fn new() -> Self { + Self { + input: Vec::new(), + output: Vec::new(), + out_cursor: 0, + decoded: false, + } + } + + /// Decode the buffered block stream into `self.output`. Idempotent. + fn decode_all(&mut self) -> Result<(), Error> { + if self.decoded { + return Ok(()); + } + let buf = &self.input[..]; + let mut pos = 0usize; + while pos < buf.len() { + // Each block header is 8 bytes: len (u32-LE) + primary (u32-LE). + if buf.len() - pos < 8 { + return Err(Error::UnexpectedEnd); + } + let len = + u32::from_le_bytes([buf[pos], buf[pos + 1], buf[pos + 2], buf[pos + 3]]) as usize; + let primary = + u32::from_le_bytes([buf[pos + 4], buf[pos + 5], buf[pos + 6], buf[pos + 7]]) + as usize; + pos += 8; + + // A zero-length block is never emitted by the encoder, and a + // primary index must address a real row. + if len == 0 || primary >= len { + return Err(Error::Corrupt); + } + // The last column must be fully present. + if buf.len() - pos < len { + return Err(Error::UnexpectedEnd); + } + let last_col = &buf[pos..pos + len]; + pos += len; + + transform::inverse(last_col, primary, &mut self.output)?; + } + self.decoded = true; + Ok(()) + } + + fn drain(&mut self, output: &mut [u8]) -> RawProgress { + let remaining = self.output.len() - self.out_cursor; + let take = remaining.min(output.len()); + output[..take].copy_from_slice(&self.output[self.out_cursor..self.out_cursor + take]); + self.out_cursor += take; + RawProgress { + consumed: 0, + written: take, + done: self.out_cursor >= self.output.len(), + } + } +} + +impl RawDecoder for Decoder { + fn raw_decode(&mut self, input: &[u8], output: &mut [u8]) -> Result { + if !self.decoded { + self.input.extend_from_slice(input); + return Ok(RawProgress { + consumed: input.len(), + written: 0, + done: false, + }); + } + let p = self.drain(output); + Ok(RawProgress { + consumed: 0, + written: p.written, + done: p.done, + }) + } + + fn raw_finish(&mut self, output: &mut [u8]) -> Result { + self.decode_all()?; + Ok(self.drain(output)) + } + + fn raw_reset(&mut self) { + self.input.clear(); + self.output.clear(); + self.out_cursor = 0; + self.decoded = false; + } +} diff --git a/src/bwt/tests.rs b/src/bwt/tests.rs new file mode 100644 index 0000000..02ea693 --- /dev/null +++ b/src/bwt/tests.rs @@ -0,0 +1,286 @@ +//! Round-trip and robustness tests for the BWT block codec. + +extern crate alloc; +use alloc::vec; +use alloc::vec::Vec; + +use super::{Bwt, Decoder, Encoder, EncoderConfig}; +use crate::error::Error; +use crate::traits::{Algorithm, Decoder as _, Encoder as _, Status}; + +/// Drive an `Encoder` to completion over `input`, returning the encoded bytes. +/// Uses a deliberately small output buffer to exercise the drain loop. +fn encode_all(mut enc: Encoder, input: &[u8]) -> Vec { + let mut out = Vec::new(); + let mut buf = [0u8; 17]; + let mut consumed = 0; + while consumed < input.len() { + let (p, status) = enc.encode(&input[consumed..], &mut buf).unwrap(); + out.extend_from_slice(&buf[..p.written]); + consumed += p.consumed; + if matches!(status, Status::InputEmpty) { + break; + } + } + loop { + let (p, status) = enc.finish(&mut buf).unwrap(); + out.extend_from_slice(&buf[..p.written]); + if matches!(status, Status::StreamEnd) { + break; + } + } + out +} + +/// Drive a `Decoder` to completion over `encoded`, returning the decoded bytes. +fn decode_all(mut dec: Decoder, encoded: &[u8]) -> Result, Error> { + let mut out = Vec::new(); + let mut buf = [0u8; 19]; + let mut consumed = 0; + while consumed < encoded.len() { + let (p, status) = dec.decode(&encoded[consumed..], &mut buf)?; + out.extend_from_slice(&buf[..p.written]); + consumed += p.consumed; + if matches!(status, Status::StreamEnd) { + return Ok(out); + } + if matches!(status, Status::InputEmpty) { + break; + } + } + loop { + let (p, status) = dec.finish(&mut buf)?; + out.extend_from_slice(&buf[..p.written]); + if matches!(status, Status::StreamEnd) { + break; + } + } + Ok(out) +} + +fn roundtrip_with(block_size: usize, input: &[u8]) { + let enc = Bwt::encoder_with(EncoderConfig::default().with_block_size(block_size)); + let encoded = encode_all(enc, input); + let dec = Bwt::decoder(); + let decoded = decode_all(dec, &encoded).expect("decode failed"); + assert_eq!( + decoded, input, + "roundtrip mismatch (block_size={block_size})" + ); +} + +fn roundtrip(input: &[u8]) { + // Default block size, plus a couple of small sizes to force multi-block. + roundtrip_with(super::DEFAULT_BLOCK_SIZE, input); + roundtrip_with(64, input); + roundtrip_with(1, input); + roundtrip_with(7, input); +} + +#[test] +fn empty_input_produces_empty_stream() { + let enc = Bwt::encoder(); + let encoded = encode_all(enc, b""); + assert!(encoded.is_empty(), "empty input must emit zero blocks"); + let decoded = decode_all(Bwt::decoder(), &encoded).unwrap(); + assert!(decoded.is_empty()); +} + +#[test] +fn one_byte() { + roundtrip(b"X"); +} + +#[test] +fn repetitive_banana() { + roundtrip(b"banana"); + roundtrip(b"bananabananabanana"); + roundtrip(b"mississippi"); +} + +#[test] +fn english_text_multiblock() { + let text = b"The Burrows-Wheeler transform rearranges a character string into \ + runs of similar characters. This is useful for compression, since \ + it tends to be easy to compress a string that has runs of repeated \ + characters by techniques such as move-to-front transform and \ + run-length encoding. More importantly, the transformation is \ + reversible, without needing to store any additional data except \ + the position of the first original character."; + roundtrip(text); +} + +#[test] +fn all_byte_values() { + let block: Vec = (0..=255u8).collect(); + roundtrip(&block); + // Repeated a few times so multiple blocks at small sizes have full alphabet. + let mut big = Vec::new(); + for _ in 0..4 { + big.extend_from_slice(&block); + } + roundtrip(&big); +} + +#[test] +fn all_same_byte() { + roundtrip(&[0u8; 100]); + roundtrip(&[0xAB; 257]); +} + +#[test] +fn pseudo_random() { + // Simple xorshift PRNG — deterministic, no deps. + let mut state = 0x1234_5678_9abc_def0u64; + let mut data = Vec::with_capacity(5000); + for _ in 0..5000 { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + data.push((state & 0xff) as u8); + } + roundtrip(&data); +} + +#[test] +fn multi_block_larger_than_block_size() { + // Input strictly larger than the block size forces >1 block. + let mut data = Vec::new(); + let mut state = 0xdead_beefu32; + for _ in 0..1000 { + state = state.wrapping_mul(1664525).wrapping_add(1013904223); + data.push((state >> 16) as u8); + } + roundtrip_with(100, &data); // 10 full blocks + roundtrip_with(333, &data); // 3 blocks + remainder +} + +#[test] +fn default_block_size_is_256k() { + assert_eq!(super::DEFAULT_BLOCK_SIZE, 256 * 1024); + assert_eq!(EncoderConfig::default().block_size, 256 * 1024); +} + +#[test] +fn block_size_clamped() { + // Zero clamps up to MIN, absurdly large clamps down to MAX. + let enc = Encoder::new(0); + assert_eq!(enc.block_size, super::MIN_BLOCK_SIZE); + let enc = Encoder::new(usize::MAX); + assert_eq!(enc.block_size, super::MAX_BLOCK_SIZE); +} + +// ─── malformed-input rejection (no panics) ─────────────────────────────── + +#[test] +fn rejects_truncated_header() { + // Fewer than 8 header bytes. + for len in 1..8 { + let bad = vec![0u8; len]; + let err = decode_all(Bwt::decoder(), &bad).unwrap_err(); + assert_eq!(err, Error::UnexpectedEnd); + } +} + +#[test] +fn rejects_zero_length_block() { + // len=0, primary=0, no payload. + let mut bad = Vec::new(); + bad.extend_from_slice(&0u32.to_le_bytes()); + bad.extend_from_slice(&0u32.to_le_bytes()); + let err = decode_all(Bwt::decoder(), &bad).unwrap_err(); + assert_eq!(err, Error::Corrupt); +} + +#[test] +fn rejects_primary_out_of_range() { + // len=3, primary=3 (== len, invalid), 3 payload bytes. + let mut bad = Vec::new(); + bad.extend_from_slice(&3u32.to_le_bytes()); + bad.extend_from_slice(&3u32.to_le_bytes()); + bad.extend_from_slice(b"abc"); + let err = decode_all(Bwt::decoder(), &bad).unwrap_err(); + assert_eq!(err, Error::Corrupt); +} + +#[test] +fn rejects_truncated_payload() { + // len=10 but only 4 payload bytes present. + let mut bad = Vec::new(); + bad.extend_from_slice(&10u32.to_le_bytes()); + bad.extend_from_slice(&0u32.to_le_bytes()); + bad.extend_from_slice(b"abcd"); + let err = decode_all(Bwt::decoder(), &bad).unwrap_err(); + assert_eq!(err, Error::UnexpectedEnd); +} + +#[test] +fn arbitrary_garbage_never_panics() { + // Feed a variety of random-ish byte strings; decode must return a result + // (Ok or Err) and never panic. + let mut state = 0xabcd_1234u32; + for _ in 0..200 { + let mut data = Vec::new(); + state = state.wrapping_mul(1103515245).wrapping_add(12345); + let n = (state >> 24) as usize % 40; + for _ in 0..n { + state = state.wrapping_mul(1103515245).wrapping_add(12345); + data.push((state >> 16) as u8); + } + let _ = decode_all(Bwt::decoder(), &data); + } +} + +// ─── classic clustering property ───────────────────────────────────────── + +#[test] +fn clusters_runs_on_repetitive_text() { + // BWT of repetitive text should produce more/longer runs than the input. + // We check that the last column of a single block has strictly fewer + // "transitions" (adjacent differing bytes) than the original for a highly + // structured input — the property that makes BWT useful. + let input = b"abcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabc"; + let enc = Bwt::encoder_with(EncoderConfig::default().with_block_size(input.len())); + let encoded = encode_all(enc, input); + // Skip the 8-byte header to get the last column. + let last_col = &encoded[8..]; + let transitions = |s: &[u8]| s.windows(2).filter(|w| w[0] != w[1]).count(); + assert!( + transitions(last_col) < transitions(input), + "BWT should cluster runs: input transitions {}, L transitions {}", + transitions(input), + transitions(last_col) + ); +} + +#[test] +fn reset_reuses_encoder_and_decoder() { + let mut enc = Bwt::encoder(); + let _ = encode_all_inplace(&mut enc, b"first stream"); + enc.reset(); + let encoded = encode_all_inplace(&mut enc, b"second stream"); + let decoded = decode_all(Bwt::decoder(), &encoded).unwrap(); + assert_eq!(decoded, b"second stream"); +} + +fn encode_all_inplace(enc: &mut Encoder, input: &[u8]) -> Vec { + let mut out = Vec::new(); + let mut buf = [0u8; 32]; + let mut consumed = 0; + while consumed < input.len() { + let (p, status) = enc.encode(&input[consumed..], &mut buf).unwrap(); + out.extend_from_slice(&buf[..p.written]); + consumed += p.consumed; + if matches!(status, Status::InputEmpty) { + break; + } + } + loop { + let (p, status) = enc.finish(&mut buf).unwrap(); + out.extend_from_slice(&buf[..p.written]); + if matches!(status, Status::StreamEnd) { + break; + } + } + out +} diff --git a/src/bwt/transform.rs b/src/bwt/transform.rs new file mode 100644 index 0000000..7d9d205 --- /dev/null +++ b/src/bwt/transform.rs @@ -0,0 +1,306 @@ +//! The BWT forward and inverse transforms for a single block. +//! +//! Kept separate from the streaming/framing wrapper in `mod.rs` so the core +//! permutation logic can be unit-tested in isolation. + +extern crate alloc; +use alloc::vec; +use alloc::vec::Vec; + +use crate::error::Error; + +/// Forward BWT of a single non-empty block. +/// +/// Returns `(last_column, primary_index)` where `last_column.len() == +/// block.len()` and `primary_index < block.len()`. +/// +/// # Panics +/// +/// Never panics for `block.len() >= 1`. Callers must not pass an empty block +/// (the encoder never produces zero-length blocks); an empty block would yield +/// an empty column and a meaningless primary index. +pub(super) fn forward(block: &[u8]) -> (Vec, usize) { + let n = block.len(); + debug_assert!(n >= 1); + + // `sa[r]` = starting offset of the cyclic rotation ranked `r`. + let sa = sort_rotations(block); + + // Last column: the byte just before the rotation's start, cyclically. + // L[r] = block[(sa[r] + n - 1) mod n]. + let mut last_col = vec![0u8; n]; + let mut primary = 0usize; + for (r, &start) in sa.iter().enumerate() { + let start = start as usize; + // (start + n - 1) % n, computed without underflow. + let prev = if start == 0 { n - 1 } else { start - 1 }; + last_col[r] = block[prev]; + if start == 0 { + primary = r; + } + } + (last_col, primary) +} + +/// Sort the `n` cyclic rotations of `block` and return their starting offsets +/// in sorted order (`sa[r]` = offset of the rotation ranked `r`). +/// +/// Uses prefix doubling (Manber–Myers): start with rank = byte value, then +/// repeatedly refine ranks by the pair `(rank[i], rank[(i + k) mod n])` for +/// doubling `k`, until every rotation has a distinct rank (or `k >= n`). Each +/// round is an `O(n)` counting sort over the rank pairs, and there are +/// `O(log n)` rounds, so the whole thing is `O(n log n)`. +fn sort_rotations(block: &[u8]) -> Vec { + let n = block.len(); + debug_assert!(n >= 1); + + // Order array: the rotation offsets, to be permuted into sorted order. + let mut order: Vec = (0..n as u32).collect(); + + if n == 1 { + return order; + } + + // `rank[i]` = current rank of the rotation starting at offset `i`. + // + // Initialise from the byte at each offset, but *densely*: map each + // distinct byte value to its 0-based order among the values that actually + // occur. This keeps every rank in `0..n` from the very first round, which + // is what the counting sort's bucket sizing (`n_keys = n`) relies on — + // raw byte values would reach 255 and overflow the buckets on small + // blocks. + let mut present = [false; 256]; + for &b in block { + present[b as usize] = true; + } + let mut byte_rank = [0u32; 256]; + let mut next_rank = 0u32; + for (v, &seen) in present.iter().enumerate() { + if seen { + byte_rank[v] = next_rank; + next_rank += 1; + } + } + let mut rank: Vec = block.iter().map(|&b| byte_rank[b as usize]).collect(); + // Scratch buffers reused across rounds. + let mut new_rank: Vec = vec![0; n]; + let mut tmp_order: Vec = vec![0; n]; + + let mut k = 1usize; + loop { + // Sort `order` by the key (rank[i], rank[(i + k) mod n]) using a + // stable two-pass counting sort (LSD radix on the two rank fields). + // Pass 1: by the second key, rank[(i + k) mod n]. + counting_sort_by(&order, &mut tmp_order, n, |i| { + rank[(i as usize + k) % n] as usize + }); + // Pass 2: by the first key, rank[i]. Stable, so ties keep the + // second-key order established above. + counting_sort_by(&tmp_order, &mut order, n, |i| rank[i as usize] as usize); + + // Recompute ranks from the freshly sorted order. Two adjacent + // rotations share a rank iff both key components are equal. + new_rank[order[0] as usize] = 0; + let mut r = 0u32; + for w in 1..n { + let prev = order[w - 1] as usize; + let cur = order[w] as usize; + let prev_key = (rank[prev], rank[(prev + k) % n]); + let cur_key = (rank[cur], rank[(cur + k) % n]); + if cur_key != prev_key { + r += 1; + } + new_rank[cur] = r; + } + rank.copy_from_slice(&new_rank); + + // All rotations distinct → fully sorted. + if r as usize == n - 1 { + break; + } + // Doubling. Once k >= n the second key spans the whole rotation, so + // one more round (already done above) suffices; guard anyway. + k <<= 1; + if k >= n { + break; + } + } + order +} + +/// Stable counting sort of `src` (a permutation of rotation offsets) into +/// `dst`, keyed by `key(offset)` which must return a value in `0..n_keys`. +/// +/// `n_keys` is an upper bound on the key range; we use `n` (the block length) +/// since every rank lies in `0..n`. +fn counting_sort_by(src: &[u32], dst: &mut [u32], n_keys: usize, key: F) +where + F: Fn(u32) -> usize, +{ + let mut counts = vec![0usize; n_keys + 1]; + for &i in src { + counts[key(i)] += 1; + } + // Prefix sums → starting offset for each key bucket. + let mut acc = 0usize; + for c in counts.iter_mut() { + let cur = *c; + *c = acc; + acc += cur; + } + // Scatter, preserving input order within a bucket (stability). + for &i in src { + let slot = key(i); + dst[counts[slot]] = i; + counts[slot] += 1; + } +} + +/// Inverse BWT: reconstruct the original block from its last column and +/// primary index, appending the result to `out`. +/// +/// `last_col.len()` is the block length `n`; `primary < n` is guaranteed by +/// the caller (the framing layer validates it). Uses the standard LF-mapping +/// walk. +pub(super) fn inverse(last_col: &[u8], primary: usize, out: &mut Vec) -> Result<(), Error> { + let n = last_col.len(); + // Defensive: the framing layer already checks these, but the transform + // must be safe in isolation and never panic on bad arguments. + if n == 0 || primary >= n { + return Err(Error::Corrupt); + } + + // Count occurrences of each byte value in the last column. + let mut counts = [0usize; 256]; + for &b in last_col { + counts[b as usize] += 1; + } + // `start[c]` = index in the (sorted) first column where value `c` begins. + let mut start = [0usize; 256]; + let mut acc = 0usize; + for c in 0..256 { + start[c] = acc; + acc += counts[c]; + } + + // `next[i]` links row `i` of the last column to the row whose first-column + // entry is the same physical symbol. Walking `next` from the primary index + // yields the original bytes in order. + let mut next = vec![0u32; n]; + let mut cursor = start; // mutable copy of bucket starts + for (i, &b) in last_col.iter().enumerate() { + let c = b as usize; + next[cursor[c]] = i as u32; + cursor[c] += 1; + } + + // Walk: start at the primary row, follow `next` n times, emitting the + // last-column byte at each visited row. + out.reserve(n); + let mut p = next[primary] as usize; + for _ in 0..n { + out.push(last_col[p]); + p = next[p] as usize; + } + Ok(()) +} + +#[cfg(test)] +mod transform_tests { + use super::*; + use alloc::vec::Vec; + + /// Reference O(n² log n) rotation sort, used to cross-check the fast + /// prefix-doubling sort. Builds every rotation explicitly and sorts. + fn reference_forward(block: &[u8]) -> (Vec, usize) { + let n = block.len(); + let mut idx: Vec = (0..n).collect(); + idx.sort_by(|&a, &b| { + for off in 0..n { + let ca = block[(a + off) % n]; + let cb = block[(b + off) % n]; + if ca != cb { + return ca.cmp(&cb); + } + } + core::cmp::Ordering::Equal + }); + let mut last = Vec::with_capacity(n); + let mut primary = 0; + for (r, &start) in idx.iter().enumerate() { + let prev = if start == 0 { n - 1 } else { start - 1 }; + last.push(block[prev]); + if start == 0 { + primary = r; + } + } + (last, primary) + } + + fn roundtrip(block: &[u8]) { + let (l, p) = forward(block); + assert_eq!(l.len(), block.len()); + let mut out = Vec::new(); + inverse(&l, p, &mut out).unwrap(); + assert_eq!(out, block, "roundtrip mismatch for {block:?}"); + } + + #[test] + fn forward_matches_reference() { + let cases: &[&[u8]] = &[ + b"banana", + b"mississippi", + b"abracadabra", + b"aaaaaa", + b"a", + b"ab", + b"ba", + b"the quick brown fox jumps over the lazy dog", + &[0, 0, 0, 1, 0, 0], + &[255, 0, 255, 0, 255], + ]; + for &c in cases { + let fast = forward(c); + let reference = reference_forward(c); + assert_eq!(fast, reference, "forward mismatch for {c:?}"); + roundtrip(c); + } + } + + #[test] + fn single_byte() { + let (l, p) = forward(b"Z"); + assert_eq!(l, b"Z"); + assert_eq!(p, 0); + roundtrip(b"Z"); + } + + #[test] + fn all_same_byte() { + let block = [7u8; 64]; + let (l, p) = forward(&block); + // Every rotation is identical, so the last column is all 7s and the + // primary index is well-defined (the stable sort keeps offset 0 first + // among equal rotations, so primary == 0). + assert_eq!(l, block); + assert_eq!(p, 0); + roundtrip(&block); + } + + #[test] + fn inverse_rejects_bad_primary() { + let mut out = Vec::new(); + assert_eq!(inverse(b"abc", 3, &mut out), Err(Error::Corrupt)); + assert_eq!(inverse(b"", 0, &mut out), Err(Error::Corrupt)); + } + + #[test] + fn banana_known_last_column() { + // Classic worked example. Rotations of "banana" sorted: + // abanan, anaban, ananab, banana, nabana, nanaba + // last column = "nnbaaa", primary index of "banana" is row 3. + let (l, p) = forward(b"banana"); + assert_eq!(l, b"nnbaaa"); + assert_eq!(p, 3); + } +} diff --git a/src/factory.rs b/src/factory.rs index 4c172d5..4985f5f 100644 --- a/src/factory.rs +++ b/src/factory.rs @@ -140,6 +140,18 @@ pub fn encoder_by_name(name: &str) -> Option> { crate::hpack::Http2Huffman::NAME => Some(Box::new( ::encoder(), )), + #[cfg(feature = "huffman")] + crate::huffman_codec::Huffman::NAME => Some(Box::new( + ::encoder(), + )), + #[cfg(feature = "rangecoder")] + crate::rangecoder::RangeCoder::NAME => Some(Box::new( + ::encoder(), + )), + #[cfg(feature = "mtf")] + crate::mtf::Mtf::NAME => Some(Box::new(::encoder())), + #[cfg(feature = "bwt")] + crate::bwt::Bwt::NAME => Some(Box::new(::encoder())), #[cfg(feature = "bcj")] crate::bcj::BcjX86::NAME => Some(Box::new(::encoder())), #[cfg(feature = "bcj")] @@ -378,6 +390,18 @@ pub fn decoder_by_name(name: &str) -> Option> { crate::hpack::Http2Huffman::NAME => Some(Box::new( ::decoder(), )), + #[cfg(feature = "huffman")] + crate::huffman_codec::Huffman::NAME => Some(Box::new( + ::decoder(), + )), + #[cfg(feature = "rangecoder")] + crate::rangecoder::RangeCoder::NAME => Some(Box::new( + ::decoder(), + )), + #[cfg(feature = "mtf")] + crate::mtf::Mtf::NAME => Some(Box::new(::decoder())), + #[cfg(feature = "bwt")] + crate::bwt::Bwt::NAME => Some(Box::new(::decoder())), #[cfg(feature = "bcj")] crate::bcj::BcjX86::NAME => Some(Box::new(::decoder())), #[cfg(feature = "bcj")] @@ -758,6 +782,20 @@ pub const fn extension(name: &str) -> Option<&'static str> { if str_eq(name, "delta") && cfg!(feature = "delta") { return Some("delta"); } + // Standalone primitives / transforms with no conventional file extension; + // the codec name doubles as the suffix the CLI appends. + if str_eq(name, "huffman") && cfg!(feature = "huffman") { + return Some("huff"); + } + if str_eq(name, "range") && cfg!(feature = "rangecoder") { + return Some("range"); + } + if str_eq(name, "mtf") && cfg!(feature = "mtf") { + return Some("mtf"); + } + if str_eq(name, "bwt") && cfg!(feature = "bwt") { + return Some("bwt"); + } if str_eq(name, "crunch") && cfg!(feature = "arc_crunch") { return Some("arc"); } @@ -887,6 +925,14 @@ pub const fn names() -> &'static [&'static str] { crate::lha::Lh7::NAME, #[cfg(feature = "hpack")] crate::hpack::Http2Huffman::NAME, + #[cfg(feature = "huffman")] + crate::huffman_codec::Huffman::NAME, + #[cfg(feature = "rangecoder")] + crate::rangecoder::RangeCoder::NAME, + #[cfg(feature = "mtf")] + crate::mtf::Mtf::NAME, + #[cfg(feature = "bwt")] + crate::bwt::Bwt::NAME, #[cfg(feature = "bcj")] crate::bcj::BcjX86::NAME, #[cfg(feature = "bcj")] diff --git a/src/huffman_codec/mod.rs b/src/huffman_codec/mod.rs new file mode 100644 index 0000000..6869923 --- /dev/null +++ b/src/huffman_codec/mod.rs @@ -0,0 +1,817 @@ +//! Standalone canonical (order-0) Huffman codec. +//! +//! Unlike `crate::huffman` — the internal, deflate-oriented canonical +//! table builder — this module is a complete, self-delimiting byte codec: +//! it builds a length-limited canonical Huffman code from the input's own +//! byte-frequency statistics, serialises that code into the stream, and +//! emits the Huffman-coded payload. Decoding needs *nothing* out of band: +//! the original length and the full code description travel inside the +//! stream. The codec is fully self-contained — it does not depend on the +//! deflate-gated `crate::huffman` internals. +//! +//! It is an order-0 model (each byte coded independently of its +//! neighbours), so it captures only the static symbol-frequency +//! redundancy of the input — text and other skewed-histogram data shrink; +//! already-compressed or uniform-random data does not. For LZ-style +//! redundancy use one of the dictionary codecs (`deflate`, `lz4`, …). +//! +//! # Stream framing +//! +//! All bits of the Huffman payload are packed **MSB-first** (the most- +//! significant bit of the first code occupies bit 7 of the first payload +//! byte) — the same bit order RFC 1951 uses *within* a code, applied here +//! to the whole stream so the codec is trivially byte-reproducible. +//! +//! ```text +//! ┌────────────────────────────────────────────────────────────────┐ +//! │ original length : LEB128 varint (unsigned) │ +//! ├────────────────────────────────────────────────────────────────┤ +//! │ code-length table : present only when length > 0 │ +//! │ 256 entries, one per byte value, each a 4-bit nibble (0..=15), │ +//! │ RLE-compressed (see below). 0 = symbol absent. │ +//! ├────────────────────────────────────────────────────────────────┤ +//! │ Huffman-coded payload : `original length` symbols, MSB-first│ +//! │ then zero-bit padding to the next byte boundary. │ +//! └────────────────────────────────────────────────────────────────┘ +//! ``` +//! +//! For an **empty input** the stream is just the single varint byte `0x00`; +//! no table, no payload. +//! +//! ## Code-length table RLE +//! +//! The 256 code lengths (one nibble each, `0..=15`) are run-length encoded +//! with a tiny byte-oriented scheme — flexible enough to make the common +//! cases (long runs of "absent", repeated lengths) cheap, and never larger +//! than ~257 bytes in the worst case. Each command is one control byte: +//! +//! * `0x00..=0x0F` — a single literal length `n` (the low nibble). +//! * `0x10..=0xEF` — a **short run**: the high nibble `(c>>4)` (1..=14) is +//! the length value, the low nibble `(c&0x0F)` plus 3 is the repeat count +//! (so 3..=18 repeats of one length in a single byte). The high nibble is +//! never 0 (that would collide with the literal commands), so zero runs +//! use the dedicated `0xF2` command below. +//! * `0xF0` — a **long run of zeros**: the next byte `k` encodes +//! `k + 19` consecutive absent symbols (19..=274). Used to skip large +//! gaps in the alphabet compactly. +//! * `0xF1` — a **long run of a length**: next byte is the length value +//! (1..=15), the byte after is `k`, meaning `k + 19` repeats (19..=274). +//! * `0xF2` — a **short run of zeros**: the next byte `k` (0..=15) encodes +//! `k + 3` consecutive absent symbols (3..=18). +//! +//! The decoder consumes commands until exactly 256 lengths have been +//! produced; a table that over- or under-fills 256, contains a length +//! `> 15`, or whose lengths violate the Kraft inequality is rejected as +//! [`Error::Corrupt`]. +//! +//! ## Single-symbol input +//! +//! When the input is one distinct byte repeated, that byte is assigned a +//! 1-bit code (the degenerate Huffman case). The payload is then one bit +//! per input byte, i.e. `ceil(len/8)` bytes — a ~8× shrink. + +#![cfg_attr(docsrs, doc(cfg(feature = "huffman")))] + +use alloc::vec; +use alloc::vec::Vec; + +use crate::error::Error; +use crate::traits::{Algorithm, RawDecoder, RawEncoder, RawProgress}; + +/// Maximum Huffman code length, in bits. Matches the deflate cap so the +/// 4-bit nibble table encoding is always sufficient (a 256-symbol alphabet +/// fits comfortably under a 15-bit length limit). +const MAX_CODE_LEN: u8 = 15; + +/// Zero-sized marker type implementing [`Algorithm`] for the standalone +/// canonical Huffman codec. +#[derive(Debug, Clone, Copy, Default)] +pub struct Huffman; + +impl Algorithm for Huffman { + const NAME: &'static str = "huffman"; + type Encoder = Encoder; + type Decoder = Decoder; + type EncoderConfig = (); + type DecoderConfig = (); + fn encoder_with(_: ()) -> Encoder { + Encoder::new() + } + fn decoder_with(_: ()) -> Decoder { + Decoder::new() + } +} + +// ─── LEB128 varint ──────────────────────────────────────────────────────── + +fn write_varint(out: &mut Vec, mut v: u64) { + loop { + let byte = (v & 0x7F) as u8; + v >>= 7; + if v == 0 { + out.push(byte); + break; + } + out.push(byte | 0x80); + } +} + +/// Read a LEB128 varint, returning the value and the number of bytes +/// consumed. Rejects overlong encodings (would shift past 64 bits) and +/// truncated input as [`Error::Corrupt`]. +fn read_varint(buf: &[u8]) -> Result<(u64, usize), Error> { + let mut v: u64 = 0; + let mut shift = 0u32; + for (i, &byte) in buf.iter().enumerate() { + if shift >= 64 { + return Err(Error::Corrupt); + } + v |= ((byte & 0x7F) as u64) << shift; + if byte & 0x80 == 0 { + return Ok((v, i + 1)); + } + shift += 7; + } + Err(Error::Corrupt) +} + +// ─── code-length table (de)serialisation ────────────────────────────────── + +/// RLE-encode the 256-entry code-length array per the scheme documented at +/// the module level. +fn encode_lengths(lengths: &[u8; 256], out: &mut Vec) { + let mut i = 0usize; + while i < 256 { + let val = lengths[i]; + // Count the run of identical values starting at `i`. + let mut run = 1usize; + while i + run < 256 && lengths[i + run] == val { + run += 1; + } + i += run; + + if val == 0 { + // Absent symbols: emit long-zero runs first, then a short tail. + while run >= 19 { + let k = (run - 19).min(255); + out.push(0xF0); + out.push(k as u8); + run -= k + 19; + } + emit_short(out, 0, run); + } else { + while run >= 19 { + let k = (run - 19).min(255); + out.push(0xF1); + out.push(val); + out.push(k as u8); + run -= k + 19; + } + emit_short(out, val, run); + } + } +} + +/// Emit `count` (0..=18) repeats of `val` using literal / short-run commands. +fn emit_short(out: &mut Vec, val: u8, count: usize) { + let mut left = count; + while left > 0 { + if left >= 3 { + let n = left.min(18); + if val == 0 { + // Zero runs can't use the high-nibble form (would collide + // with literal commands), so use the dedicated 0xF2 command. + out.push(0xF2); + out.push((n - 3) as u8); + left -= n; + } else if val <= 14 { + // High nibble = length value (1..=14). Low nibble = count-3. + out.push((val << 4) | ((n - 3) as u8)); + left -= n; + } else { + // val == 15: no short-run form (0xF* reserved), emit a literal. + out.push(val); // 0x0F + left -= 1; + } + } else { + out.push(val); // literal 0x00..=0x0F + left -= 1; + } + } +} + +/// Decode the RLE code-length table into a fresh `[u8; 256]`, returning the +/// number of input bytes consumed. Rejects malformed tables as +/// [`Error::Corrupt`]. +fn decode_lengths(buf: &[u8]) -> Result<([u8; 256], usize), Error> { + let mut lengths = [0u8; 256]; + let mut pos = 0usize; // index into output (0..=256) + let mut i = 0usize; // index into input + + while pos < 256 { + let c = *buf.get(i).ok_or(Error::Corrupt)?; + i += 1; + match c { + 0x00..=0x0F => { + lengths[pos] = c; + pos += 1; + } + 0xF0 => { + let k = *buf.get(i).ok_or(Error::Corrupt)? as usize; + i += 1; + let count = k + 19; + if pos + count > 256 { + return Err(Error::Corrupt); + } + // zeros: already zero-initialised; just advance. + pos += count; + } + 0xF1 => { + let val = *buf.get(i).ok_or(Error::Corrupt)?; + i += 1; + let k = *buf.get(i).ok_or(Error::Corrupt)? as usize; + i += 1; + if val == 0 || val > MAX_CODE_LEN { + return Err(Error::Corrupt); + } + let count = k + 19; + if pos + count > 256 { + return Err(Error::Corrupt); + } + for slot in &mut lengths[pos..pos + count] { + *slot = val; + } + pos += count; + } + 0xF2 => { + let k = *buf.get(i).ok_or(Error::Corrupt)? as usize; + i += 1; + let count = k + 3; + if pos + count > 256 { + return Err(Error::Corrupt); + } + // zeros: already zero-initialised; just advance. + pos += count; + } + // Short run: high nibble = value (1..=14), low nibble = count-3. + _ => { + let val = c >> 4; + let count = (c & 0x0F) as usize + 3; + // val is 1..=14 here by construction (0xF0/0xF1 handled above, + // 0xFE/0xFF would give val==15 which the encoder never emits; + // accept and let Kraft validation reject if inconsistent). + if pos + count > 256 { + return Err(Error::Corrupt); + } + for slot in &mut lengths[pos..pos + count] { + *slot = val; + } + pos += count; + } + } + } + + Ok((lengths, i)) +} + +// ─── length-limited canonical Huffman (self-contained package-merge) ────── + +/// Compute optimal code lengths bounded by `max_length` for `freqs` via the +/// Larmore–Hirschberg package-merge algorithm. `out[i] == 0` iff +/// `freqs[i] == 0`; otherwise `1 ≤ out[i] ≤ max_length`. Single-symbol +/// alphabets get a 1-bit code (the degenerate case). +/// +/// Self-contained reimplementation (so the `huffman` feature does not pull +/// in the deflate-gated `crate::huffman`). +fn length_limited_lengths(freqs: &[u32; 256], max_length: u8) -> [u8; 256] { + let mut out = [0u8; 256]; + + // Coins: (freq, symbol) for present symbols, ascending by frequency. + let mut coins: Vec<(u32, u16)> = freqs + .iter() + .enumerate() + .filter_map(|(i, &f)| if f > 0 { Some((f, i as u16)) } else { None }) + .collect(); + let n = coins.len(); + if n == 0 { + return out; + } + if n == 1 { + out[coins[0].1 as usize] = 1; + return out; + } + coins.sort_by_key(|&(f, _)| f); + + // Pool of package-merge elements. A coin references a symbol; a pair + // references two pool indices. + #[derive(Clone, Copy)] + enum Kind { + Coin(u16), + Pair(u32, u32), + } + struct Elem { + cost: u64, + kind: Kind, + } + let mut pool: Vec = Vec::with_capacity(n * (max_length as usize) * 2 + 8); + + // Deepest level: one coin per symbol, ascending. + let mut current: Vec = Vec::with_capacity(2 * n); + for &(f, sym) in &coins { + pool.push(Elem { + cost: f as u64, + kind: Kind::Coin(sym), + }); + current.push((pool.len() - 1) as u32); + } + + for _ in 1..max_length { + // Pair consecutive entries into packages. + let mut packages: Vec = Vec::with_capacity(current.len() / 2); + let mut i = 0; + while i + 1 < current.len() { + let a = current[i]; + let b = current[i + 1]; + let cost = pool[a as usize].cost + pool[b as usize].cost; + pool.push(Elem { + cost, + kind: Kind::Pair(a, b), + }); + packages.push((pool.len() - 1) as u32); + i += 2; + } + + // Fresh coins for this level. + let coin_start = pool.len(); + for &(f, sym) in &coins { + pool.push(Elem { + cost: f as u64, + kind: Kind::Coin(sym), + }); + } + let fresh: Vec = (coin_start..pool.len()).map(|i| i as u32).collect(); + + // Merge two cost-sorted lists. + let mut merged: Vec = Vec::with_capacity(fresh.len() + packages.len()); + let (mut ci, mut pi) = (0usize, 0usize); + while ci < fresh.len() && pi < packages.len() { + if pool[fresh[ci] as usize].cost <= pool[packages[pi] as usize].cost { + merged.push(fresh[ci]); + ci += 1; + } else { + merged.push(packages[pi]); + pi += 1; + } + } + merged.extend_from_slice(&fresh[ci..]); + merged.extend_from_slice(&packages[pi..]); + current = merged; + } + + // Take the 2n-2 smallest level-1 items; each Coin reached contributes + // one bit to its symbol's length. + let pick = 2 * n - 2; + let mut stack: Vec = Vec::with_capacity(32); + for &root in ¤t[..pick] { + stack.clear(); + stack.push(root); + while let Some(idx) = stack.pop() { + match pool[idx as usize].kind { + Kind::Coin(sym) => out[sym as usize] += 1, + Kind::Pair(a, b) => { + stack.push(a); + stack.push(b); + } + } + } + } + + out +} + +// ─── canonical code build (self-contained, MSB-first) ───────────────────── + +/// Build the canonical MSB-first code value for each symbol from its code +/// length, per RFC 1951 §3.2.2. `codes[i]` is meaningful only when +/// `lengths[i] > 0`. +fn canonical_codes(lengths: &[u8; 256]) -> [u16; 256] { + let mut count = [0u32; 16]; + for &len in lengths.iter() { + if len > 0 { + count[len as usize] += 1; + } + } + let mut next_code = [0u32; 16]; + let mut code: u32 = 0; + for bits in 1..=15usize { + code = (code + count[bits - 1]) << 1; + next_code[bits] = code; + } + let mut codes = [0u16; 256]; + for (i, &len) in lengths.iter().enumerate() { + if len > 0 { + codes[i] = next_code[len as usize] as u16; + next_code[len as usize] += 1; + } + } + codes +} + +/// A canonical decode table: counts per length, the symbols in canonical +/// order, and the first code value at each length. Validates the Kraft +/// inequality at build time so [`decode_stream`] can trust the table. +struct CanonicalTable { + counts: [u16; 16], + first_code: [u32; 16], + first_idx: [u16; 16], + symbols: Vec, + max_length: u8, + /// The single symbol when the tree is degenerate (one 1-bit code). + single: Option, +} + +impl CanonicalTable { + fn from_lengths(lengths: &[u8; 256]) -> Result { + let mut counts = [0u16; 16]; + let mut max_length = 0u8; + let mut present = 0usize; + for &len in lengths.iter() { + if len > MAX_CODE_LEN { + return Err(Error::Corrupt); + } + if len > 0 { + counts[len as usize] += 1; + present += 1; + if len > max_length { + max_length = len; + } + } + } + + if present == 0 { + // No symbols: only valid for an empty stream, which never reaches + // here (the encoder writes no table for empty input). Reject. + return Err(Error::Corrupt); + } + + // Degenerate single-symbol tree: exactly one symbol, 1-bit code. + let single = if present == 1 { + if counts[1] != 1 { + return Err(Error::Corrupt); + } + let sym = lengths + .iter() + .position(|&l| l > 0) + .expect("present == 1 guarantees one nonzero length") as u16; + Some(sym) + } else { + None + }; + + // Kraft inequality: Σ counts[l] · 2^(15-l) compared against 2^15. + // For a complete code (more than one symbol) we require equality so + // the decoder can never encounter an undefined code; the encoder + // always produces complete codes. The single-symbol 1-bit code is + // the documented incomplete exception (kraft == 2^14). + let mut kraft: u32 = 0; + for l in 1..=15u32 { + kraft += (counts[l as usize] as u32) << (15 - l); + } + if single.is_none() && kraft != (1 << 15) { + return Err(Error::Corrupt); + } + + let mut first_code = [0u32; 16]; + let mut first_idx = [0u16; 16]; + let mut code: u32 = 0; + let mut idx: u16 = 0; + for l in 1..=15usize { + code <<= 1; + first_code[l] = code; + first_idx[l] = idx; + code += counts[l] as u32; + idx += counts[l]; + } + + let mut symbols = vec![0u16; present]; + let mut next = first_idx; + for (sym, &len) in lengths.iter().enumerate() { + if len > 0 { + symbols[next[len as usize] as usize] = sym as u16; + next[len as usize] += 1; + } + } + + Ok(Self { + counts, + first_code, + first_idx, + symbols, + max_length, + single, + }) + } +} + +// ─── MSB-first bit writer / reader (self-contained) ─────────────────────── + +struct BitWriter { + out: Vec, + cur: u8, + nbits: u8, +} + +impl BitWriter { + fn new() -> Self { + Self { + out: Vec::new(), + cur: 0, + nbits: 0, + } + } + + /// Write the low `len` bits of `code`, MSB-first. + fn write(&mut self, code: u16, len: u8) { + let mut i = len; + while i > 0 { + i -= 1; + let bit = ((code >> i) & 1) as u8; + self.cur = (self.cur << 1) | bit; + self.nbits += 1; + if self.nbits == 8 { + self.out.push(self.cur); + self.cur = 0; + self.nbits = 0; + } + } + } + + /// Flush any partial byte, padding with zero bits. + fn finish(mut self) -> Vec { + if self.nbits > 0 { + self.cur <<= 8 - self.nbits; + self.out.push(self.cur); + } + self.out + } +} + +/// MSB-first bit reader over a borrowed slice. +struct BitReader<'a> { + buf: &'a [u8], + byte: usize, + bit: u8, // 0..=7, counts from MSB +} + +impl<'a> BitReader<'a> { + fn new(buf: &'a [u8]) -> Self { + Self { + buf, + byte: 0, + bit: 0, + } + } + + /// Read one bit, or `None` if the stream is exhausted. + fn read_bit(&mut self) -> Option { + if self.byte >= self.buf.len() { + return None; + } + let b = (self.buf[self.byte] >> (7 - self.bit)) & 1; + self.bit += 1; + if self.bit == 8 { + self.bit = 0; + self.byte += 1; + } + Some(b) + } +} + +// ─── core transforms ────────────────────────────────────────────────────── + +/// Encode `input` into a complete self-delimiting Huffman stream. +fn encode_stream(input: &[u8]) -> Vec { + let mut out = Vec::new(); + write_varint(&mut out, input.len() as u64); + + if input.is_empty() { + return out; + } + + // Frequencies. + let mut freqs = [0u32; 256]; + for &b in input { + freqs[b as usize] += 1; + } + + // Length-limited canonical lengths (single-symbol → 1-bit code). + let lengths = length_limited_lengths(&freqs, MAX_CODE_LEN); + encode_lengths(&lengths, &mut out); + + let codes = canonical_codes(&lengths); + let mut bw = BitWriter::new(); + for &b in input { + let s = b as usize; + bw.write(codes[s], lengths[s]); + } + out.extend_from_slice(&bw.finish()); + out +} + +/// Decode a self-delimiting Huffman stream back into the original bytes. +fn decode_stream(input: &[u8]) -> Result, Error> { + let (orig_len, vlen) = read_varint(input)?; + let orig_len = orig_len as usize; + let mut rest = &input[vlen..]; + + if orig_len == 0 { + return Ok(Vec::new()); + } + + let (lengths, consumed) = decode_lengths(rest)?; + rest = &rest[consumed..]; + + let table = CanonicalTable::from_lengths(&lengths)?; + let mut out = Vec::with_capacity(orig_len); + + // Degenerate single-symbol stream: every code is one bit, the symbol is + // fixed. We don't need to inspect the payload bits. + if let Some(sym) = table.single { + out.resize(orig_len, sym as u8); + return Ok(out); + } + + let mut reader = BitReader::new(rest); + let max = table.max_length as u32; + while out.len() < orig_len { + let mut code: u32 = 0; + let mut matched = false; + for length in 1..=max { + let bit = reader.read_bit().ok_or(Error::UnexpectedEnd)? as u32; + code = (code << 1) | bit; + let count = table.counts[length as usize] as u32; + if count > 0 { + let first = table.first_code[length as usize]; + if code >= first && code < first + count { + let sym_idx = table.first_idx[length as usize] as u32 + (code - first); + out.push(table.symbols[sym_idx as usize] as u8); + matched = true; + break; + } + } + } + if !matched { + // Ran past max_length without a valid code: corrupt payload. + return Err(Error::Corrupt); + } + } + + Ok(out) +} + +// ─── encoder ────────────────────────────────────────────────────────────── + +/// Streaming canonical-Huffman encoder. +/// +/// Buffers all input (the code is built from whole-stream statistics, so +/// no byte can be emitted until the input ends), transforms at +/// `raw_finish`, then drains. Memory is +/// `O(input)`. +#[derive(Debug)] +pub struct Encoder { + input: Vec, + output: Vec, + cursor: usize, + finalized: bool, +} + +impl Encoder { + /// Construct a fresh encoder. + pub const fn new() -> Self { + Self { + input: Vec::new(), + output: Vec::new(), + cursor: 0, + finalized: false, + } + } +} + +impl Default for Encoder { + fn default() -> Self { + Self::new() + } +} + +impl RawEncoder for Encoder { + fn raw_encode(&mut self, input: &[u8], _output: &mut [u8]) -> Result { + self.input.extend_from_slice(input); + Ok(RawProgress { + consumed: input.len(), + written: 0, + done: false, + }) + } + + fn raw_finish(&mut self, output: &mut [u8]) -> Result { + if !self.finalized { + self.output = encode_stream(&self.input); + self.finalized = true; + } + let remaining = self.output.len() - self.cursor; + let take = remaining.min(output.len()); + output[..take].copy_from_slice(&self.output[self.cursor..self.cursor + take]); + self.cursor += take; + Ok(RawProgress { + consumed: 0, + written: take, + done: self.cursor >= self.output.len(), + }) + } + + fn raw_reset(&mut self) { + self.input.clear(); + self.output.clear(); + self.cursor = 0; + self.finalized = false; + } +} + +// ─── decoder ────────────────────────────────────────────────────────────── + +/// Streaming canonical-Huffman decoder. +/// +/// Buffers the whole compressed stream (the payload is a single MSB-first +/// bitstream that can't be resumed across `decode` calls without a +/// resumable bit-reader state machine), decodes once the stream ends, then +/// drains the decoded bytes. Output is bounded by the in-stream length +/// header, so a crafted small input cannot expand without limit. +#[derive(Debug)] +pub struct Decoder { + input: Vec, + output: Vec, + cursor: usize, + decoded: bool, +} + +impl Decoder { + /// Construct a fresh decoder. + pub const fn new() -> Self { + Self { + input: Vec::new(), + output: Vec::new(), + cursor: 0, + decoded: false, + } + } + + fn drain(&mut self, output: &mut [u8]) -> RawProgress { + let remaining = self.output.len() - self.cursor; + let take = remaining.min(output.len()); + output[..take].copy_from_slice(&self.output[self.cursor..self.cursor + take]); + self.cursor += take; + RawProgress { + consumed: 0, + written: take, + done: self.cursor >= self.output.len(), + } + } +} + +impl Default for Decoder { + fn default() -> Self { + Self::new() + } +} + +impl RawDecoder for Decoder { + fn raw_decode(&mut self, input: &[u8], output: &mut [u8]) -> Result { + if !self.decoded { + self.input.extend_from_slice(input); + return Ok(RawProgress { + consumed: input.len(), + written: 0, + done: false, + }); + } + Ok(self.drain(output)) + } + + fn raw_finish(&mut self, output: &mut [u8]) -> Result { + if !self.decoded { + self.output = decode_stream(&self.input)?; + self.decoded = true; + } + Ok(self.drain(output)) + } + + fn raw_reset(&mut self) { + self.input.clear(); + self.output.clear(); + self.cursor = 0; + self.decoded = false; + } +} + +#[cfg(test)] +mod tests; diff --git a/src/huffman_codec/tests.rs b/src/huffman_codec/tests.rs new file mode 100644 index 0000000..9448956 --- /dev/null +++ b/src/huffman_codec/tests.rs @@ -0,0 +1,288 @@ +//! Round-trip and malformed-input tests for the standalone Huffman codec. + +use super::*; +use crate::traits::{Decoder as _, Encoder as _, Status}; +use alloc::vec; +use alloc::vec::Vec; + +/// Encode `input` through the streaming encoder, draining into a Vec. +fn encode(input: &[u8]) -> Vec { + let mut enc = Encoder::new(); + let mut out = Vec::new(); + let mut scratch = [0u8; 7]; // deliberately tiny to exercise drain loops + let mut consumed = 0; + while consumed < input.len() { + let (p, st) = enc.encode(&input[consumed..], &mut scratch).unwrap(); + out.extend_from_slice(&scratch[..p.written]); + consumed += p.consumed; + match st { + Status::InputEmpty => break, + Status::OutputFull => continue, + Status::StreamEnd => break, + } + } + loop { + let (p, st) = enc.finish(&mut scratch).unwrap(); + out.extend_from_slice(&scratch[..p.written]); + if matches!(st, Status::StreamEnd) { + break; + } + assert!(p.written > 0, "finish stalled"); + } + out +} + +/// Decode `stream` through the streaming decoder, draining into a Vec. +fn decode(stream: &[u8]) -> Result, Error> { + let mut dec = Decoder::new(); + let mut out = Vec::new(); + let mut scratch = [0u8; 5]; // tiny to exercise drain loops + // Feed all input first (decoder buffers until finish). + let mut consumed = 0; + while consumed < stream.len() { + let (p, _st) = dec.decode(&stream[consumed..], &mut scratch)?; + out.extend_from_slice(&scratch[..p.written]); + consumed += p.consumed; + if p.consumed == 0 && p.written == 0 { + break; + } + } + loop { + let (p, st) = dec.finish(&mut scratch)?; + out.extend_from_slice(&scratch[..p.written]); + if matches!(st, Status::StreamEnd) { + break; + } + assert!(p.written > 0, "finish stalled"); + } + Ok(out) +} + +fn roundtrip(input: &[u8]) -> Vec { + let enc = encode(input); + let dec = decode(&enc).expect("decode should succeed"); + assert_eq!(dec, input, "round-trip mismatch"); + enc +} + +#[test] +fn empty_input() { + let enc = roundtrip(&[]); + // Empty stream is just the varint 0. + assert_eq!(enc, vec![0x00]); +} + +#[test] +fn single_byte() { + roundtrip(&[0x42]); +} + +#[test] +fn single_symbol_repeated() { + let input = vec![0xABu8; 10_000]; + let enc = roundtrip(&input); + // 1-bit code → ~ceil(10000/8) payload + small header. Must be a big shrink. + assert!( + enc.len() < input.len() / 4, + "single-symbol input should shrink ~8x, got {} from {}", + enc.len(), + input.len() + ); +} + +#[test] +fn english_text() { + let text = b"the quick brown fox jumps over the lazy dog. \ + the quick brown fox jumps over the lazy dog. \ + pack my box with five dozen liquor jugs. \ + how vexingly quick daft zebras jump!"; + let mut input = Vec::new(); + for _ in 0..50 { + input.extend_from_slice(text); + } + let enc = roundtrip(&input); + assert!( + enc.len() < input.len(), + "english text must shrink: {} -> {}", + input.len(), + enc.len() + ); +} + +#[test] +fn all_byte_values_once() { + let input: Vec = (0..=255u16).map(|b| b as u8).collect(); + roundtrip(&input); +} + +#[test] +fn all_byte_values_many() { + // Each value present, varied frequencies. + let mut input = Vec::new(); + for b in 0..=255u16 { + for _ in 0..(1 + (b % 7)) { + input.push(b as u8); + } + } + roundtrip(&input); +} + +#[test] +fn highly_skewed() { + // One symbol dominates; a few rare others. + let mut input = vec![0u8; 5000]; + input.extend_from_slice(&[1, 2, 3, 4, 5, 1, 2, 1]); + input.extend(vec![0u8; 5000]); + let enc = roundtrip(&input); + assert!( + enc.len() < input.len() / 4, + "skewed input should shrink hard" + ); +} + +#[test] +fn pseudo_random() { + // A simple xorshift PRNG — high entropy, should round-trip exactly even + // if it doesn't shrink. + let mut state: u32 = 0x1234_5678; + let mut input = Vec::with_capacity(4096); + for _ in 0..4096 { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + input.push((state & 0xFF) as u8); + } + roundtrip(&input); +} + +#[test] +fn two_symbols() { + let mut input = Vec::new(); + for i in 0..1000 { + input.push(if i % 3 == 0 { b'A' } else { b'B' }); + } + roundtrip(&input); +} + +#[test] +fn one_shot_vec_helpers() { + let input = b"compression makes things smaller, sometimes, when there is redundancy"; + let enc = crate::vec::compress_to_vec::(input).unwrap(); + let dec = crate::vec::decompress_to_vec::(&enc).unwrap(); + assert_eq!(dec, input); +} + +// ─── code-length table RLE unit tests ───────────────────────────────────── + +#[test] +fn length_table_roundtrip_various() { + let cases: &[[u8; 256]] = &[ + [0u8; 256], + { + let mut a = [0u8; 256]; + a[65] = 1; // single symbol, 1-bit + a + }, + { + let mut a = [3u8; 256]; // all length 3 (not valid Kraft, but RLE only) + a[0] = 15; + a[255] = 1; + a + }, + ]; + for case in cases { + let mut buf = Vec::new(); + encode_lengths(case, &mut buf); + let (decoded, consumed) = decode_lengths(&buf).unwrap(); + assert_eq!(consumed, buf.len()); + assert_eq!(&decoded, case); + } +} + +// ─── malformed-input rejection (no panic, Error::Corrupt) ───────────────── + +#[test] +fn truncated_varint_is_corrupt() { + // A varint with continuation bit set but no following byte. + assert_eq!(decode(&[0x80]).unwrap_err(), Error::Corrupt); +} + +#[test] +fn truncated_table_is_corrupt() { + // Nonzero length but the table commands run out before 256 entries. + let stream = [0x05u8, 0x01]; // len=5, then one literal-length command, then EOF + let err = decode(&stream).unwrap_err(); + assert_eq!(err, Error::Corrupt); +} + +#[test] +fn oversubscribed_tree_is_corrupt() { + // Build a table that over-fills the Kraft budget: many short codes. + // 256 symbols all with length 1 → kraft = 256 * 2^14 ≫ 2^15. + let mut stream = Vec::new(); + write_varint(&mut stream, 4); // claim 4 output bytes + let lengths = [1u8; 256]; + encode_lengths(&lengths, &mut stream); + // Append some payload bytes so we don't fail on EOF first. + stream.extend_from_slice(&[0xFF, 0xFF, 0xFF, 0xFF]); + let err = decode(&stream).unwrap_err(); + assert_eq!(err, Error::Corrupt); +} + +#[test] +fn incomplete_tree_is_corrupt() { + // Two symbols each with length 2: kraft = 2 * 2^13 = 2^14 < 2^15. + // Incomplete (not single-symbol) → rejected. + let mut stream = Vec::new(); + write_varint(&mut stream, 2); + let mut lengths = [0u8; 256]; + lengths[10] = 2; + lengths[20] = 2; + encode_lengths(&lengths, &mut stream); + stream.extend_from_slice(&[0x00, 0x00]); + let err = decode(&stream).unwrap_err(); + assert_eq!(err, Error::Corrupt); +} + +#[test] +fn length_over_15_is_corrupt() { + // Hand-craft a stream whose F1 command declares a length of 16. + let mut stream = Vec::new(); + write_varint(&mut stream, 1); + stream.push(0xF1); + stream.push(16); // illegal length + stream.push(0); // k=0 → 19 repeats + let err = decode(&stream).unwrap_err(); + assert_eq!(err, Error::Corrupt); +} + +#[test] +fn reset_reuses_encoder_and_decoder() { + let mut enc = Encoder::new(); + let mut scratch = [0u8; 64]; + enc.encode(b"first", &mut scratch).unwrap(); + let mut first = Vec::new(); + loop { + let (p, st) = enc.finish(&mut scratch).unwrap(); + first.extend_from_slice(&scratch[..p.written]); + if matches!(st, Status::StreamEnd) { + break; + } + } + enc.reset(); + // After reset, encoder produces a fresh stream for new input. + let again = encode(b"second"); + assert_eq!(decode(&again).unwrap(), b"second"); + assert_eq!(decode(&first).unwrap(), b"first"); +} + +#[test] +fn truncated_payload_is_unexpected_end() { + // Valid header but payload too short to decode the claimed length. + let full = encode(b"abracadabra"); + // Drop the final payload byte. + let truncated = &full[..full.len() - 1]; + let err = decode(truncated).unwrap_err(); + // Either UnexpectedEnd (ran out of bits) — must not panic. + assert!(matches!(err, Error::UnexpectedEnd | Error::Corrupt)); +} diff --git a/src/lib.rs b/src/lib.rs index c9d7560..1b986af 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -179,6 +179,21 @@ pub mod lha; #[cfg(feature = "hpack")] pub mod hpack; +#[cfg(feature = "qpack")] +pub mod qpack; + +#[cfg(feature = "huffman")] +pub mod huffman_codec; + +#[cfg(feature = "rangecoder")] +pub mod rangecoder; + +#[cfg(feature = "mtf")] +pub mod mtf; + +#[cfg(feature = "bwt")] +pub mod bwt; + #[cfg(feature = "factory")] pub mod factory; diff --git a/src/mtf/mod.rs b/src/mtf/mod.rs new file mode 100644 index 0000000..d1f322e --- /dev/null +++ b/src/mtf/mod.rs @@ -0,0 +1,431 @@ +//! Move-To-Front (MTF) transform — a reversible, length-preserving filter. +//! +//! This is a *filter*, not a compressor: it makes data more compressible +//! for a downstream entropy coder (it does not shrink the data on its own — +//! output length always equals input length). MTF is the classic transform +//! that sits between the BWT and the entropy stage inside bzip2; here it is +//! exposed standalone. +//! +//! ## Transform +//! +//! A 256-entry table is initialised to the identity permutation +//! `0, 1, …, 255`. The encoder, for each input byte `b`: +//! +//! 1. finds the current index `i` of `b` in the table, +//! 2. emits `i`, then +//! 3. moves `b` to the front of the table (index 0), shifting the bytes +//! that were ahead of it one slot back. +//! +//! ```text +//! out[k] = position of in[k] in table; then table.move_to_front(in[k]) +//! ``` +//! +//! The decoder is the exact inverse: for each input byte `i` it emits +//! `table[i]`, then moves that symbol to the front: +//! +//! ```text +//! out[k] = table[in[k]]; then table.move_to_front(out[k]) +//! ``` +//! +//! Because the encoder and decoder keep their tables in lock-step, the +//! decoder reconstructs the original byte before it needs to mutate the +//! table, so `decode ∘ encode` is the identity for *every* input. +//! +//! ## Why it helps +//! +//! Recently-seen bytes live near the front of the table, so a run of one +//! symbol encodes as a single high index followed by zeros, and locally +//! skewed data maps to a stream dominated by small values. That heavily +//! biased distribution is exactly what a Huffman / range coder exploits. +//! +//! ## State +//! +//! The only state is the 256-byte table, carried across calls. The filter +//! is therefore fully streaming: feeding one byte or a megabyte per call +//! produces identical output, since each output byte depends only on the +//! input bytes seen so far. No input buffering is required and there is no +//! header or trailer. +//! +//! References: +//! * J. L. Bentley, D. D. Sleator, R. E. Tarjan, V. K. Wei, +//! "A Locally Adaptive Data Compression Scheme" (CACM, 1986). +//! * The MTF stage of the bzip2 pipeline. Implemented clean-room from the +//! transform described above. + +#![cfg_attr(docsrs, doc(cfg(feature = "mtf")))] + +use crate::error::Error; +use crate::traits::{Algorithm, RawDecoder, RawEncoder, RawProgress}; + +/// Zero-sized marker type implementing [`Algorithm`] for the MTF filter. +#[derive(Debug, Clone, Copy, Default)] +pub struct Mtf; + +impl Algorithm for Mtf { + const NAME: &'static str = "mtf"; + type Encoder = Encoder; + type Decoder = Decoder; + type EncoderConfig = (); + type DecoderConfig = (); + fn encoder_with(_cfg: ()) -> Encoder { + Encoder::new() + } + fn decoder_with(_cfg: ()) -> Decoder { + Decoder::new() + } +} + +/// The 256-entry move-to-front table. +/// +/// `table[k]` is the symbol currently at rank `k`; rank 0 is the front. +/// Starts as the identity permutation, which both directions share so they +/// stay in lock-step across the whole stream. +#[derive(Debug, Clone)] +struct Table { + table: [u8; 256], +} + +impl Table { + const fn new() -> Self { + // `0, 1, …, 255` — built in a const-friendly loop. + let mut table = [0u8; 256]; + let mut i = 0usize; + while i < 256 { + table[i] = i as u8; + i += 1; + } + Self { table } + } + + fn reset(&mut self) { + let mut i = 0usize; + while i < 256 { + self.table[i] = i as u8; + i += 1; + } + } + + /// Encode one byte: return its current rank and move it to the front. + fn encode_byte(&mut self, b: u8) -> u8 { + // Linear scan of 256 entries — `b` is guaranteed present (the table + // is always a permutation of all byte values), so the loop always + // finds it. + let mut rank = 0usize; + while self.table[rank] != b { + rank += 1; + } + self.move_to_front(rank); + rank as u8 + } + + /// Decode one byte: return the symbol at `rank` and move it to the front. + fn decode_byte(&mut self, rank: u8) -> u8 { + let b = self.table[rank as usize]; + self.move_to_front(rank as usize); + b + } + + /// Move the entry at `rank` to the front, shifting `[0, rank)` back one. + fn move_to_front(&mut self, rank: usize) { + if rank == 0 { + return; + } + let b = self.table[rank]; + self.table.copy_within(0..rank, 1); + self.table[0] = b; + } +} + +// ─── encoder ───────────────────────────────────────────────────────────── + +/// Streaming MTF-filter encoder. +#[derive(Debug, Clone)] +pub struct Encoder { + table: Table, +} + +impl Encoder { + /// Construct an encoder with a fresh identity table. + pub const fn new() -> Self { + Self { + table: Table::new(), + } + } +} + +impl Default for Encoder { + fn default() -> Self { + Self::new() + } +} + +impl RawEncoder for Encoder { + fn raw_encode(&mut self, input: &[u8], output: &mut [u8]) -> Result { + let n = input.len().min(output.len()); + for i in 0..n { + output[i] = self.table.encode_byte(input[i]); + } + Ok(RawProgress { + consumed: n, + written: n, + done: false, + }) + } + + fn raw_finish(&mut self, _output: &mut [u8]) -> Result { + // 1:1 transform with no trailer: once all input is consumed the + // stream is complete and there is nothing buffered to flush. + Ok(RawProgress { + consumed: 0, + written: 0, + done: true, + }) + } + + fn raw_reset(&mut self) { + self.table.reset(); + } +} + +// ─── decoder ───────────────────────────────────────────────────────────── + +/// Streaming MTF-filter decoder (inverse of [`Encoder`]). +#[derive(Debug, Clone)] +pub struct Decoder { + table: Table, +} + +impl Decoder { + /// Construct a decoder with a fresh identity table. + pub const fn new() -> Self { + Self { + table: Table::new(), + } + } +} + +impl Default for Decoder { + fn default() -> Self { + Self::new() + } +} + +impl RawDecoder for Decoder { + fn raw_decode(&mut self, input: &[u8], output: &mut [u8]) -> Result { + let n = input.len().min(output.len()); + for i in 0..n { + output[i] = self.table.decode_byte(input[i]); + } + Ok(RawProgress { + consumed: n, + written: n, + done: false, + }) + } + + fn raw_finish(&mut self, _output: &mut [u8]) -> Result { + Ok(RawProgress { + consumed: 0, + written: 0, + done: true, + }) + } + + fn raw_reset(&mut self) { + self.table.reset(); + } +} + +// ─── tests ─────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + use crate::traits::{Decoder as _, Encoder as _, Status}; + use alloc::vec; + use alloc::vec::Vec; + + /// One-shot encode: assumes `output` is large enough (it always is for a + /// length-preserving filter when sized to the input). + fn encode_all(data: &[u8]) -> Vec { + let mut enc = Mtf::encoder(); + let mut out = vec![0u8; data.len()]; + let (p, status) = enc.encode(data, &mut out).unwrap(); + assert_eq!(p.consumed, data.len()); + assert_eq!(p.written, data.len()); + assert_eq!(status, Status::InputEmpty); + let (fp, fstatus) = enc.finish(&mut []).unwrap(); + assert_eq!(fp.written, 0); + assert_eq!(fstatus, Status::StreamEnd); + out + } + + fn decode_all(data: &[u8]) -> Vec { + let mut dec = Mtf::decoder(); + let mut out = vec![0u8; data.len()]; + let (p, _status) = dec.decode(data, &mut out).unwrap(); + assert_eq!(p.consumed, data.len()); + assert_eq!(p.written, data.len()); + out + } + + fn assert_round_trip(data: &[u8]) { + let encoded = encode_all(data); + assert_eq!( + encoded.len(), + data.len(), + "filter must be length-preserving" + ); + let decoded = decode_all(&encoded); + assert_eq!(decoded, data, "round-trip must be byte-identical"); + } + + #[test] + fn round_trip_empty() { + assert_round_trip(&[]); + } + + #[test] + fn round_trip_single_byte() { + assert_round_trip(&[0x00]); + assert_round_trip(&[0x7F]); + assert_round_trip(&[0xFF]); + } + + #[test] + fn round_trip_repeated_bytes() { + assert_round_trip(&[0x41; 1000]); + assert_round_trip(&[0xFF; 257]); + } + + #[test] + fn round_trip_english_text() { + let text = b"the quick brown fox jumps over the lazy dog. \ + The QUICK Brown Fox Jumps Over The Lazy Dog!"; + assert_round_trip(text); + } + + #[test] + fn round_trip_all_byte_values() { + let data: Vec = (0..=255u8).collect(); + assert_round_trip(&data); + // And the reverse ordering. + let rev: Vec = (0..=255u8).rev().collect(); + assert_round_trip(&rev); + } + + #[test] + fn round_trip_pseudo_random() { + // Deterministic xorshift PRNG — no deps. + let mut state = 0x2545_F491_4F6C_DD1Du64; + let mut data = Vec::with_capacity(4096); + for _ in 0..4096 { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + data.push((state & 0xFF) as u8); + } + assert_round_trip(&data); + } + + /// Feeding the input one byte at a time (or in small chunks) must yield + /// the same output as one shot — proves the table state persists across + /// calls. + #[test] + fn streaming_matches_one_shot() { + let mut state = 0x9E37_79B9_7F4A_7C15u64; + let mut data = Vec::with_capacity(3000); + for _ in 0..3000 { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + data.push((state & 0xFF) as u8); + } + + let one_shot = encode_all(&data); + + // Encode in chunks of varying small sizes through a single encoder. + for &chunk in &[1usize, 2, 3, 7, 64] { + let mut enc = Mtf::encoder(); + let mut streamed = Vec::with_capacity(data.len()); + let mut scratch = vec![0u8; chunk]; + let mut off = 0; + while off < data.len() { + let end = (off + chunk).min(data.len()); + let (p, _) = enc.encode(&data[off..end], &mut scratch).unwrap(); + assert_eq!(p.consumed, end - off); + streamed.extend_from_slice(&scratch[..p.written]); + off += p.consumed; + } + assert_eq!(streamed, one_shot, "chunk size {chunk} diverged"); + + // And decoding the streamed output (also chunked) recovers data. + let mut dec = Mtf::decoder(); + let mut decoded = Vec::with_capacity(data.len()); + let mut doff = 0; + while doff < streamed.len() { + let end = (doff + chunk).min(streamed.len()); + let (p, _) = dec.decode(&streamed[doff..end], &mut scratch).unwrap(); + decoded.extend_from_slice(&scratch[..p.written]); + doff += p.consumed; + } + assert_eq!(decoded, data, "chunked decode (chunk {chunk}) diverged"); + } + } + + /// A long run of a single byte after a fresh table: the first occurrence + /// emits that byte's rank, every subsequent one emits 0. This is the + /// low/zero-byte property that makes MTF useful. + #[test] + fn repetitive_data_yields_low_bytes() { + let data = [0x42u8; 500]; + let encoded = encode_all(&data); + assert_eq!(encoded[0], 0x42, "first occurrence emits the symbol's rank"); + assert!( + encoded[1..].iter().all(|&b| b == 0), + "subsequent identical bytes must encode as 0" + ); + + // Mixed but locally-repetitive data should be dominated by zeros. + let mixed: Vec = b"aaaabbbbccccddddaaaabbbb".to_vec(); + let enc_mixed = encode_all(&mixed); + let zeros = enc_mixed.iter().filter(|&&b| b == 0).count(); + assert!( + zeros >= mixed.len() / 2, + "locally repetitive data should be majority zeros, got {zeros}/{}", + mixed.len() + ); + } + + /// Length is preserved and a spread of inputs round-trips cleanly. + #[test] + fn length_preserving_across_sizes() { + for len in [0usize, 1, 2, 17, 256, 257, 1024] { + let data: Vec = (0..len).map(|i| (i * 31 + 7) as u8).collect(); + let encoded = encode_all(&data); + assert_eq!(encoded.len(), data.len()); + let decoded = decode_all(&encoded); + assert_eq!(decoded, data); + } + } + + /// `reset` returns the encoder to the identity table so a second stream + /// encodes independently of the first. + #[test] + fn reset_restores_initial_state() { + let first = b"hello world"; + let second = b"different payload entirely"; + + let mut enc = Mtf::encoder(); + let mut out = vec![0u8; 64]; + + let (_p, _) = enc.encode(first, &mut out).unwrap(); + enc.reset(); + let (p, _) = enc.encode(second, &mut out).unwrap(); + let after_reset = out[..p.written].to_vec(); + + // Encoding `second` on a fresh encoder must match the post-reset run. + let fresh = encode_all(second); + assert_eq!(after_reset, fresh); + } +} diff --git a/src/qpack/dynamic_table.rs b/src/qpack/dynamic_table.rs new file mode 100644 index 0000000..3dce22b --- /dev/null +++ b/src/qpack/dynamic_table.rs @@ -0,0 +1,156 @@ +//! QPACK dynamic table — RFC 9204 §3.2. +//! +//! The dynamic table is a FIFO of `(name, value)` entries with byte-size +//! accounting (§3.2.1: each entry costs `name.len() + value.len() + 32`). +//! Entries are addressed by an **absolute index** that is assigned at insertion +//! and never changes: the first inserted entry is absolute index 0, the next 1, +//! and so on. Eviction drops the lowest absolute indices. The number of +//! insertions ever performed is the *Insert Count* (§2.1.4), which equals the +//! absolute index of the next entry to be inserted. +//! +//! Field-section and encoder-stream instructions reference entries with +//! *relative* and *post-base* indices, which this module converts to absolute +//! indices (see [`DynamicTable::relative_to_absolute`] and +//! [`DynamicTable::post_base_to_absolute`]). + +extern crate alloc; +use alloc::collections::VecDeque; +use alloc::vec::Vec; + +/// Per-entry overhead added to name+value lengths for size accounting +/// (RFC 9204 §3.2.1). +pub(crate) const ENTRY_OVERHEAD: usize = 32; + +/// QPACK dynamic table. `entries` holds live entries oldest-first; the oldest +/// live entry has absolute index `dropped` (the count of evicted entries), and +/// the youngest has absolute index `insert_count - 1`. +#[derive(Debug, Default)] +pub(crate) struct DynamicTable { + entries: VecDeque<(Vec, Vec)>, + /// Current byte size (sum of entry sizes). + size: usize, + /// Current capacity (max byte size). Starts at 0 — the encoder must send a + /// Set Dynamic Table Capacity instruction before any insert (§3.2.3). + capacity: usize, + /// Total number of entries ever inserted (the Insert Count, §2.1.4). + insert_count: usize, + /// Total number of entries ever evicted (absolute index of the oldest live + /// entry, or of the next insert if the table is empty). + dropped: usize, +} + +impl DynamicTable { + pub(crate) fn new() -> Self { + DynamicTable::default() + } + + /// Entry byte cost per §3.2.1. + pub(crate) fn entry_size(name: &[u8], value: &[u8]) -> usize { + name.len() + value.len() + ENTRY_OVERHEAD + } + + /// The Insert Count: number of entries ever inserted (§2.1.4). Also the + /// absolute index that the next inserted entry will receive. + pub(crate) fn insert_count(&self) -> usize { + self.insert_count + } + + /// Current byte size of the table. + #[cfg(test)] + pub(crate) fn size(&self) -> usize { + self.size + } + + /// Current capacity. + #[cfg(test)] + pub(crate) fn capacity(&self) -> usize { + self.capacity + } + + /// Number of live entries. + #[cfg(test)] + pub(crate) fn len(&self) -> usize { + self.entries.len() + } + + /// Set the table capacity (§3.2.3), evicting entries as needed. Returns + /// `false` if `new_capacity` exceeds `max` (the caller's connection limit). + pub(crate) fn set_capacity(&mut self, new_capacity: usize, max: usize) -> bool { + if new_capacity > max { + return false; + } + self.capacity = new_capacity; + self.evict_to_fit(0); + true + } + + fn evict_to_fit(&mut self, incoming: usize) { + while self.size + incoming > self.capacity { + match self.entries.pop_front() { + Some((n, v)) => { + self.size -= Self::entry_size(&n, &v); + self.dropped += 1; + } + None => break, + } + } + } + + /// Whether an entry of `(name, value)` can be inserted given the current + /// capacity and live contents (§3.2.2: an insert that cannot fit even after + /// evicting everything is an error). Used by the decoder to reject bad + /// encoder streams. + pub(crate) fn can_insert(&self, name: &[u8], value: &[u8]) -> bool { + Self::entry_size(name, value) <= self.capacity + } + + /// Insert `(name, value)` at the next absolute index (§3.2.2), evicting + /// older entries as needed. Returns the absolute index assigned, or `None` + /// if the entry cannot fit even in an empty table at the current capacity. + pub(crate) fn insert(&mut self, name: &[u8], value: &[u8]) -> Option { + let need = Self::entry_size(name, value); + if need > self.capacity { + return None; + } + self.evict_to_fit(need); + let abs = self.insert_count; + self.entries.push_back((name.to_vec(), value.to_vec())); + self.size += need; + self.insert_count += 1; + Some(abs) + } + + /// Look up an entry by **absolute** index. Returns `None` if the index has + /// been evicted or never inserted. + pub(crate) fn get_absolute(&self, abs: usize) -> Option<(&[u8], &[u8])> { + if abs < self.dropped || abs >= self.insert_count { + return None; + } + self.entries + .get(abs - self.dropped) + .map(|(n, v)| (n.as_slice(), v.as_slice())) + } + + /// Convert a relative index (used on the encoder stream and in + /// Duplicate/Insert-with-Name-Reference: relative to the most recent + /// insertion) to an absolute index. Relative index 0 is the newest entry, + /// i.e. absolute `insert_count - 1` (§3.2.5). + pub(crate) fn relative_to_absolute_encoder(&self, rel: usize) -> Option { + // abs = insert_count - 1 - rel + let top = self.insert_count.checked_sub(1)?; + top.checked_sub(rel) + } + + /// Convert a relative index in a field section (relative to `base`) to an + /// absolute index. Relative index 0 is `base - 1` (§3.2.5). + pub(crate) fn field_relative_to_absolute(base: usize, rel: usize) -> Option { + // abs = base - rel - 1 + base.checked_sub(rel)?.checked_sub(1) + } + + /// Convert a post-base index in a field section to an absolute index. + /// Post-base index 0 is `base` (§3.2.6). + pub(crate) fn post_base_to_absolute(base: usize, idx: usize) -> Option { + base.checked_add(idx) + } +} diff --git a/src/qpack/integer.rs b/src/qpack/integer.rs new file mode 100644 index 0000000..db82c5b --- /dev/null +++ b/src/qpack/integer.rs @@ -0,0 +1,67 @@ +//! QPACK prefixed-integer representation — RFC 9204 §4.1.1. +//! +//! Identical in form to the HPACK integer (RFC 7541 §5.1): an `N`-bit prefix +//! shares its byte with preceding flag bits, values below `2^N − 1` fit in the +//! prefix, and larger values store `2^N − 1` in the prefix plus a +//! little-endian base-128 continuation (high bit = "more"). HPACK's copy is +//! private to that module, so QPACK carries its own (the spec mandates the +//! same primitive but the modules stay decoupled). + +extern crate alloc; +use alloc::vec::Vec; + +use crate::error::Error; + +/// Encode `value` with an `n`-bit prefix (`1 <= n <= 8`), OR-ing `flags` +/// (the high `8 - n` bits, already positioned) into the prefix byte. Appends +/// to `out`. +pub(crate) fn encode_int(out: &mut Vec, value: usize, n: u32, flags: u8) { + debug_assert!((1..=8).contains(&n)); + let max_prefix = (1usize << n) - 1; + if value < max_prefix { + out.push(flags | value as u8); + return; + } + out.push(flags | max_prefix as u8); + let mut v = value - max_prefix; + while v >= 128 { + out.push((v & 0x7f) as u8 | 0x80); + v >>= 7; + } + out.push(v as u8); +} + +/// Decode an `n`-bit-prefix integer starting at `buf[pos]` (`1 <= n <= 8`). +/// +/// Returns `(value, next_pos)`. Prefix flag bits above the low `n` bits of +/// `buf[pos]` are ignored (the caller already dispatched on them). Rejects +/// continuations that would overflow `usize` or run past the buffer +/// (`Error::Corrupt` / `Error::UnexpectedEnd`) — the standard integer-overflow +/// guard (§4.1.1 mandates support for values up to at least 62 bits; this +/// caps at `usize` and rejects anything longer rather than wrapping). +pub(crate) fn decode_int(buf: &[u8], pos: usize, n: u32) -> Result<(usize, usize), Error> { + debug_assert!((1..=8).contains(&n)); + let max_prefix = (1usize << n) - 1; + let first = *buf.get(pos).ok_or(Error::UnexpectedEnd)? as usize; + let value = first & max_prefix; + let mut p = pos + 1; + if value < max_prefix { + return Ok((value, p)); + } + let mut value = value; + let mut shift = 0u32; + loop { + let b = *buf.get(p).ok_or(Error::UnexpectedEnd)? as usize; + p += 1; + if shift >= usize::BITS { + return Err(Error::Corrupt); + } + let add = (b & 0x7f).checked_shl(shift).ok_or(Error::Corrupt)?; + value = value.checked_add(add).ok_or(Error::Corrupt)?; + if b & 0x80 == 0 { + break; + } + shift += 7; + } + Ok((value, p)) +} diff --git a/src/qpack/mod.rs b/src/qpack/mod.rs new file mode 100644 index 0000000..396c85e --- /dev/null +++ b/src/qpack/mod.rs @@ -0,0 +1,506 @@ +//! HTTP/3 QPACK header compression — [RFC 9204]. +//! +//! QPACK is HTTP/3's header-compression format. Like [HPACK](crate::hpack) it +//! compresses an ordered list of `(name, value)` header fields against a +//! static table (99 common fields, RFC 9204 Appendix A) and a per-connection +//! dynamic table — but it is designed for QUIC's out-of-order streams, so the +//! dynamic table is mutated by a **separate, ordered encoder stream** while +//! field sections (the per-request header blocks) reference it through a +//! prefix that names how many insertions the decoder must have seen first. +//! +//! This module reuses HPACK's machinery where the specs agree: the string +//! Huffman code is identical, so [`HeaderField`] and the +//! [`crate::hpack::huffman`] primitive come straight from +//! [`crate::hpack`]. The prefixed-integer and table primitives are +//! QPACK-specific (different index spaces) and live here. +//! +//! # Decoder — full +//! +//! [`QpackDecoder`] implements the complete decode path: the static table, the +//! dynamic table built from the encoder stream +//! ([`feed_encoder_stream`](QpackDecoder::feed_encoder_stream): Set Dynamic +//! Table Capacity, Insert with Name Reference, Insert with Literal Name, +//! Duplicate), and every field-line representation +//! ([`decode_field_section`](QpackDecoder::decode_field_section): indexed +//! static/dynamic/post-base, literal with static/dynamic/post-base name +//! reference, literal with literal name). +//! +//! Because this is a synchronous API it cannot *block* on a field section that +//! references dynamic entries not yet inserted: if a section's Required Insert +//! Count exceeds the decoder's current Insert Count, it returns +//! [`Error::Corrupt`] rather than waiting. Feed the encoder stream first. +//! +//! # Encoder — static-table + literal only +//! +//! [`QpackEncoder`] emits fully spec-compliant, interoperable field sections +//! that **never insert into the dynamic table**: the prefix is always Required +//! Insert Count = 0, Base = 0, and fields are coded with static-table indexed / +//! name-reference representations or literal names. This needs no encoder +//! stream and never blocks a peer decoder. Dynamic-table *encoding* (driving +//! the encoder stream, post-base references, eviction policy) is a deliberate +//! future extension; the decoder here already accepts a peer that does it. +//! +//! ``` +//! use compcol::qpack::{QpackEncoder, QpackDecoder}; +//! use compcol::hpack::HeaderField; +//! +//! let mut enc = QpackEncoder::new(); +//! let block = enc.encode_field_section(&[ +//! HeaderField::new(b":path", b"/index.html"), +//! HeaderField::new(b"custom", b"value"), +//! ]); +//! let mut dec = QpackDecoder::new(); +//! let out = dec.decode_field_section(&block).unwrap(); +//! assert_eq!(out[0].name, b":path"); +//! assert_eq!(out[1].value, b"value"); +//! ``` +//! +//! Clean-room from RFC 9204 (the static table is transcribed from Appendix A; +//! the string Huffman table is HPACK's, shared per the spec). +//! +//! [RFC 9204]: https://www.rfc-editor.org/rfc/rfc9204 + +#![cfg_attr(docsrs, doc(cfg(feature = "qpack")))] + +extern crate alloc; +use alloc::vec::Vec; + +use crate::error::Error; +use crate::hpack::HeaderField; +use crate::hpack::huffman; + +mod dynamic_table; +mod integer; +mod static_table; + +use dynamic_table::DynamicTable; +use integer::{decode_int, encode_int}; + +/// QPACK's default maximum dynamic-table capacity used by [`QpackDecoder::new`] +/// when no explicit bound is given. A peer's `SETTINGS_QPACK_MAX_TABLE_CAPACITY` +/// would normally set this; 4096 mirrors the HPACK default and is a safe +/// general-purpose ceiling. +pub const DEFAULT_MAX_TABLE_CAPACITY: usize = 4096; + +// ─── encoder ─────────────────────────────────────────────────────────────── + +/// QPACK encoder (static-table + literal only). +/// +/// Encodes each field section against the static table, emitting a Required +/// Insert Count = 0 / Base = 0 prefix and never inserting into the dynamic +/// table. This is stateless across calls and fully interoperable. See the +/// [module docs](crate::qpack) for why dynamic-table encoding is out of scope. +#[derive(Debug)] +pub struct QpackEncoder { + use_huffman: bool, +} + +impl Default for QpackEncoder { + fn default() -> Self { + Self::new() + } +} + +impl QpackEncoder { + /// New encoder with Huffman string coding enabled. + pub fn new() -> Self { + QpackEncoder { use_huffman: true } + } + + /// Enable/disable Huffman coding of string literals (default on). When on, + /// the shorter of Huffman/raw is chosen per string (§4.1.2). + pub fn set_huffman(&mut self, on: bool) { + self.use_huffman = on; + } + + /// Encode one field section. The returned block begins with the §4.5.1 + /// prefix (Required Insert Count = 0, Base = 0 — encoded as two `0x00` + /// bytes) followed by one representation per field. + pub fn encode_field_section(&mut self, fields: &[HeaderField]) -> Vec { + let mut out = Vec::new(); + // §4.5.1 prefix. With no dynamic-table references, Required Insert + // Count encodes as 0 (8-bit prefix) and Delta Base as 0 with Sign 0. + out.push(0x00); // Required Insert Count = 0 + out.push(0x00); // S = 0, Delta Base = 0 → Base = 0 + for f in fields { + self.encode_field(&mut out, f); + } + out + } + + fn encode_field(&self, out: &mut Vec, f: &HeaderField) { + match static_table::find(&f.name, &f.value) { + Some((idx, true)) if !f.sensitive => { + // §4.5.2 Indexed Field Line, static table: 1 T(=1) index(6+). + encode_int(out, idx, 6, 0b1100_0000); + } + Some((idx, _)) => { + // §4.5.4 Literal Field Line with Name Reference, static table. + // Pattern 0 1 N T, 4-bit name index. T=1 (static). + let n_bit = if f.sensitive { 0b0010_0000 } else { 0 }; + encode_int(out, idx, 4, 0b0101_0000 | n_bit); + self.emit_string(out, &f.value, 7, 0); + } + None => { + // §4.5.6 Literal Field Line with Literal Name. Pattern + // 0 0 1 N H, 3-bit name length. emit_string handles the H bit. + let n_bit = if f.sensitive { 0b0001_0000 } else { 0 }; + self.emit_string(out, &f.name, 3, 0b0010_0000 | n_bit); + self.emit_string(out, &f.value, 7, 0); + } + } + } + + /// Emit a string literal (§4.1.2) with an `n`-bit length prefix. `pattern` + /// holds the fixed high bits already positioned; the Huffman (`H`) flag is + /// the bit at value `1 << n` and is OR-ed in when Huffman is chosen. + fn emit_string(&self, out: &mut Vec, s: &[u8], n: u32, pattern: u8) { + let h_flag = 1u8 << n; + if self.use_huffman && huffman::encoded_len(s) < s.len() { + let coded = huffman::encode(s); + encode_int(out, coded.len(), n, pattern | h_flag); + out.extend_from_slice(&coded); + } else { + encode_int(out, s.len(), n, pattern); + out.extend_from_slice(s); + } + } +} + +// ─── decoder ─────────────────────────────────────────────────────────────── + +/// QPACK decoder (full: static + dynamic tables + all field representations). +/// +/// Feed the encoder stream with +/// [`feed_encoder_stream`](Self::feed_encoder_stream) (which builds the dynamic +/// table) before decoding the field sections that reference it with +/// [`decode_field_section`](Self::decode_field_section). The dynamic table and +/// Insert Count persist across calls for the lifetime of the connection. +#[derive(Debug)] +pub struct QpackDecoder { + table: DynamicTable, + /// Connection limit on dynamic-table capacity + /// (`SETTINGS_QPACK_MAX_TABLE_CAPACITY`, §3.2.3). A Set Dynamic Table + /// Capacity instruction may not exceed this. + max_capacity: usize, +} + +impl Default for QpackDecoder { + fn default() -> Self { + Self::new() + } +} + +impl QpackDecoder { + /// New decoder allowing a dynamic table up to + /// [`DEFAULT_MAX_TABLE_CAPACITY`] bytes. + pub fn new() -> Self { + Self::with_max_table_capacity(DEFAULT_MAX_TABLE_CAPACITY) + } + + /// New decoder whose dynamic-table capacity ceiling is `max` bytes (the + /// value it would advertise as `SETTINGS_QPACK_MAX_TABLE_CAPACITY`). The + /// table starts empty at capacity 0 until the encoder stream raises it. + pub fn with_max_table_capacity(max: usize) -> Self { + QpackDecoder { + table: DynamicTable::new(), + max_capacity: max, + } + } + + /// Process encoder-stream instructions (§4.3), mutating the dynamic table: + /// Set Dynamic Table Capacity, Insert with Name Reference, Insert with + /// Literal Name, and Duplicate. Returns [`Error::Corrupt`] on a malformed + /// instruction, a bad table reference, an over-limit capacity, or an insert + /// that cannot fit; [`Error::UnexpectedEnd`] on truncation. + pub fn feed_encoder_stream(&mut self, data: &[u8]) -> Result<(), Error> { + let mut pos = 0; + while pos < data.len() { + let b = data[pos]; + if b & 0b1000_0000 != 0 { + // §4.3.2 Insert with Name Reference: 1 T name-index(6+). + let t_static = b & 0b0100_0000 != 0; + let (name_idx, np) = decode_int(data, pos, 6)?; + pos = np; + let (value, np) = read_string(data, pos, 7)?; + pos = np; + let name = self.resolve_insert_name(name_idx, t_static)?; + self.do_insert(&name, &value)?; + } else if b & 0b0100_0000 != 0 { + // §4.3.3 Insert with Literal Name: 0 1 H name-len(5+). + let (name, np) = read_string(data, pos, 5)?; + pos = np; + let (value, np) = read_string(data, pos, 7)?; + pos = np; + self.do_insert(&name, &value)?; + } else if b & 0b0010_0000 != 0 { + // §4.3.1 Set Dynamic Table Capacity: 0 0 1 capacity(5+). + let (cap, np) = decode_int(data, pos, 5)?; + pos = np; + if !self.table.set_capacity(cap, self.max_capacity) { + return Err(Error::Corrupt); + } + } else { + // §4.3.4 Duplicate: 0 0 0 index(5+) (relative index). + let (rel, np) = decode_int(data, pos, 5)?; + pos = np; + let abs = self + .table + .relative_to_absolute_encoder(rel) + .ok_or(Error::Corrupt)?; + let (n, v) = self.table.get_absolute(abs).ok_or(Error::Corrupt)?; + let (n, v) = (n.to_vec(), v.to_vec()); + self.do_insert(&n, &v)?; + } + } + Ok(()) + } + + /// Resolve the name for an Insert with Name Reference (§4.3.2): static index + /// or dynamic relative index (relative to the most recent insertion). + fn resolve_insert_name(&self, idx: usize, t_static: bool) -> Result, Error> { + if t_static { + let (n, _) = static_table::get(idx).ok_or(Error::Corrupt)?; + Ok(n.to_vec()) + } else { + let abs = self + .table + .relative_to_absolute_encoder(idx) + .ok_or(Error::Corrupt)?; + let (n, _) = self.table.get_absolute(abs).ok_or(Error::Corrupt)?; + Ok(n.to_vec()) + } + } + + fn do_insert(&mut self, name: &[u8], value: &[u8]) -> Result<(), Error> { + if !self.table.can_insert(name, value) { + return Err(Error::Corrupt); + } + self.table.insert(name, value).ok_or(Error::Corrupt)?; + Ok(()) + } + + /// Decode one field section (§4.5) into its field list. Returns + /// [`Error::Corrupt`] on a malformed representation, a bad table reference, + /// or a Required Insert Count that exceeds what has been inserted so far + /// (a blocked reference this synchronous API cannot wait on); + /// [`Error::UnexpectedEnd`] on truncation. + pub fn decode_field_section(&mut self, block: &[u8]) -> Result, Error> { + // §4.5.1 prefix. + let (req_insert_count, mut pos) = self.decode_required_insert_count(block)?; + let base = self.decode_base(block, &mut pos, req_insert_count)?; + + // A field section may only reference dynamic entries with absolute + // index < Required Insert Count, and the decoder must have inserted at + // least that many. We can't block, so reject if it hasn't. + if req_insert_count > self.table.insert_count() { + return Err(Error::Corrupt); + } + + let mut fields = Vec::new(); + while pos < block.len() { + let b = block[pos]; + if b & 0b1000_0000 != 0 { + // §4.5.2 Indexed Field Line: 1 T index(6+). + let t_static = b & 0b0100_0000 != 0; + let (idx, np) = decode_int(block, pos, 6)?; + pos = np; + let (n, v) = self.lookup_indexed(idx, t_static, base, req_insert_count)?; + fields.push(HeaderField::new(n.as_slice(), v.as_slice())); + } else if b & 0b0100_0000 != 0 { + // §4.5.4 Literal Field Line with Name Reference: 0 1 N T idx(4+). + let sensitive = b & 0b0010_0000 != 0; + let t_static = b & 0b0001_0000 != 0; + let (idx, np) = decode_int(block, pos, 4)?; + pos = np; + let name = self.lookup_name_ref(idx, t_static, base, req_insert_count)?; + let (value, np) = read_string(block, pos, 7)?; + pos = np; + fields.push(HeaderField { + name, + value, + sensitive, + }); + } else if b & 0b0010_0000 != 0 { + // §4.5.6 Literal Field Line with Literal Name: 0 0 1 N H len(3+). + let sensitive = b & 0b0001_0000 != 0; + let (name, np) = read_string(block, pos, 3)?; + pos = np; + let (value, np) = read_string(block, pos, 7)?; + pos = np; + fields.push(HeaderField { + name, + value, + sensitive, + }); + } else if b & 0b0001_0000 != 0 { + // §4.5.3 Indexed Field Line with Post-Base Index: 0 0 0 1 idx(4+). + let (idx, np) = decode_int(block, pos, 4)?; + pos = np; + let abs = DynamicTable::post_base_to_absolute(base, idx).ok_or(Error::Corrupt)?; + let (n, v) = self.lookup_dynamic_abs(abs, req_insert_count)?; + fields.push(HeaderField::new(n.as_slice(), v.as_slice())); + } else { + // §4.5.5 Literal Field Line with Post-Base Name Reference: + // 0 0 0 0 N idx(3+). + let sensitive = b & 0b0000_1000 != 0; + let (idx, np) = decode_int(block, pos, 3)?; + pos = np; + let abs = DynamicTable::post_base_to_absolute(base, idx).ok_or(Error::Corrupt)?; + let (n, _) = self.lookup_dynamic_abs(abs, req_insert_count)?; + let (value, np) = read_string(block, pos, 7)?; + pos = np; + fields.push(HeaderField { + name: n, + value, + sensitive, + }); + } + } + Ok(fields) + } + + /// Decode the Required Insert Count (§4.5.1): an 8-bit-prefix integer + /// `EncInsertCount` reconstructed against the current Insert Count. + fn decode_required_insert_count(&self, block: &[u8]) -> Result<(usize, usize), Error> { + let (enc, pos) = decode_int(block, 0, 8)?; + if enc == 0 { + return Ok((0, pos)); + } + let max_entries = self.max_capacity / 32; + let full_range = 2 * max_entries; + if full_range == 0 || enc > full_range { + return Err(Error::Corrupt); + } + let total_inserts = self.table.insert_count(); + let max_value = total_inserts + max_entries; + let max_wrapped = (max_value / full_range) * full_range; + let mut req = max_wrapped + enc - 1; + if req > max_value { + if req <= full_range { + return Err(Error::Corrupt); + } + req -= full_range; + } + if req == 0 { + return Err(Error::Corrupt); + } + Ok((req, pos)) + } + + /// Decode the Base (§4.5.1): a sign bit + 7-bit-prefix Delta Base. + fn decode_base( + &self, + block: &[u8], + pos: &mut usize, + req_insert_count: usize, + ) -> Result { + let sign = *block.get(*pos).ok_or(Error::UnexpectedEnd)? & 0x80 != 0; + let (delta, np) = decode_int(block, *pos, 7)?; + *pos = np; + if sign { + // Base = ReqInsertCount - DeltaBase - 1; reject negative. + req_insert_count + .checked_sub(delta) + .and_then(|x| x.checked_sub(1)) + .ok_or(Error::Corrupt) + } else { + req_insert_count.checked_add(delta).ok_or(Error::Corrupt) + } + } + + /// §4.5.2 indexed lookup: static or dynamic (relative to `base`). + fn lookup_indexed( + &self, + idx: usize, + t_static: bool, + base: usize, + req_insert_count: usize, + ) -> Result<(Vec, Vec), Error> { + if t_static { + let (n, v) = static_table::get(idx).ok_or(Error::Corrupt)?; + Ok((n.to_vec(), v.to_vec())) + } else { + let abs = DynamicTable::field_relative_to_absolute(base, idx).ok_or(Error::Corrupt)?; + self.lookup_dynamic_abs(abs, req_insert_count) + } + } + + /// §4.5.4 name-reference lookup: static or dynamic (relative to `base`). + fn lookup_name_ref( + &self, + idx: usize, + t_static: bool, + base: usize, + req_insert_count: usize, + ) -> Result, Error> { + if t_static { + let (n, _) = static_table::get(idx).ok_or(Error::Corrupt)?; + Ok(n.to_vec()) + } else { + let abs = DynamicTable::field_relative_to_absolute(base, idx).ok_or(Error::Corrupt)?; + Ok(self.lookup_dynamic_abs(abs, req_insert_count)?.0) + } + } + + /// Look up a dynamic entry by absolute index, enforcing the field section's + /// Required Insert Count bound (§4.5: a reference may not name an entry with + /// absolute index >= Required Insert Count). + fn lookup_dynamic_abs( + &self, + abs: usize, + req_insert_count: usize, + ) -> Result<(Vec, Vec), Error> { + if abs >= req_insert_count { + return Err(Error::Corrupt); + } + let (n, v) = self.table.get_absolute(abs).ok_or(Error::Corrupt)?; + Ok((n.to_vec(), v.to_vec())) + } + + /// Current Insert Count (entries inserted via the encoder stream). + pub fn insert_count(&self) -> usize { + self.table.insert_count() + } + + /// Current dynamic-table byte size (for tests/inspection). + #[cfg(test)] + pub(crate) fn table_size(&self) -> usize { + self.table.size() + } + + /// Current dynamic-table capacity (for tests/inspection). + #[cfg(test)] + pub(crate) fn table_capacity(&self) -> usize { + self.table.capacity() + } + + /// Number of live dynamic-table entries (for tests/inspection). + #[cfg(test)] + pub(crate) fn table_len(&self) -> usize { + self.table.len() + } +} + +/// Read a QPACK string literal (§4.1.2) at `pos`: an `n`-bit length prefix +/// whose `1 << n` bit is the Huffman flag, then that many octets, +/// Huffman-decoded if the flag was set. +fn read_string(block: &[u8], pos: usize, n: u32) -> Result<(Vec, usize), Error> { + let first = *block.get(pos).ok_or(Error::UnexpectedEnd)?; + let huff = first & (1u8 << n) != 0; + let (len, p) = decode_int(block, pos, n)?; + let end = p.checked_add(len).ok_or(Error::Corrupt)?; + if end > block.len() { + return Err(Error::UnexpectedEnd); + } + let raw = &block[p..end]; + let data = if huff { + huffman::decode(raw)? + } else { + raw.to_vec() + }; + Ok((data, end)) +} + +#[cfg(test)] +mod tests; diff --git a/src/qpack/static_table.rs b/src/qpack/static_table.rs new file mode 100644 index 0000000..43e867a --- /dev/null +++ b/src/qpack/static_table.rs @@ -0,0 +1,135 @@ +//! QPACK static table — RFC 9204 Appendix A. +//! +//! Unlike HPACK (whose static table is 1-indexed), QPACK's static table is +//! **0-indexed**: the first entry (`:authority`) is index 0. There are 99 +//! entries (indices 0..=98). Transcribed verbatim from RFC 9204 Appendix A. + +/// QPACK static table (RFC 9204 Appendix A), 99 `(name, value)` entries. +/// Index 0 is `STATIC_TABLE[0]`. +#[rustfmt::skip] +pub(crate) const STATIC_TABLE: [(&[u8], &[u8]); 99] = [ + (b":authority", b""), + (b":path", b"/"), + (b"age", b"0"), + (b"content-disposition", b""), + (b"content-length", b"0"), + (b"cookie", b""), + (b"date", b""), + (b"etag", b""), + (b"if-modified-since", b""), + (b"if-none-match", b""), + (b"last-modified", b""), + (b"link", b""), + (b"location", b""), + (b"referer", b""), + (b"set-cookie", b""), + (b":method", b"CONNECT"), + (b":method", b"DELETE"), + (b":method", b"GET"), + (b":method", b"HEAD"), + (b":method", b"OPTIONS"), + (b":method", b"POST"), + (b":method", b"PUT"), + (b":scheme", b"http"), + (b":scheme", b"https"), + (b":status", b"103"), + (b":status", b"200"), + (b":status", b"304"), + (b":status", b"404"), + (b":status", b"503"), + (b"accept", b"*/*"), + (b"accept", b"application/dns-message"), + (b"accept-encoding", b"gzip, deflate, br"), + (b"accept-ranges", b"bytes"), + (b"access-control-allow-headers", b"cache-control"), + (b"access-control-allow-headers", b"content-type"), + (b"access-control-allow-origin", b"*"), + (b"cache-control", b"max-age=0"), + (b"cache-control", b"max-age=2592000"), + (b"cache-control", b"max-age=604800"), + (b"cache-control", b"no-cache"), + (b"cache-control", b"no-store"), + (b"cache-control", b"public, max-age=31536000"), + (b"content-encoding", b"br"), + (b"content-encoding", b"gzip"), + (b"content-type", b"application/dns-message"), + (b"content-type", b"application/javascript"), + (b"content-type", b"application/json"), + (b"content-type", b"application/x-www-form-urlencoded"), + (b"content-type", b"image/gif"), + (b"content-type", b"image/jpeg"), + (b"content-type", b"image/png"), + (b"content-type", b"text/css"), + (b"content-type", b"text/html; charset=utf-8"), + (b"content-type", b"text/plain"), + (b"content-type", b"text/plain;charset=utf-8"), + (b"range", b"bytes=0-"), + (b"strict-transport-security", b"max-age=31536000"), + (b"strict-transport-security", b"max-age=31536000; includesubdomains"), + (b"strict-transport-security", b"max-age=31536000; includesubdomains; preload"), + (b"vary", b"accept-encoding"), + (b"vary", b"origin"), + (b"x-content-type-options", b"nosniff"), + (b"x-xss-protection", b"1; mode=block"), + (b":status", b"100"), + (b":status", b"204"), + (b":status", b"206"), + (b":status", b"302"), + (b":status", b"400"), + (b":status", b"403"), + (b":status", b"421"), + (b":status", b"425"), + (b":status", b"500"), + (b"accept-language", b""), + (b"access-control-allow-credentials", b"FALSE"), + (b"access-control-allow-credentials", b"TRUE"), + (b"access-control-allow-headers", b"*"), + (b"access-control-allow-methods", b"get"), + (b"access-control-allow-methods", b"get, post, options"), + (b"access-control-allow-methods", b"options"), + (b"access-control-expose-headers", b"content-length"), + (b"access-control-request-headers", b"content-type"), + (b"access-control-request-method", b"get"), + (b"access-control-request-method", b"post"), + (b"alt-svc", b"clear"), + (b"authorization", b""), + (b"content-security-policy", b"script-src 'none'; object-src 'none'; base-uri 'none'"), + (b"early-data", b"1"), + (b"expect-ct", b""), + (b"forwarded", b""), + (b"if-range", b""), + (b"origin", b""), + (b"purpose", b"prefetch"), + (b"server", b""), + (b"timing-allow-origin", b"*"), + (b"upgrade-insecure-requests", b"1"), + (b"user-agent", b""), + (b"x-forwarded-for", b""), + (b"x-frame-options", b"deny"), + (b"x-frame-options", b"sameorigin"), +]; + +/// Number of static-table entries (99). +#[cfg(test)] +pub(crate) const STATIC_LEN: usize = STATIC_TABLE.len(); + +/// Look up a 0-based static-table index. Returns `(name, value)`. +pub(crate) fn get(index: usize) -> Option<(&'static [u8], &'static [u8])> { + STATIC_TABLE.get(index).copied() +} + +/// Find the best static-table entry for `(name, value)`: prefer a full +/// name+value match (returning `value_matched = true`), else a name-only +/// match. Returns `(index, value_matched)`. +pub(crate) fn find(name: &[u8], value: &[u8]) -> Option<(usize, bool)> { + let mut name_only: Option = None; + for (i, (n, v)) in STATIC_TABLE.iter().enumerate() { + if *n == name { + if *v == value { + return Some((i, true)); + } + name_only.get_or_insert(i); + } + } + name_only.map(|i| (i, false)) +} diff --git a/src/qpack/tests.rs b/src/qpack/tests.rs new file mode 100644 index 0000000..e5cd1fa --- /dev/null +++ b/src/qpack/tests.rs @@ -0,0 +1,341 @@ +//! RFC 9204 Appendix B worked-example vectors + round-trip / error tests. + +use super::*; +use alloc::vec; + +fn f(name: &[u8], value: &[u8]) -> HeaderField { + HeaderField::new(name, value) +} + +// ─── static table sanity ───────────────────────────────────────────────── + +#[test] +fn static_table_has_99_entries_and_boundaries() { + assert_eq!(static_table::STATIC_LEN, 99); + assert_eq!(static_table::get(0), Some((&b":authority"[..], &b""[..]))); + assert_eq!(static_table::get(1), Some((&b":path"[..], &b"/"[..]))); + assert_eq!(static_table::get(17), Some((&b":method"[..], &b"GET"[..]))); + assert_eq!( + static_table::get(98), + Some((&b"x-frame-options"[..], &b"sameorigin"[..])) + ); + assert_eq!(static_table::get(99), None); +} + +// ─── B.1: literal field line with name reference (static) ──────────────── + +#[test] +fn rfc_b1_literal_field_line_static_name_ref() { + // Stream 0: 0x00 0x00 (Required Insert Count = 0, Base = 0) + // 0x51 0x0b "/index.html" (Literal w/ Name Ref, static idx 1) + let expected: &[u8] = &[ + 0x00, 0x00, 0x51, 0x0b, b'/', b'i', b'n', b'd', b'e', b'x', b'.', b'h', b't', b'm', b'l', + ]; + + let mut enc = QpackEncoder::new(); + enc.set_huffman(false); // RFC example is raw + let block = enc.encode_field_section(&[f(b":path", b"/index.html")]); + assert_eq!(block, expected); + + let mut dec = QpackDecoder::new(); + let out = dec.decode_field_section(expected).unwrap(); + assert_eq!(out, vec![f(b":path", b"/index.html")]); + // No dynamic-table activity. + assert_eq!(dec.insert_count(), 0); + assert_eq!(dec.table_size(), 0); +} + +// ─── B.2: dynamic table (encoder-stream inserts + post-base field section) ─ + +#[test] +fn rfc_b2_dynamic_table_inserts_and_post_base() { + let mut dec = QpackDecoder::new(); + + // Encoder stream: + // 3f bd 01 Set Dynamic Table Capacity = 220 + // c0 0f "www.example.com" Insert w/ Name Ref, static idx 0 (:authority) + // c1 0c "/sample/path" Insert w/ Name Ref, static idx 1 (:path) + let mut estream: Vec = vec![0x3f, 0xbd, 0x01, 0xc0, 0x0f]; + estream.extend_from_slice(b"www.example.com"); + estream.extend_from_slice(&[0xc1, 0x0c]); + estream.extend_from_slice(b"/sample/path"); + + dec.feed_encoder_stream(&estream).unwrap(); + + // Dynamic table state after inserts. + assert_eq!(dec.table_capacity(), 220); + assert_eq!(dec.insert_count(), 2); + assert_eq!(dec.table_len(), 2); + // Size = (10+15+32) + (5+12+32) = 57 + 49 = 106. + assert_eq!(dec.table_size(), 106); + + // Field section (stream 4): + // 03 81 Required Insert Count = 2, Base = 0 + // 10 Indexed, post-base index 0 → abs 0 (:authority=www.example.com) + // 11 Indexed, post-base index 1 → abs 1 (:path=/sample/path) + let block: &[u8] = &[0x03, 0x81, 0x10, 0x11]; + let out = dec.decode_field_section(block).unwrap(); + assert_eq!( + out, + vec![ + f(b":authority", b"www.example.com"), + f(b":path", b"/sample/path"), + ] + ); +} + +// ─── B.3: speculative insert (literal name) ────────────────────────────── + +#[test] +fn rfc_b3_speculative_insert_literal_name() { + let mut dec = QpackDecoder::new(); + + // Set capacity first (the example continues B.2's table; here we make it + // self-contained by raising capacity, then perform the literal insert). + let mut estream: Vec = vec![0x3f, 0xbd, 0x01, 0xc0, 0x0f]; + estream.extend_from_slice(b"www.example.com"); + estream.extend_from_slice(&[0xc1, 0x0c]); + estream.extend_from_slice(b"/sample/path"); + dec.feed_encoder_stream(&estream).unwrap(); + + // Encoder stream (B.3): + // 4a "custom-key" 0c "custom-value" Insert with Literal Name + let mut ins: Vec = vec![0x4a]; + ins.extend_from_slice(b"custom-key"); + ins.push(0x0c); + ins.extend_from_slice(b"custom-value"); + dec.feed_encoder_stream(&ins).unwrap(); + + assert_eq!(dec.insert_count(), 3); + assert_eq!(dec.table_len(), 3); + // Size = 106 + (10+12+32) = 106 + 54 = 160. + assert_eq!(dec.table_size(), 160); +} + +// ─── B.4: duplicate + field section with dynamic + static refs ─────────── + +#[test] +fn rfc_b4_duplicate_and_dynamic_field_section() { + let mut dec = QpackDecoder::new(); + + // Build B.2 + B.3 table state. + let mut estream: Vec = vec![0x3f, 0xbd, 0x01, 0xc0, 0x0f]; + estream.extend_from_slice(b"www.example.com"); + estream.extend_from_slice(&[0xc1, 0x0c]); + estream.extend_from_slice(b"/sample/path"); + let mut ins: Vec = vec![0x4a]; + ins.extend_from_slice(b"custom-key"); + ins.push(0x0c); + ins.extend_from_slice(b"custom-value"); + estream.extend_from_slice(&ins); + dec.feed_encoder_stream(&estream).unwrap(); + assert_eq!(dec.insert_count(), 3); + + // Encoder stream (B.4): 02 Duplicate(relative index 2) + // abs = InsertCount(3) - Index(2) - 1 = 0 (:authority=www.example.com) + dec.feed_encoder_stream(&[0x02]).unwrap(); + assert_eq!(dec.insert_count(), 4); + assert_eq!(dec.table_len(), 4); + // Size = 160 + 57 = 217. + assert_eq!(dec.table_size(), 217); + + // Field section (stream 8): + // 05 00 Required Insert Count = 4, Base = 4 + // 80 Indexed dynamic, abs = Base(4) - 0 - 1 = 3 (:authority=...) + // c1 Indexed static index 1 (:path=/) + // 81 Indexed dynamic, abs = Base(4) - 1 - 1 = 2 (custom-key=custom-value) + let block: &[u8] = &[0x05, 0x00, 0x80, 0xc1, 0x81]; + let out = dec.decode_field_section(block).unwrap(); + assert_eq!( + out, + vec![ + f(b":authority", b"www.example.com"), + f(b":path", b"/"), + f(b"custom-key", b"custom-value"), + ] + ); +} + +// ─── B.5: insert with name reference (dynamic) + eviction ──────────────── + +#[test] +fn rfc_b5_dynamic_name_ref_insert_with_eviction() { + let mut dec = QpackDecoder::new(); + + // Build through B.4 (4 entries, size 217, capacity 220). + let mut estream: Vec = vec![0x3f, 0xbd, 0x01, 0xc0, 0x0f]; + estream.extend_from_slice(b"www.example.com"); + estream.extend_from_slice(&[0xc1, 0x0c]); + estream.extend_from_slice(b"/sample/path"); + let mut ins: Vec = vec![0x4a]; + ins.extend_from_slice(b"custom-key"); + ins.push(0x0c); + ins.extend_from_slice(b"custom-value"); + estream.extend_from_slice(&ins); + estream.push(0x02); // duplicate + dec.feed_encoder_stream(&estream).unwrap(); + assert_eq!(dec.insert_count(), 4); + assert_eq!(dec.table_size(), 217); + + // Encoder stream (B.5): + // 81 0d "custom-value2" Insert w/ Name Ref, dynamic relative idx 1 + // abs = InsertCount(4) - Index(1) - 1 = 2 (name custom-key) + // Inserting (custom-key=custom-value2) costs 10+13+32 = 55; 217+55 = 272 > + // 220, so the oldest entry (abs 0, :authority/www.example.com, size 57) is + // evicted → 217 - 57 + 55 = 215. + let mut b5: Vec = vec![0x81, 0x0d]; + b5.extend_from_slice(b"custom-value2"); + dec.feed_encoder_stream(&b5).unwrap(); + + assert_eq!(dec.insert_count(), 5); + assert_eq!(dec.table_len(), 4); // 5 inserted, 1 evicted + assert_eq!(dec.table_size(), 215); + + // abs 0 is gone; abs 1..=4 live. New entry (abs 4) is custom-value2. + // Verify via a field section: Required Insert Count = 5, Base = 5, + // 80 dynamic abs = 5 - 0 - 1 = 4 (custom-key=custom-value2) + // Required Insert Count enc = (5 mod 12) + 1 = 6 → 0x06; Base delta 0. + let block: &[u8] = &[0x06, 0x00, 0x80]; + let out = dec.decode_field_section(block).unwrap(); + assert_eq!(out, vec![f(b"custom-key", b"custom-value2")]); +} + +// ─── static encoder round-trips ────────────────────────────────────────── + +#[test] +fn encode_indexed_static_full_match() { + // :path=/ is static index 1 (full match) → Indexed Field Line static. + let mut enc = QpackEncoder::new(); + let block = enc.encode_field_section(&[f(b":path", b"/")]); + assert_eq!(block, &[0x00, 0x00, 0xc1]); // prefix + (1 T=1 idx=1) + + let mut dec = QpackDecoder::new(); + assert_eq!( + dec.decode_field_section(&block).unwrap(), + vec![f(b":path", b"/")] + ); +} + +#[test] +fn round_trip_static_and_literal_huffman() { + let mut enc = QpackEncoder::new(); // Huffman on + let mut dec = QpackDecoder::new(); + let fields = vec![ + f(b":method", b"GET"), + f(b":scheme", b"https"), + f(b":path", b"/index.html"), + f(b":authority", b"www.example.com"), + f(b"custom-key", b"custom-value"), + f(b"accept", b"*/*"), + ]; + let block = enc.encode_field_section(&fields); + assert_eq!(dec.decode_field_section(&block).unwrap(), fields); +} + +#[test] +fn round_trip_many_fields_no_huffman() { + let mut enc = QpackEncoder::new(); + enc.set_huffman(false); + let mut dec = QpackDecoder::new(); + let fields: Vec = (0..40) + .map(|i| { + let name = alloc::format!("x-header-{i}"); + let val = alloc::format!("value-{}-{}", i, "blahblah".repeat(i % 3)); + f(name.as_bytes(), val.as_bytes()) + }) + .collect(); + let block = enc.encode_field_section(&fields); + assert_eq!(dec.decode_field_section(&block).unwrap(), fields); +} + +#[test] +fn sensitive_field_sets_never_index_bit() { + let mut enc = QpackEncoder::new(); + enc.set_huffman(false); + let mut dec = QpackDecoder::new(); + // authorization is static index 84 (name match only) → Literal w/ Name Ref, + // N bit set. + let fields = vec![HeaderField::sensitive(b"authorization", b"secret")]; + let block = enc.encode_field_section(&fields); + // byte[2] = 0 1 N T idx(4+). N=1, T=1, idx=84 (>15 so prefix 0x5f + cont). + assert_eq!(block[2] & 0b0010_0000, 0b0010_0000); // N bit + let out = dec.decode_field_section(&block).unwrap(); + assert_eq!(out, fields); + assert!(out[0].sensitive); + + // A literal-literal-name sensitive field too. + let fields2 = vec![HeaderField::sensitive(b"x-secret-hdr", b"v")]; + let block2 = enc.encode_field_section(&fields2); + // byte[2] = 0 0 1 N H len(3+). N bit is 0x10. + assert_eq!(block2[2] & 0b0001_0000, 0b0001_0000); + let out2 = dec.decode_field_section(&block2).unwrap(); + assert!(out2[0].sensitive); +} + +// ─── error handling ────────────────────────────────────────────────────── + +#[test] +fn blocked_reference_rejected() { + // Field section claims Required Insert Count = 2 but nothing inserted. + let mut dec = QpackDecoder::new(); + // enc=3 → RIC=2 (TotalInserts=0, MaxEntries=128, FullRange=256). Base 0. + let block: &[u8] = &[0x03, 0x81, 0x10]; + assert!(matches!( + dec.decode_field_section(block), + Err(Error::Corrupt) + )); +} + +#[test] +fn over_limit_capacity_rejected() { + let mut dec = QpackDecoder::with_max_table_capacity(100); + // Set Dynamic Table Capacity = 220 (> 100) → 0x3f 0xbd 0x01. + assert!(matches!( + dec.feed_encoder_stream(&[0x3f, 0xbd, 0x01]), + Err(Error::Corrupt) + )); +} + +#[test] +fn insert_without_capacity_rejected() { + let mut dec = QpackDecoder::new(); + // Insert with Name Reference at capacity 0 → cannot fit → Corrupt. + let mut ins: Vec = vec![0xc0, 0x0f]; + ins.extend_from_slice(b"www.example.com"); + assert!(matches!(dec.feed_encoder_stream(&ins), Err(Error::Corrupt))); +} + +#[test] +fn bad_static_index_rejected() { + let mut dec = QpackDecoder::new(); + // Indexed static index 99 (out of range): 1 T=1 idx=99. + // 0xc0 | 63 prefix + continuation for 99-63=36 → 0xff 0x24. + let block: &[u8] = &[0x00, 0x00, 0xff, 0x24]; + assert!(matches!( + dec.decode_field_section(block), + Err(Error::Corrupt) + )); +} + +#[test] +fn truncated_value_string_rejected() { + let mut dec = QpackDecoder::new(); + // Literal w/ literal name, name len 1 "x", value length 5 but truncated. + // prefix 0x00 0x00, then 0x21 (0 0 1 0 0 len=1) 'x', 0x05 'a' 'b' + let block: &[u8] = &[0x00, 0x00, 0x21, b'x', 0x05, b'a', b'b']; + assert!(matches!( + dec.decode_field_section(block), + Err(Error::UnexpectedEnd) + )); +} + +#[test] +fn duplicate_bad_index_rejected() { + let mut dec = QpackDecoder::new(); + // Raise capacity, then Duplicate(relative 0) with an empty table → Corrupt. + dec.feed_encoder_stream(&[0x3f, 0xbd, 0x01]).unwrap(); + assert!(matches!( + dec.feed_encoder_stream(&[0x00]), + Err(Error::Corrupt) + )); +} diff --git a/src/rangecoder/mod.rs b/src/rangecoder/mod.rs new file mode 100644 index 0000000..5a161bf --- /dev/null +++ b/src/rangecoder/mod.rs @@ -0,0 +1,523 @@ +//! Adaptive order-0 binary range coder — a standalone entropy codec. +//! +//! Most range/arithmetic coders in this crate are buried inside larger +//! container codecs (LZMA, zstd, arsenic). This module exposes a clean, +//! self-contained one: a carry-less binary range coder driving an +//! order-0 adaptive bit-tree model over bytes. It is the literal-coder +//! core of LZMA, stripped of the LZ layer and any context — useful as a +//! building block, a teaching reference, and a quick entropy stage for +//! skewed byte streams. +//! +//! ## The coder +//! +//! A LZMA-style **binary range coder**: a 32-bit `range` and a 32-bit +//! `low` accumulator (kept in a 64-bit field on the encoder so carries +//! propagate cleanly). Each coded bit narrows `range` by a probability +//! split; whenever `range` drops below `2^24` the coder renormalizes by +//! shifting out one byte and scaling `range` up by 256. The encoder +//! handles carry propagation with the classic cache / cache-size / +//! pending-`0xFF` scheme so a late carry ripples through any run of +//! `0xFF` bytes already staged. The decoder mirrors the renormalization +//! exactly, so encoder and decoder are exact inverses. +//! +//! ## The model +//! +//! Order-0, context-free. Each byte is coded as 8 binary decisions +//! walking a 255-node probability tree (indices `1..=255`, the classic +//! "bit-tree" shape): start at node 1, and for each of the 8 bits +//! (most-significant first) code the bit against `probs[node]`, then +//! descend to `node*2 + bit`. After 8 bits `node` holds `256 + byte`, +//! confirming the walk visited a distinct node per prefix. +//! +//! Probabilities are 11-bit adaptive counters (`kProb = 2048`, the +//! midpoint, is the initial value). After coding a bit they adapt by the +//! standard LZMA rule with a move-shift of 5: +//! +//! * bit 0: `prob += (2048 - prob) >> 5` +//! * bit 1: `prob -= prob >> 5` +//! +//! Both encoder and decoder run the identical update, so their models +//! stay in lock-step without transmitting any model data. +//! +//! ## Byte layout +//! +//! ```text +//! ┌────────────────────────┬───────────────────────────────┐ +//! │ u64 length (LE) │ range-coded payload │ +//! │ 8 bytes │ variable │ +//! └────────────────────────┴───────────────────────────────┘ +//! ``` +//! +//! * **Bytes 0..8** — the original (decoded) length as a little-endian +//! `u64`. The decoder reads this first so it knows exactly how many +//! bytes to emit; the payload itself carries no end marker. +//! * **Bytes 8..** — the range-coder output. The encoder flushes 5 +//! trailing bytes at end-of-stream (one cache byte + four to drain +//! `low`), so a non-empty payload is always ≥ 5 bytes. An empty input +//! produces just the 8-byte header and no payload. +//! +//! Because the model adapts from a uniform start, the first few bytes of +//! any stream cost close to 8 bits each; the win comes once the counters +//! have skewed. On 64 KiB of zeros the payload collapses by well over +//! 40x; on English text it lands well under 8 bits/byte; on +//! incompressible input it is at most a few bytes larger than the +//! original plus the 8-byte header — round-tripping is always lossless. +//! +//! ## Errors +//! +//! The decoder never panics on malformed input. A header shorter than 8 +//! bytes, or a declared length the payload cannot satisfy, yields +//! [`Error::UnexpectedEnd`]; a length so large it could not have been +//! produced by this coder yields [`Error::Corrupt`]. + +#![cfg_attr(docsrs, doc(cfg(feature = "rangecoder")))] + +extern crate alloc; +use alloc::vec::Vec; + +use crate::error::Error; +use crate::traits::{Algorithm, RawDecoder, RawEncoder, RawProgress}; + +/// Number of probability counters in the bit-tree: indices `1..=255` are +/// used (index 0 is dead), one per internal node of the 8-level tree. +const TREE_NODES: usize = 256; +/// Initial (and midpoint) probability for an 11-bit counter: kProb/2 = 1024. +const PROB_INIT: u16 = 1 << 10; +/// Total probability scale exponent (kProb = 2048). Splits use the top +/// 11 bits of range. +const PROB_BITS: u32 = 11; +/// Adaptation move-shift. +const MOVE_BITS: u32 = 5; +/// Renormalization threshold: keep `range >= 2^24`. +const TOP: u32 = 1 << 24; + +/// Zero-sized marker type implementing [`Algorithm`] for the range coder. +#[derive(Debug, Clone, Copy, Default)] +pub struct RangeCoder; + +impl Algorithm for RangeCoder { + const NAME: &'static str = "range"; + type Encoder = Encoder; + type Decoder = Decoder; + type EncoderConfig = (); + type DecoderConfig = (); + fn encoder_with(_: ()) -> Encoder { + Encoder::new() + } + fn decoder_with(_: ()) -> Decoder { + Decoder::new() + } +} + +// ─── adaptive bit-tree model ────────────────────────────────────────────── + +/// The order-0 model: a flat array of 11-bit probability counters indexed +/// by bit-tree node. Shared (by identical construction + update) between +/// encoder and decoder. +#[derive(Debug, Clone)] +struct Model { + probs: [u16; TREE_NODES], +} + +impl Model { + fn new() -> Self { + Self { + probs: [PROB_INIT; TREE_NODES], + } + } +} + +#[inline] +fn adapt(prob: &mut u16, bit: u32) { + if bit == 0 { + *prob += ((1u16 << PROB_BITS) - *prob) >> MOVE_BITS; + } else { + *prob -= *prob >> MOVE_BITS; + } +} + +// ─── encoder ────────────────────────────────────────────────────────────── + +/// Streaming adaptive order-0 range encoder. +/// +/// Buffers the entire input (the framing needs the original length up front +/// for the header, and the range coder produces output only at flush), then +/// emits the framed stream on [`finish`](crate::Encoder::finish). This is +/// the same buffer-then-transform shape used by the block codecs in this +/// crate. +#[derive(Debug)] +pub struct Encoder { + input: Vec, + /// Encoded stream (header + payload), produced lazily on first finish. + out: Vec, + head: usize, + finished: bool, +} + +impl Encoder { + /// Construct a fresh encoder. + pub fn new() -> Self { + Self { + input: Vec::new(), + out: Vec::new(), + head: 0, + finished: false, + } + } + + /// Range-encode `self.input` into `self.out` (header + payload). + fn encode_all(&mut self) { + self.out.clear(); + self.out + .extend_from_slice(&(self.input.len() as u64).to_le_bytes()); + + if self.input.is_empty() { + return; + } + + let mut rc = RangeEncoder::new(); + let mut model = Model::new(); + // Take the input out so we can borrow `self.out` mutably while + // reading the bytes; swap a placeholder in to avoid cloning. + let input = core::mem::take(&mut self.input); + for &byte in &input { + // Bit-tree walk, MSB first. + let mut node = 1usize; + for i in (0..8).rev() { + let bit = ((byte >> i) & 1) as u32; + let prob = &mut model.probs[node]; + rc.encode_bit(&mut self.out, prob, bit); + node = (node << 1) | (bit as usize); + } + } + rc.flush(&mut self.out); + self.input = input; + } + + fn drain(&mut self, output: &mut [u8]) -> usize { + let avail = self.out.len() - self.head; + let n = avail.min(output.len()); + output[..n].copy_from_slice(&self.out[self.head..self.head + n]); + self.head += n; + n + } +} + +impl Default for Encoder { + fn default() -> Self { + Self::new() + } +} + +impl RawEncoder for Encoder { + fn raw_encode(&mut self, input: &[u8], _output: &mut [u8]) -> Result { + // Pure buffering — no output until finish. + self.input.extend_from_slice(input); + Ok(RawProgress { + consumed: input.len(), + written: 0, + done: false, + }) + } + + fn raw_finish(&mut self, output: &mut [u8]) -> Result { + if !self.finished { + self.encode_all(); + self.finished = true; + } + let written = self.drain(output); + let done = self.head >= self.out.len(); + Ok(RawProgress { + consumed: 0, + written, + done, + }) + } + + fn raw_reset(&mut self) { + self.input.clear(); + self.out.clear(); + self.head = 0; + self.finished = false; + } +} + +/// The carry-handling binary range encoder. +struct RangeEncoder { + low: u64, + range: u32, + cache: u8, + cache_size: u64, +} + +impl RangeEncoder { + fn new() -> Self { + // cache_size starts at 1: the first shift_low produces a leading + // cache byte that is always 0 (LZMA's well-known leading zero), + // which the decoder reads and discards on init. + Self { + low: 0, + range: 0xFFFF_FFFF, + cache: 0, + cache_size: 1, + } + } + + #[inline] + fn encode_bit(&mut self, out: &mut Vec, prob: &mut u16, bit: u32) { + // bound = (range >> 11) * prob — the size of the "bit 0" subrange. + let bound = (self.range >> PROB_BITS) * (*prob as u32); + if bit == 0 { + self.range = bound; + } else { + self.low += bound as u64; + self.range -= bound; + } + adapt(prob, bit); + while self.range < TOP { + self.range <<= 8; + self.shift_low(out); + } + } + + #[inline] + fn shift_low(&mut self, out: &mut Vec) { + // If the top byte of low is not 0xFF (or a carry is pending), + // flush the cached byte plus any staged 0xFF run, adjusted by the + // carry bit (low >> 32). + if self.low < 0xFF00_0000 || self.low > 0xFFFF_FFFF { + let carry = (self.low >> 32) as u8; + let mut temp = self.cache; + loop { + out.push(temp.wrapping_add(carry)); + temp = 0xFF; + self.cache_size -= 1; + if self.cache_size == 0 { + break; + } + } + self.cache = (self.low >> 24) as u8; + } + self.cache_size += 1; + self.low = (self.low << 8) & 0xFFFF_FFFF; + } + + fn flush(&mut self, out: &mut Vec) { + // Drain the 32-bit low accumulator: 5 shift_low calls move every + // byte (plus the cache) out to the stream. + for _ in 0..5 { + self.shift_low(out); + } + } +} + +// ─── decoder ────────────────────────────────────────────────────────────── + +/// Streaming adaptive order-0 range decoder. +/// +/// Buffers the entire compressed stream, then decodes the framed payload on +/// [`finish`](crate::Decoder::finish). Truncated or malformed input is +/// reported as an [`Error`] — the decoder never panics. +#[derive(Debug)] +pub struct Decoder { + input: Vec, + out: Vec, + head: usize, + finished: bool, +} + +impl Decoder { + /// Construct a fresh decoder. + pub fn new() -> Self { + Self { + input: Vec::new(), + out: Vec::new(), + head: 0, + finished: false, + } + } + + fn decode_all(&mut self) -> Result<(), Error> { + self.out.clear(); + if self.input.len() < 8 { + // A valid stream always carries the 8-byte length header. + // An empty stream (0 bytes) is also too short — there is no + // unambiguous "empty" encoding without the header. + return Err(Error::UnexpectedEnd); + } + let mut len_bytes = [0u8; 8]; + len_bytes.copy_from_slice(&self.input[..8]); + let out_len = u64::from_le_bytes(len_bytes); + + if out_len == 0 { + // Header says zero bytes — payload must be empty. + if self.input.len() != 8 { + return Err(Error::Corrupt); + } + return Ok(()); + } + + // Guard against an absurd declared length that no real payload of + // this size could produce (decompression-bomb / corruption guard). + // Each output byte needs at least ~1 bit; the payload is + // `input.len() - 8` bytes. Allow generous slack (256x) before + // rejecting, since low-entropy data compresses hugely. + let payload_len = self.input.len() - 8; + let max_plausible = (payload_len as u64) + .saturating_mul(256) + .saturating_add(1024); + if out_len > max_plausible { + return Err(Error::Corrupt); + } + let out_len = out_len as usize; + self.out.reserve(out_len); + + let payload = core::mem::take(&mut self.input); + let result = (|| { + let mut rc = RangeDecoder::new(&payload[8..])?; + let mut model = Model::new(); + for _ in 0..out_len { + let mut node = 1usize; + for _ in 0..8 { + let prob = &mut model.probs[node]; + let bit = rc.decode_bit(prob)?; + node = (node << 1) | (bit as usize); + } + // node now holds 256 + byte. + self.out.push((node & 0xFF) as u8); + } + Ok(()) + })(); + self.input = payload; + result + } + + fn drain(&mut self, output: &mut [u8]) -> usize { + let avail = self.out.len() - self.head; + let n = avail.min(output.len()); + output[..n].copy_from_slice(&self.out[self.head..self.head + n]); + self.head += n; + n + } +} + +impl Default for Decoder { + fn default() -> Self { + Self::new() + } +} + +impl RawDecoder for Decoder { + fn raw_decode(&mut self, input: &[u8], _output: &mut [u8]) -> Result { + // Pure buffering — output is produced on finish. + self.input.extend_from_slice(input); + Ok(RawProgress { + consumed: input.len(), + written: 0, + done: false, + }) + } + + fn raw_finish(&mut self, output: &mut [u8]) -> Result { + if !self.finished { + self.decode_all()?; + self.finished = true; + } + let written = self.drain(output); + let done = self.head >= self.out.len(); + Ok(RawProgress { + consumed: 0, + written, + done, + }) + } + + fn raw_reset(&mut self) { + self.input.clear(); + self.out.clear(); + self.head = 0; + self.finished = false; + } +} + +/// The binary range decoder, mirror of [`RangeEncoder`]. +struct RangeDecoder<'a> { + payload: &'a [u8], + pos: usize, + range: u32, + code: u32, + /// Set once a renormalization tried to read past the end of the + /// payload. A correct, complete stream never sets this; a stream + /// truncated relative to its declared length does. + overran: bool, +} + +impl<'a> RangeDecoder<'a> { + fn new(payload: &'a [u8]) -> Result { + // The encoder's first shift_low emits a leading 0x00 cache byte; + // the decoder reads (and ignores) it, then primes `code` with the + // next 4 bytes. So a non-empty payload is always at least 5 bytes. + if payload.len() < 5 { + return Err(Error::UnexpectedEnd); + } + // First byte is the leading zero — skip it. + let mut d = Self { + payload, + pos: 1, + range: 0xFFFF_FFFF, + code: 0, + overran: false, + }; + for _ in 0..4 { + d.code = (d.code << 8) | d.next_byte() as u32; + } + Ok(d) + } + + #[inline] + fn next_byte(&mut self) -> u8 { + // Past-end reads return 0 and flag `overran`. A correct, complete + // stream never over-reads (the encoder's 5 flush bytes cover every + // renormalization the decoder performs). A stream truncated + // relative to its declared length will over-read, which the caller + // turns into `Error::UnexpectedEnd`. Bounds are always respected + // (no panic, no out-of-range index). + match self.payload.get(self.pos) { + Some(&b) => { + self.pos += 1; + b + } + None => { + self.pos += 1; + self.overran = true; + 0 + } + } + } + + #[inline] + fn decode_bit(&mut self, prob: &mut u16) -> Result { + let bound = (self.range >> PROB_BITS) * (*prob as u32); + let bit; + if self.code < bound { + self.range = bound; + bit = 0; + } else { + self.code -= bound; + self.range -= bound; + bit = 1; + } + adapt(prob, bit); + while self.range < TOP { + self.range <<= 8; + self.code = (self.code << 8) | self.next_byte() as u32; + } + if self.overran { + return Err(Error::UnexpectedEnd); + } + Ok(bit) + } +} + +#[cfg(test)] +mod tests; diff --git a/src/rangecoder/tests.rs b/src/rangecoder/tests.rs new file mode 100644 index 0000000..68c2249 --- /dev/null +++ b/src/rangecoder/tests.rs @@ -0,0 +1,336 @@ +//! Round-trip, compression, and robustness tests for the adaptive order-0 +//! range coder. All tests drive the public streaming [`Encoder`] / +//! [`Decoder`] traits exactly as a downstream caller would. + +extern crate alloc; +use alloc::vec; +use alloc::vec::Vec; + +use super::RangeCoder; +use crate::error::Error; +use crate::traits::{Algorithm, Decoder, Encoder, Status}; + +/// Encode `input` to a single owned buffer, driving the streaming loop with +/// modestly-sized output chunks so the drain paths get exercised. +fn encode(input: &[u8]) -> Vec { + let mut enc = RangeCoder::encoder(); + let mut out = Vec::new(); + let mut buf = [0u8; 64]; + let mut consumed = 0; + while consumed < input.len() { + let (p, status) = enc.encode(&input[consumed..], &mut buf).unwrap(); + out.extend_from_slice(&buf[..p.written]); + consumed += p.consumed; + if matches!(status, Status::InputEmpty) { + break; + } + } + loop { + let (p, status) = enc.finish(&mut buf).unwrap(); + out.extend_from_slice(&buf[..p.written]); + if matches!(status, Status::StreamEnd) { + break; + } + } + out +} + +/// Decode `input`, returning the produced bytes or the decoder error. +fn decode(input: &[u8]) -> Result, Error> { + let mut dec = RangeCoder::decoder(); + let mut out = Vec::new(); + let mut buf = [0u8; 64]; + let mut consumed = 0; + while consumed < input.len() { + let (p, status) = dec.decode(&input[consumed..], &mut buf)?; + out.extend_from_slice(&buf[..p.written]); + consumed += p.consumed; + if matches!(status, Status::StreamEnd) { + return Ok(out); + } + if matches!(status, Status::InputEmpty) { + break; + } + } + loop { + let (p, status) = dec.finish(&mut buf)?; + out.extend_from_slice(&buf[..p.written]); + if matches!(status, Status::StreamEnd) { + break; + } + } + Ok(out) +} + +fn round_trip(input: &[u8]) { + let enc = encode(input); + let dec = decode(&enc).expect("decode should succeed on our own output"); + assert_eq!(dec, input, "round-trip mismatch (len {})", input.len()); +} + +// ─── round-trip correctness ─────────────────────────────────────────────── + +#[test] +fn empty_round_trips() { + round_trip(&[]); + // Empty input: 8-byte header, no payload. + assert_eq!(encode(&[]).len(), 8); +} + +#[test] +fn single_byte_round_trips() { + for b in 0u16..=255 { + round_trip(&[b as u8]); + } +} + +#[test] +fn two_bytes_round_trip() { + round_trip(&[0x00, 0xFF]); + round_trip(&[0xFF, 0x00]); + round_trip(&[0xAA, 0x55]); +} + +#[test] +fn all_byte_values_round_trip() { + let data: Vec = (0..=255u16).map(|b| b as u8).collect(); + round_trip(&data); + // Repeated, so the model gets to adapt across the full alphabet. + let mut big = Vec::new(); + for _ in 0..16 { + big.extend((0..=255u16).map(|b| b as u8)); + } + round_trip(&big); +} + +#[test] +fn zeros_round_trip() { + round_trip(&[0u8; 1]); + round_trip(&vec![0u8; 1000]); + round_trip(&vec![0u8; 64 * 1024]); +} + +#[test] +fn english_text_round_trips() { + let text = b"The quick brown fox jumps over the lazy dog. \ + Pack my box with five dozen liquor jugs. \ + How vexingly quick daft zebras jump! \ + Sphinx of black quartz, judge my vow."; + let mut data = Vec::new(); + for _ in 0..64 { + data.extend_from_slice(text); + } + round_trip(&data); +} + +#[test] +fn carry_heavy_round_trips() { + // Bytes that drive `low` toward the 0xFF.. carry-propagation path. + let data: Vec = (0..5000u32) + .map(|i| (i.wrapping_mul(131) >> 3) as u8) + .collect(); + round_trip(&data); + round_trip(&vec![0xFFu8; 4096]); +} + +#[test] +fn pseudo_random_round_trips() { + // A simple LCG — deterministic, no deps. "Incompressible"-ish. + let mut state: u32 = 0x1234_5678; + let mut data = Vec::with_capacity(8192); + for _ in 0..8192 { + state = state.wrapping_mul(1_103_515_245).wrapping_add(12345); + data.push((state >> 16) as u8); + } + round_trip(&data); +} + +#[test] +fn many_sizes_round_trip() { + let mut state: u32 = 0xDEAD_BEEF; + for len in 0..300usize { + let data: Vec = (0..len) + .map(|_| { + state = state.wrapping_mul(1_103_515_245).wrapping_add(12345); + (state >> 16) as u8 + }) + .collect(); + round_trip(&data); + } +} + +// ─── compression effectiveness ──────────────────────────────────────────── + +#[test] +fn compresses_zeros_hugely() { + let data = vec![0u8; 64 * 1024]; + let enc = encode(&data); + // 64 KiB of a single symbol collapses dramatically. The order-0 + // adaptive counter floors at ~0.02 bits/symbol per bit-tree level + // (move-shift 5), so the payload settles around 1.5 KiB — a >40x + // ratio. Assert a comfortable < input/30 to prove genuine, large + // compression without being brittle about the exact floor. + assert!( + enc.len() < data.len() / 30, + "64 KiB of zeros should shrink >30x, got {} bytes ({}x)", + enc.len(), + data.len() / enc.len().max(1) + ); +} + +#[test] +fn compresses_skewed_input() { + // 95% zeros, 5% spread — low entropy, must compress well below input. + let mut state: u32 = 0xABCD_1234; + let mut data = Vec::with_capacity(32 * 1024); + for _ in 0..32 * 1024 { + state = state.wrapping_mul(1_103_515_245).wrapping_add(12345); + if (state >> 24).is_multiple_of(20) { + data.push((state >> 8) as u8); + } else { + data.push(0); + } + } + let enc = encode(&data); + assert!( + enc.len() < data.len() / 2, + "skewed input ({} bytes) should compress to <50%, got {}", + data.len(), + enc.len() + ); +} + +#[test] +fn compresses_english_text() { + let text = b"the quick brown fox jumps over the lazy dog "; + let mut data = Vec::new(); + for _ in 0..512 { + data.extend_from_slice(text); + } + let enc = encode(&data); + assert!( + enc.len() < data.len(), + "English text ({} bytes) should compress, got {}", + data.len(), + enc.len() + ); +} + +#[test] +fn incompressible_overhead_is_bounded() { + // Random data may not shrink, but must not blow up: at most a small + // fraction larger than the original plus the 8-byte header. + let mut state: u32 = 0x0BAD_F00D; + let mut data = Vec::with_capacity(16 * 1024); + for _ in 0..16 * 1024 { + state = state.wrapping_mul(1_103_515_245).wrapping_add(12345); + data.push((state >> 16) as u8); + } + let enc = encode(&data); + assert!( + enc.len() <= data.len() + data.len() / 16 + 16, + "incompressible expansion too large: {} -> {}", + data.len(), + enc.len() + ); +} + +// ─── robustness: never panic on bad input ───────────────────────────────── + +#[test] +fn truncated_header_errors() { + for n in 0..8 { + let bytes = vec![0u8; n]; + let err = decode(&bytes).unwrap_err(); + assert_eq!(err, Error::UnexpectedEnd, "n={n}"); + } +} + +#[test] +fn truncated_payload_errors() { + // Encode something non-trivial, then lop off the tail of the payload. + let data = vec![7u8; 4096]; + let enc = encode(&data); + assert!(enc.len() > 13); + // Drop the final flush bytes — decoder must over-read and error. + for cut in 1..=6 { + let truncated = &enc[..enc.len() - cut]; + let r = decode(truncated); + assert!( + r.is_err(), + "truncating {cut} bytes should error, got {:?}", + r.map(|v| v.len()) + ); + } +} + +#[test] +fn garbage_does_not_panic() { + // A header claiming a small length, with random payload bytes: must + // either decode to *something* of that length or error — never panic. + let mut state: u32 = 0xFACE_CAFE; + for _ in 0..200 { + let mut bytes = Vec::new(); + state = state.wrapping_mul(1_103_515_245).wrapping_add(12345); + let declared = (state >> 24) as u64 % 64; // 0..63 + bytes.extend_from_slice(&declared.to_le_bytes()); + let payload_len = (state >> 8) as usize % 40; + for _ in 0..payload_len { + state = state.wrapping_mul(1_103_515_245).wrapping_add(12345); + bytes.push((state >> 16) as u8); + } + // Just must not panic; result is don't-care. + let _ = decode(&bytes); + } +} + +#[test] +fn absurd_length_header_errors() { + // Header claims u64::MAX bytes with a tiny payload — reject as corrupt + // rather than attempting a gigantic allocation. + let mut bytes = u64::MAX.to_le_bytes().to_vec(); + bytes.extend_from_slice(&[0u8; 5]); + assert_eq!(decode(&bytes).unwrap_err(), Error::Corrupt); +} + +#[test] +fn zero_length_with_payload_is_corrupt() { + let mut bytes = 0u64.to_le_bytes().to_vec(); + bytes.push(0xAB); // length says 0, but there are extra bytes + assert_eq!(decode(&bytes).unwrap_err(), Error::Corrupt); +} + +// ─── reset reuse ────────────────────────────────────────────────────────── + +#[test] +fn encoder_and_decoder_reset() { + let mut enc = RangeCoder::encoder(); + let mut buf = [0u8; 256]; + + // First stream. + let _ = enc.encode(b"first stream payload", &mut buf).unwrap(); + let mut s1 = Vec::new(); + loop { + let (p, st) = enc.finish(&mut buf).unwrap(); + s1.extend_from_slice(&buf[..p.written]); + if matches!(st, Status::StreamEnd) { + break; + } + } + + enc.reset(); + + // Second stream after reset must be independent and correct. + let _ = enc.encode(b"second!", &mut buf).unwrap(); + let mut s2 = Vec::new(); + loop { + let (p, st) = enc.finish(&mut buf).unwrap(); + s2.extend_from_slice(&buf[..p.written]); + if matches!(st, Status::StreamEnd) { + break; + } + } + + assert_eq!(decode(&s1).unwrap(), b"first stream payload"); + assert_eq!(decode(&s2).unwrap(), b"second!"); +} diff --git a/tests/new_codecs_batch.rs b/tests/new_codecs_batch.rs new file mode 100644 index 0000000..acc98e5 --- /dev/null +++ b/tests/new_codecs_batch.rs @@ -0,0 +1,108 @@ +//! Integration coverage for the newly added codecs: the four standalone +//! `Algorithm` primitives reachable through the factory, and the QPACK +//! header-codec module API. + +#![cfg(all( + feature = "factory", + feature = "huffman", + feature = "rangecoder", + feature = "mtf", + feature = "bwt", + feature = "qpack" +))] + +use compcol::{Decoder, Encoder, Status}; + +/// Drive a boxed encoder, then a boxed decoder, over `data` and return the +/// decoded output. +fn round_trip_by_name(name: &str, data: &[u8]) -> Vec { + let mut enc = compcol::factory::encoder_by_name(name).expect("encoder"); + let mut dec = compcol::factory::decoder_by_name(name).expect("decoder"); + + let mut encoded = Vec::new(); + let mut buf = vec![0u8; 512]; + let mut pos = 0; + while pos < data.len() { + let (p, _) = enc.encode(&data[pos..], &mut buf).unwrap(); + encoded.extend_from_slice(&buf[..p.written]); + pos += p.consumed; + if p.consumed == 0 && p.written == 0 { + break; + } + } + loop { + let (p, st) = enc.finish(&mut buf).unwrap(); + encoded.extend_from_slice(&buf[..p.written]); + if matches!(st, Status::StreamEnd) { + break; + } + } + + let mut decoded = Vec::new(); + let mut pos = 0; + while pos < encoded.len() { + let (p, _) = dec.decode(&encoded[pos..], &mut buf).unwrap(); + decoded.extend_from_slice(&buf[..p.written]); + pos += p.consumed; + if p.consumed == 0 && p.written == 0 { + break; + } + } + loop { + let (p, st) = dec.finish(&mut buf).unwrap(); + decoded.extend_from_slice(&buf[..p.written]); + if matches!(st, Status::StreamEnd) { + break; + } + } + decoded +} + +#[test] +fn factory_round_trips_new_primitives() { + let corpus: [&[u8]; 4] = [ + b"", + b"the quick brown fox jumps over the lazy dog", + &[0u8; 4096], + b"mississippi banana bandana ananas", + ]; + for name in ["huffman", "range", "mtf", "bwt"] { + for data in corpus { + assert_eq!( + round_trip_by_name(name, data), + data, + "round-trip mismatch for codec {name} on {} bytes", + data.len() + ); + } + } +} + +#[test] +fn new_codecs_registered_in_names() { + let names = compcol::factory::names(); + for n in ["huffman", "range", "mtf", "bwt"] { + assert!(names.contains(&n), "{n} not registered in factory::names()"); + assert!( + compcol::factory::extension(n).is_some(), + "{n} has no extension" + ); + } +} + +#[test] +fn qpack_module_round_trips() { + use compcol::hpack::HeaderField; + use compcol::qpack::{QpackDecoder, QpackEncoder}; + + let mut enc = QpackEncoder::new(); + let mut dec = QpackDecoder::new(); + let fields = [ + HeaderField::new(b":method", b"GET"), + HeaderField::new(b":path", b"/"), + HeaderField::new(b"user-agent", b"compcol-test/1.0"), + ]; + let block = enc.encode_field_section(&fields); + let out = dec.decode_field_section(&block).unwrap(); + assert_eq!(out, fields); +}