feat: Adds pre-trained dictionary support

.clusterfuzzlite/build.sh

@@ -6,9 +6,9 @@
 #  This source code is licensed under the BSD-style license found in the
 #  LICENSE file in the root directory of this source tree.
 
-AVAILABLE_FUZZERS="decompress roundtrip seekable pstream"
+AVAILABLE_FUZZERS="decompress roundtrip seekable pstream dict"
 
-LIB_SOURCES="src/lib/zxc_common.c src/lib/zxc_compress.c src/lib/zxc_decompress.c src/lib/zxc_driver.c src/lib/zxc_dispatch.c src/lib/zxc_huffman.c src/lib/zxc_seekable.c src/lib/zxc_pstream.c"
+LIB_SOURCES="src/lib/zxc_common.c src/lib/zxc_compress.c src/lib/zxc_decompress.c src/lib/zxc_dict.c src/lib/zxc_driver.c src/lib/zxc_dispatch.c src/lib/zxc_huffman.c src/lib/zxc_seekable.c src/lib/zxc_pstream.c"
 
 for fuzzer in $AVAILABLE_FUZZERS; do
     if [ -z "${FUZZER_TARGET:-}" ] || [ "${FUZZER_TARGET}" == "$fuzzer" ]; then

.github/workflows/benchmark.yml

@@ -47,7 +47,7 @@ jobs:
       - name: Clone LZbench
         run: |
           # git clone --depth 1 https://github.com/inikep/lzbench "${LZBENCH_DIR}"
-          git clone -b zxc-0.12.x https://github.com/hellobertrand/lzbench "${LZBENCH_DIR}"
+          git clone -b zxc-0.12.x-dict https://github.com/hellobertrand/lzbench "${LZBENCH_DIR}"
 
       - name: Copy Lib ZXC
         run: |

.github/workflows/fuzzing.yml

@@ -42,7 +42,7 @@ jobs:
       fail-fast: false
       matrix:
         sanitizer: [address, undefined]
-        fuzzer: [decompress, roundtrip, seekable, pstream]
+        fuzzer: [decompress, roundtrip, seekable, pstream, dict]
 
     steps:
     - name: Checkout Repository
@@ -132,7 +132,7 @@ jobs:
           CFLAGS="-g -O1 -fprofile-instr-generate -fcoverage-mapping"
           DEFS="-DZXC_FUNCTION_SUFFIX=_default -DZXC_ONLY_DEFAULT"
           INCLUDES="-I include -I src/lib/vendors"
-          SOURCES="src/lib/zxc_common.c src/lib/zxc_compress.c src/lib/zxc_decompress.c src/lib/zxc_driver.c src/lib/zxc_dispatch.c src/lib/zxc_huffman.c src/lib/zxc_seekable.c src/lib/zxc_pstream.c"
+          SOURCES="src/lib/zxc_common.c src/lib/zxc_compress.c src/lib/zxc_decompress.c src/lib/zxc_dict.c src/lib/zxc_driver.c src/lib/zxc_dispatch.c src/lib/zxc_huffman.c src/lib/zxc_seekable.c src/lib/zxc_pstream.c"
 
           clang $CFLAGS $INCLUDES $DEFS $SOURCES tests/fuzz_roundtrip.c \
             -fsanitize=fuzzer -lm -lpthread -o build-cov/fuzz_roundtrip
@@ -146,6 +146,9 @@ jobs:
           clang $CFLAGS $INCLUDES $DEFS $SOURCES tests/fuzz_pstream.c \
             -fsanitize=fuzzer -lm -lpthread -o build-cov/fuzz_pstream
 
+          clang $CFLAGS $INCLUDES $DEFS $SOURCES tests/fuzz_dict.c \
+            -fsanitize=fuzzer -lm -lpthread -o build-cov/fuzz_dict
+
       - name: Replay Corpora
         run: |
           LLVM_PROFILE_FILE="build-cov/roundtrip.profraw" \

CMakeLists.txt

@@ -214,13 +214,15 @@ endif()
 if(CMAKE_SYSTEM_NAME STREQUAL "Emscripten")
     set(ZXC_CORE_SOURCES
         src/lib/zxc_common.c
+        src/lib/zxc_dict.c
         src/lib/zxc_dispatch.c
         src/lib/zxc_pstream.c
         src/lib/zxc_seekable.c
     )
 else()
     set(ZXC_CORE_SOURCES
         src/lib/zxc_common.c
+        src/lib/zxc_dict.c
         src/lib/zxc_driver.c
         src/lib/zxc_dispatch.c
         src/lib/zxc_pstream.c
@@ -419,6 +421,7 @@ if(ZXC_BUILD_TESTS)
         tests/test_seekable_mt.c
         tests/test_format.c
         tests/test_misc.c
+        tests/test_dict.c
     )
 
     # When building shared libraries, create a static version for tests
@@ -429,6 +432,7 @@ if(ZXC_BUILD_TESTS)
         # and is already pulled in via ${ZXC_VARIANT_OBJECTS} below.
         add_library(zxc_lib_static STATIC
             src/lib/zxc_common.c
+            src/lib/zxc_dict.c
             src/lib/zxc_driver.c
             src/lib/zxc_dispatch.c
             src/lib/zxc_pstream.c

README.md

@@ -460,6 +460,56 @@ zxc_compress_opts_t opts = {
 
 ---
 
+## Dictionary Compression
+
+For workloads compressed in **small blocks** (4 KB–128 KB), a pre-trained dictionary dramatically improves compression ratio. Because the dictionary prefills the LZ77 sliding window at the *start of each block*, the benefit is per-block: a block only has its own preceding bytes as history, so the smaller the block, the more it leans on the dictionary for representative patterns. This applies whether the input is a single small payload or a large payload split into many small blocks — any time the block size is small enough that early bytes would otherwise lack history to match against.
+
+**Typical use cases:** JSON API responses, small game assets, structured logs, key-value store records, RPC messages, and any large but homogeneous corpus compressed in small blocks for random access (e.g. seekable archives).
+
+### Training a dictionary
+
+```bash
+# Train a dictionary from a corpus of similar files
+zxc --train-dict corpus.zxd samples/*.json
+```
+
+```c
+// C API
+const void* samples[] = { buf1, buf2, buf3 };
+size_t sizes[] = { len1, len2, len3 };
+uint8_t dict[32768];
+int64_t dict_sz = zxc_train_dict(samples, sizes, 3, dict, sizeof(dict));
+```
+
+### Compressing with a dictionary
+
+```bash
+# CLI
+zxc -z -D corpus.zxd input.json
+zxc -d -D corpus.zxd input.json.zxc
+```
+
+```c
+// C API — compression
+zxc_compress_opts_t copts = {
+    .level = ZXC_LEVEL_DEFAULT,
+    .dict = dict_content,
+    .dict_size = dict_sz,
+};
+int64_t compressed_size = zxc_compress(src, src_size, dst, dst_cap, &copts);
+
+// C API — decompression (same dictionary required)
+zxc_decompress_opts_t dopts = {
+    .dict = dict_content,
+    .dict_size = dict_sz,
+};
+int64_t original_size = zxc_decompress(compressed, comp_size, out, out_cap, &dopts);
+```
+
+The dictionary is stored as an external `.zxd` file and referenced by a 32-bit ID in the ZXC file header. Decompressing without the matching dictionary returns `ZXC_ERROR_DICT_REQUIRED` or `ZXC_ERROR_DICT_MISMATCH`. See [FORMAT.md](docs/FORMAT.md) §12 for the full specification.
+
+---
+
 ## Usage
 
 ### 1. CLI

conformance/test_conformance.c

@@ -18,6 +18,7 @@
 #endif
 
 #include "../include/zxc_buffer.h"
+#include "../include/zxc_dict.h"
 #include "../include/zxc_error.h"
 
 /* ---------- helpers ------------------------------------------------------ */
@@ -70,6 +71,63 @@ static int has_suffix(const char *s, const char *suffix)
 
 /* ---------- valid vector test -------------------------------------------- */
 
+/**
+ * @brief Searches for a .zxd dictionary file in the same directory as @p zxc_path
+ *        whose dict_id matches @p target_id. Returns the loaded content (caller frees).
+ */
+static uint8_t *find_dict_for_id(const char *zxc_path, uint32_t target_id,
+                                 const void **content_out, size_t *content_size_out)
+{
+    /* Derive directory from zxc_path */
+    char dir[512];
+    strncpy(dir, zxc_path, sizeof(dir) - 1);
+    dir[sizeof(dir) - 1] = '\0';
+    char *sep = strrchr(dir, '/');
+    if (sep) *(sep + 1) = '\0'; else strcpy(dir, "./");
+
+#ifdef _WIN32
+    char pattern[512];
+    snprintf(pattern, sizeof(pattern), "%s*.zxd", dir);
+    WIN32_FIND_DATAA fd;
+    HANDLE hf = FindFirstFileA(pattern, &fd);
+    if (hf == INVALID_HANDLE_VALUE) return NULL;
+    do {
+        char path[1024];
+        snprintf(path, sizeof(path), "%s%s", dir, fd.cFileName);
+        size_t sz = 0;
+        uint8_t *buf = read_file(path, &sz);
+        if (buf && zxc_dict_get_id(buf, sz) == target_id) {
+            if (zxc_dict_load(buf, sz, content_out, content_size_out, NULL) == 0) {
+                FindClose(hf);
+                return buf;
+            }
+        }
+        free(buf);
+    } while (FindNextFileA(hf, &fd));
+    FindClose(hf);
+#else
+    DIR *dp = opendir(dir);
+    if (!dp) return NULL;
+    const struct dirent *ent;
+    while ((ent = readdir(dp)) != NULL) {
+        if (!has_suffix(ent->d_name, ".zxd")) continue;
+        char path[1024];
+        snprintf(path, sizeof(path), "%s%s", dir, ent->d_name);
+        size_t sz = 0;
+        uint8_t *buf = read_file(path, &sz);
+        if (buf && zxc_dict_get_id(buf, sz) == target_id) {
+            if (zxc_dict_load(buf, sz, content_out, content_size_out, NULL) == 0) {
+                closedir(dp);
+                return buf;
+            }
+        }
+        free(buf);
+    }
+    closedir(dp);
+#endif
+    return NULL;
+}
+
 static int test_valid_vector(const char *zxc_path, const char *expected_path)
 {
     size_t comp_sz = 0, expected_sz = 0;
@@ -87,6 +145,21 @@ static int test_valid_vector(const char *zxc_path, const char *expected_path)
         return 0;
     }
 
+    /* Auto-detect dictionary: if the archive has a dict_id, find the .zxd */
+    const void *dict = NULL;
+    size_t dict_size = 0;
+    uint8_t *dict_buf = NULL;
+    uint32_t did = zxc_get_dict_id(comp, comp_sz);
+    if (did != 0) {
+        dict_buf = find_dict_for_id(zxc_path, did, &dict, &dict_size);
+        if (!dict_buf) {
+            fprintf(stderr, "FAIL: %s  requires dict 0x%08X but no matching .zxd found\n",
+                    zxc_path, did);
+            free(comp); free(expected);
+            return 0;
+        }
+    }
+
     int ok = 1;
 
     uint64_t dec_sz = zxc_get_decompressed_size(comp, comp_sz);
@@ -103,8 +176,10 @@ static int test_valid_vector(const char *zxc_path, const char *expected_path)
             fprintf(stderr, "FAIL: %s  OOM\n", zxc_path);
             ok = 0;
         } else {
+            zxc_decompress_opts_t dopts = {0};
+            if (dict) { dopts.dict = dict; dopts.dict_size = dict_size; }
             int64_t result = zxc_decompress(comp, comp_sz,
-                                            output, (size_t)dec_sz, NULL);
+                                            output, (size_t)dec_sz, &dopts);
             if (result < 0) {
                 fprintf(stderr, "FAIL: %s  decompress failed: %s\n",
                         zxc_path, zxc_error_name((int)result));
@@ -121,6 +196,7 @@ static int test_valid_vector(const char *zxc_path, const char *expected_path)
         }
     }
 
+    free(dict_buf);
     free(comp);
     free(expected);
     return ok;

conformance/valid/text_1k_dict.expected

@@ -0,0 +1 @@
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Curabitur pretium tincidunt lacus. Nulla gravida orci a odio. Nullam varius, turpis et commodo pharetra, est eros bibendum elit, nec luctus magna felis sollicitudin mauris. Integer in mauris eu nibh euismod gravida. Duis ac tellus et risus vulputate vehicula. Donec lobortis risus a elit. Etiam tempor. Ut ullamcorper, ligula ut dictum pharetra, nisi nunc fringilla magna, in commodo elit erat nec turpis. Ut pharetra augue nec augue. Nam elit agna, endrerit sit amet, tincidunt ac.

conformance/valid/text_1k_dict_seekable.expected

@@ -0,0 +1 @@
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. Curabitur pretium tincidunt lacus. Nulla gravida orci a odio. Nullam varius, turpis et commodo pharetra, est eros bibendum elit, nec luctus magna felis sollicitudin mauris. Integer in mauris eu nibh euismod gravida. Duis ac tellus et risus vulputate vehicula. Donec lobortis risus a elit. Etiam tempor. Ut ullamcorper, ligula ut dictum pharetra, nisi nunc fringilla magna, in commodo elit erat nec turpis. Ut pharetra augue nec augue. Nam elit agna, endrerit sit amet, tincidunt ac.

docs/API.md

@@ -28,6 +28,7 @@ For the on-disk binary format see [`FORMAT.md`](FORMAT.md).
 - [10. Streaming API](#10-streaming-api)
 - [10b. Push Streaming API](#10b-push-streaming-api)
 - [11. Seekable API](#11-seekable-api)
+- [11b. Dictionary API](#11b-dictionary-api)
 - [12. Error Handling](#12-error-handling)
 - [13. Thread Safety](#13-thread-safety)
 - [14. Exported Symbols Summary](#14-exported-symbols-summary)
@@ -40,7 +41,9 @@ For the on-disk binary format see [`FORMAT.md`](FORMAT.md).
 zxc.h                  <- freestanding umbrella (no <stdio.h>; kernel-safe)
 ├── zxc_buffer.h       <- Buffer API + Reusable Context API
 │   └── zxc_export.h   <- visibility macros
-├── zxc_constants.h    <- version macros, compression levels, block sizes
+├── zxc_constants.h    <- version macros, compression levels, block sizes, dict sizes
+├── zxc_dict.h         <- Dictionary training, save/load, identification
+│   └── zxc_export.h
 ├── zxc_error.h        <- error codes + zxc_error_name()
 │   └── zxc_export.h
 ├── zxc_opts.h         <- compression / decompression options structs
@@ -204,7 +207,10 @@ typedef enum {
     ZXC_ERROR_IO            = -11,  // file read/write/seek failure
     ZXC_ERROR_NULL_INPUT    = -12,  // required pointer is NULL
     ZXC_ERROR_BAD_BLOCK_TYPE = -13, // unknown block type
-    ZXC_ERROR_BAD_BLOCK_SIZE = -14  // invalid block size
+    ZXC_ERROR_BAD_BLOCK_SIZE = -14, // invalid block size
+    ZXC_ERROR_DICT_REQUIRED  = -15, // file requires a dictionary but none provided
+    ZXC_ERROR_DICT_MISMATCH  = -16, // provided dictionary ID does not match header
+    ZXC_ERROR_DICT_TOO_LARGE = -17  // dictionary exceeds ZXC_DICT_SIZE_MAX
 } zxc_error_t;
 ```
 
@@ -225,6 +231,8 @@ typedef struct {
     size_t block_size;        // Block size in bytes (0 = 512 KB default).
     int    checksum_enabled;  // 1 = enable checksums, 0 = disable.
     int    seekable;          // 1 = append seek table for random access.
+    const void* dict;         // Pre-trained dictionary content (NULL = none).
+    size_t dict_size;         // Dictionary size in bytes (0 = none, max 64 KB).
     zxc_progress_callback_t progress_cb;  // Optional callback (NULL to disable).
     void*  user_data;                     // Passed through to progress_cb.
 } zxc_compress_opts_t;
@@ -236,6 +244,8 @@ typedef struct {
 typedef struct {
     int    n_threads;         // Worker thread count (0 = auto-detect).
     int    checksum_enabled;  // 1 = verify checksums, 0 = skip.
+    const void* dict;         // Pre-trained dictionary content (NULL = none).
+    size_t dict_size;         // Dictionary size in bytes (0 = none).
     zxc_progress_callback_t progress_cb;  // Optional callback.
     void*  user_data;                     // Passed through to progress_cb.
 } zxc_decompress_opts_t;
@@ -1278,6 +1288,73 @@ Returns the encoded byte size of a seek table for `num_blocks` blocks.
 
 ---
 
+## 11b. Dictionary API
+
+Declared in `<zxc_dict.h>`. Provides dictionary training, serialization (`.zxd` format), and identification.
+
+### `zxc_train_dict`
+
+```c
+ZXC_EXPORT int64_t zxc_train_dict(
+    const void* const* samples,
+    const size_t*       sample_sizes,
+    size_t              n_samples,
+    void*               dict_buf,
+    size_t              dict_capacity    // max ZXC_DICT_SIZE_MAX (64KB - 1)
+);
+```
+
+Trains a dictionary from a corpus of representative samples. Returns the size of the trained dictionary, or a negative `zxc_error_t` code.
+
+### `zxc_dict_id`
+
+```c
+ZXC_EXPORT uint32_t zxc_dict_id(const void* dict, size_t dict_size);
+```
+
+Returns a deterministic 32-bit hash of the dictionary content. This ID is stored in the ZXC file header and verified at decompression time. Returns 0 for NULL/empty input.
+
+### `zxc_dict_save`
+
+```c
+ZXC_EXPORT int64_t zxc_dict_save(
+    const void* content, size_t content_size,
+    void* buf, size_t buf_capacity
+);
+```
+
+Serializes dictionary content to the `.zxd` file format. Use `zxc_dict_save_bound(content_size)` to compute the required buffer capacity.
+
+### `zxc_dict_load`
+
+```c
+ZXC_EXPORT int zxc_dict_load(
+    const void* buf, size_t buf_size,
+    const void** content_out, size_t* content_size_out,
+    uint32_t* dict_id_out        // may be NULL
+);
+```
+
+Validates and parses a `.zxd` file from memory. On success, `content_out` points into the input buffer (zero-copy). Returns `ZXC_OK` or a negative error code.
+
+### `zxc_dict_save_bound`
+
+```c
+ZXC_EXPORT size_t zxc_dict_save_bound(size_t content_size);
+```
+
+Returns the maximum `.zxd` file size for a given content size (`ZXC_DICT_HEADER_SIZE + content_size`).
+
+### `zxc_seekable_set_dict`
+
+```c
+ZXC_EXPORT int zxc_seekable_set_dict(zxc_seekable* s, const void* dict, size_t dict_size);
+```
+
+Attaches a dictionary to a seekable handle for random-access decompression. The content is copied internally. Must be called before any `zxc_seekable_decompress_range()` call.
+
+---
+
 ## 12. Error Handling
 
 ### `zxc_error_name`
@@ -1374,6 +1451,12 @@ The shared library exports **47 symbols** (verified with `nm -gU`):
 | 49 | `zxc_write_seek_table` | Seekable | `zxc_seekable.h` |
 | 50 | `zxc_seek_table_size` | Seekable | `zxc_seekable.h` |
 | 51 | `zxc_error_name` | Error | `zxc_error.h` |
+| 52 | `zxc_train_dict` | Dictionary | `zxc_dict.h` |
+| 53 | `zxc_dict_id` | Dictionary | `zxc_dict.h` |
+| 54 | `zxc_dict_save` | Dictionary | `zxc_dict.h` |
+| 55 | `zxc_dict_load` | Dictionary | `zxc_dict.h` |
+| 56 | `zxc_dict_save_bound` | Dictionary | `zxc_dict.h` |
+| 57 | `zxc_seekable_set_dict` | Seekable | `zxc_seekable.h` |
 
 No internal symbols leak into the public ABI. FMV dispatch variants
 (`_default`, `_neon`, `_avx2`, `_avx512`) are compiled with