diff --git a/.gitignore b/.gitignore index a01f895..9300f8a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,12 @@ build/ archive/ -externals/ +# externals/ holds downloaded fixtures (kernel, rootfs, packages) that are +# fetched on demand; tracking them in git would balloon the repo. The +# vendored cJSON and zstd trees are exceptions: they ship with the source +# so the OCI parser and layer unpacker build out of the box. +externals/* +!externals/cjson/ +!externals/zstd/ lib/modules/ *.o *.bin diff --git a/Makefile b/Makefile index 7f4814f..4f2c9a3 100644 --- a/Makefile +++ b/Makefile @@ -25,6 +25,7 @@ SRCS := \ core/vdso.c \ core/bootstrap.c \ core/rosetta.c \ + core/launch.c \ core/sysroot.c \ runtime/thread.c \ runtime/futex.c \ @@ -65,15 +66,80 @@ SRCS := \ debug/gdbstub.c \ debug/gdbstub-reg.c \ debug/gdbstub-rsp.c \ - debug/log.c + debug/log.c \ + oci/ref.c \ + oci/cli.c \ + oci/digest.c \ + oci/digest-set.c \ + oci/blob-store.c \ + oci/media-type.c \ + oci/manifest.c \ + oci/fetch.c \ + oci/store.c \ + oci/pull.c \ + oci/inspect.c \ + oci/dedup-metrics.c \ + oci/status.c \ + oci/policy.c \ + oci/tar.c \ + oci/decompress.c \ + oci/layer-meta.c \ + oci/layer-apply.c \ + oci/origin-meta.c \ + oci/volume.c \ + oci/volume-list.c \ + oci/clone-rootfs.c \ + oci/unpack.c \ + oci/rebuild-cache.c \ + oci/runspec.c \ + oci/user-lookup.c \ + oci/path-resolve.c \ + oci/runtime-files.c \ + oci/run.c SRCS := $(addprefix src/,$(SRCS)) OBJS := $(patsubst src/%.c,$(BUILD_DIR)/%.o,$(SRCS)) +# Vendored cJSON: third-party MIT JSON parser pinned at v1.7.18. Only OCI +# translation units include it. Compiles cleanly with the project warning +# posture, so no per-file CFLAGS override is required. +CJSON_DIR := externals/cjson +CJSON_OBJ := $(BUILD_DIR)/externals/cjson/cJSON.o +OBJS += $(CJSON_OBJ) + +$(CJSON_OBJ): $(CJSON_DIR)/cJSON.c $(CJSON_DIR)/cJSON.h | $(BUILD_DIR) + @mkdir -p $(dir $@) + @echo " CC $<" + $(Q)$(CC) $(CFLAGS) -c -o $@ $< + +# Vendored zstd v1.5.6 (decode-only). Phase 2 OCI layer unpack consumes +# zstd-compressed layer media types. Compression, dictBuilder, deprecated, +# and legacy v01-v06 paths are NOT vendored; do not call ZSTD_compress*. +# Only src/oci/decompress.c includes externals/zstd/lib/zstd.h. +ZSTD_DIR := externals/zstd +ZSTD_SRCS := $(wildcard $(ZSTD_DIR)/lib/common/*.c) \ + $(wildcard $(ZSTD_DIR)/lib/decompress/*.c) +ZSTD_OBJS := $(patsubst $(ZSTD_DIR)/%.c,$(BUILD_DIR)/externals/zstd/%.o,$(ZSTD_SRCS)) +OBJS += $(ZSTD_OBJS) + +ZSTD_CFLAGS := -DZSTD_DISABLE_ASM=1 -DZSTD_LEGACY_SUPPORT=0 \ + -DZSTD_MULTITHREAD=0 -DZSTDLIB_VISIBILITY= \ + -Wno-pedantic -Wno-shadow -Wno-strict-prototypes \ + -Wno-missing-prototypes -Wno-unused-parameter \ + -Wno-cast-align -Wno-implicit-fallthrough \ + -I$(ZSTD_DIR)/lib -I$(ZSTD_DIR)/lib/common + +$(BUILD_DIR)/externals/zstd/%.o: $(ZSTD_DIR)/%.c | $(BUILD_DIR) + @mkdir -p $(dir $@) + @echo " CC $<" + $(Q)$(CC) $(CFLAGS) $(ZSTD_CFLAGS) -c -o $@ $< + DISPATCH_MANIFEST := src/syscall/dispatch.tbl DISPATCH_GENERATOR := scripts/gen-syscall-dispatch.py DISPATCH_HEADER := $(BUILD_DIR)/dispatch.h -HVF_LDFLAGS := -framework Hypervisor -arch arm64 +# -lz: gzip-compressed OCI layers route through zlib (system library). +# -lcurl: HTTPS fetch for the Phase 1 oci pull path. +HVF_LDFLAGS := -framework Hypervisor -arch arm64 -lcurl -lz # Generated headers under build/ that must exist before compiling sources that # include them. @@ -139,6 +205,215 @@ $(BUILD_DIR)/test-proctitle-host: $(BUILD_DIR)/test-proctitle-host.o \ @echo " LD $@" $(Q)$(CC) $(CFLAGS) -o $@ $^ +## Build the OCI reference parser unit test (native macOS binary). +## Pure C, no HVF, no codesign required. +$(BUILD_DIR)/test-oci-ref: $(BUILD_DIR)/test-oci-ref.o $(BUILD_DIR)/oci/ref.o | $(BUILD_DIR) + @echo " LD $@" + $(Q)$(CC) $(CFLAGS) -o $@ $^ + +## Build the OCI digest unit test (native macOS binary). Pure C, no HVF. +$(BUILD_DIR)/test-oci-digest: $(BUILD_DIR)/test-oci-digest.o $(BUILD_DIR)/oci/digest.o | $(BUILD_DIR) + @echo " LD $@" + $(Q)$(CC) $(CFLAGS) -o $@ $^ + +## Build the OCI blob store unit test (native macOS binary). Pure C, no HVF. +$(BUILD_DIR)/test-oci-blob-store: $(BUILD_DIR)/test-oci-blob-store.o $(BUILD_DIR)/oci/blob-store.o $(BUILD_DIR)/oci/digest.o | $(BUILD_DIR) + @echo " LD $@" + $(Q)$(CC) $(CFLAGS) -o $@ $^ + +## Build the OCI manifest / index / config parser unit test (native, no HVF). +$(BUILD_DIR)/test-oci-manifest: $(BUILD_DIR)/test-oci-manifest.o $(BUILD_DIR)/oci/manifest.o $(BUILD_DIR)/oci/media-type.o $(BUILD_DIR)/oci/digest.o $(CJSON_OBJ) | $(BUILD_DIR) + @echo " LD $@" + $(Q)$(CC) $(CFLAGS) -o $@ $^ + +## Build the shared OCI mock HTTPS server helper. tests/lib/oci-mock.{c,h} +## terminates TLS via libssl from brew openssl@3; both the fetch and pull +## suites link against the same compiled object to avoid duplicating ~400 LOC +## of scaffolding in their own translation units. +$(BUILD_DIR)/lib/oci-mock.o: CFLAGS += $(OPENSSL_CFLAGS) + +## Build the OCI fetch (libcurl) unit test (native macOS, no HVF). Pulls in +## blob-store + digest + manifest models + cJSON; links against system libcurl +## and the platform pthread runtime for the in-process mock HTTP server. The +## test mock terminates TLS using libssl from brew openssl@3 so the ca_file +## negative cases exercise a real certificate verification path. +$(BUILD_DIR)/test-oci-fetch.o: CFLAGS += $(OPENSSL_CFLAGS) +$(BUILD_DIR)/test-oci-fetch: $(BUILD_DIR)/test-oci-fetch.o $(BUILD_DIR)/lib/oci-mock.o $(BUILD_DIR)/oci/fetch.o $(BUILD_DIR)/oci/policy.o $(BUILD_DIR)/oci/blob-store.o $(BUILD_DIR)/oci/digest.o $(BUILD_DIR)/oci/manifest.o $(BUILD_DIR)/oci/media-type.o $(BUILD_DIR)/oci/ref.o $(CJSON_OBJ) | $(BUILD_DIR) + @echo " LD $@" + $(Q)$(CC) $(CFLAGS) -o $@ $^ -lcurl -lpthread $(OPENSSL_LDFLAGS) + +## Build the OCI local store unit test (native macOS, no HVF). Pure C; links +## against the store wrapper plus its blob-store, digest, and cJSON deps. +## cJSON is required because store.c now reads / writes index.json. +$(BUILD_DIR)/test-oci-store: $(BUILD_DIR)/test-oci-store.o $(BUILD_DIR)/oci/store.o $(BUILD_DIR)/oci/blob-store.o $(BUILD_DIR)/oci/digest.o $(BUILD_DIR)/oci/digest-set.o $(BUILD_DIR)/oci/manifest.o $(BUILD_DIR)/oci/media-type.o $(BUILD_DIR)/oci/origin-meta.o $(BUILD_DIR)/oci/volume-list.o $(BUILD_DIR)/oci/ref.o $(CJSON_OBJ) | $(BUILD_DIR) + @echo " LD $@" + $(Q)$(CC) $(CFLAGS) -o $@ $^ + +## Build the OCI pull pipeline unit test (native macOS, no HVF). Shares the +## TLS-terminating mock server with test-oci-fetch via tests/lib/oci-mock. +$(BUILD_DIR)/test-oci-pull.o: CFLAGS += $(OPENSSL_CFLAGS) +$(BUILD_DIR)/test-oci-pull: $(BUILD_DIR)/test-oci-pull.o $(BUILD_DIR)/lib/oci-mock.o $(BUILD_DIR)/oci/pull.o $(BUILD_DIR)/oci/store.o $(BUILD_DIR)/oci/fetch.o $(BUILD_DIR)/oci/policy.o $(BUILD_DIR)/oci/blob-store.o $(BUILD_DIR)/oci/digest.o $(BUILD_DIR)/oci/digest-set.o $(BUILD_DIR)/oci/manifest.o $(BUILD_DIR)/oci/media-type.o $(BUILD_DIR)/oci/origin-meta.o $(BUILD_DIR)/oci/volume-list.o $(BUILD_DIR)/oci/ref.o $(CJSON_OBJ) | $(BUILD_DIR) + @echo " LD $@" + $(Q)$(CC) $(CFLAGS) -o $@ $^ -lcurl -lpthread $(OPENSSL_LDFLAGS) + +## Build the OCI inspect renderer unit test (native macOS, no HVF). Pure +## offline: no fetcher, no mock server, no libcurl. Pre-populates the store +## via oci_blob_store_put_bytes + oci_store_put_ref. +$(BUILD_DIR)/test-oci-inspect: $(BUILD_DIR)/test-oci-inspect.o $(BUILD_DIR)/oci/inspect.o $(BUILD_DIR)/oci/dedup-metrics.o $(BUILD_DIR)/oci/store.o $(BUILD_DIR)/oci/blob-store.o $(BUILD_DIR)/oci/digest.o $(BUILD_DIR)/oci/digest-set.o $(BUILD_DIR)/oci/manifest.o $(BUILD_DIR)/oci/media-type.o $(BUILD_DIR)/oci/origin-meta.o $(BUILD_DIR)/oci/volume-list.o $(BUILD_DIR)/oci/ref.o $(CJSON_OBJ) | $(BUILD_DIR) + @echo " LD $@" + $(Q)$(CC) $(CFLAGS) -o $@ $^ + +## Build the OCI cross-image dedup metrics unit test (native macOS, no HVF). +## Drives oci_dedup_metrics_compute against scratch stores hand-populated +## via oci_blob_store_put_bytes + oci_store_put_ref. Same dependency set +## as test-oci-inspect, plus oci/dedup-metrics.o. +$(BUILD_DIR)/test-oci-dedup-metrics: $(BUILD_DIR)/test-oci-dedup-metrics.o $(BUILD_DIR)/oci/dedup-metrics.o $(BUILD_DIR)/oci/store.o $(BUILD_DIR)/oci/blob-store.o $(BUILD_DIR)/oci/digest.o $(BUILD_DIR)/oci/digest-set.o $(BUILD_DIR)/oci/manifest.o $(BUILD_DIR)/oci/media-type.o $(BUILD_DIR)/oci/origin-meta.o $(BUILD_DIR)/oci/volume-list.o $(BUILD_DIR)/oci/ref.o $(CJSON_OBJ) | $(BUILD_DIR) + @echo " LD $@" + $(Q)$(CC) $(CFLAGS) -o $@ $^ + +## Build the OCI rebuild-cache unit test (native macOS, no HVF). Drives +## oci_rebuild_cache against scratch stores hand-populated via oci_origin_write +## into a fixture /images/sha256-/ tree, then asserts that +## /layers/stacks/sha256// entries are created (commit) or left +## absent (dry-run). Same dependency set as test-oci-store plus oci/rebuild- +## cache.o. +$(BUILD_DIR)/test-oci-rebuild-cache: $(BUILD_DIR)/test-oci-rebuild-cache.o $(BUILD_DIR)/oci/rebuild-cache.o $(BUILD_DIR)/oci/store.o $(BUILD_DIR)/oci/blob-store.o $(BUILD_DIR)/oci/digest.o $(BUILD_DIR)/oci/digest-set.o $(BUILD_DIR)/oci/manifest.o $(BUILD_DIR)/oci/media-type.o $(BUILD_DIR)/oci/origin-meta.o $(BUILD_DIR)/oci/volume-list.o $(BUILD_DIR)/oci/ref.o $(CJSON_OBJ) | $(BUILD_DIR) + @echo " LD $@" + $(Q)$(CC) $(CFLAGS) -o $@ $^ + +## Build the OCI store-wide status unit test (native macOS, no HVF). Drives +## oci_status_compute against scratch stores hand-populated via +## stage_image / oci_origin_write fixture helpers and asserts the aggregated +## struct fields (pin entries, unpacked entries, reachable + populated +## ratios, store totals). Same dependency set as test-oci-store plus +## oci/status.o. +$(BUILD_DIR)/test-oci-status: $(BUILD_DIR)/test-oci-status.o $(BUILD_DIR)/oci/status.o $(BUILD_DIR)/oci/store.o $(BUILD_DIR)/oci/blob-store.o $(BUILD_DIR)/oci/digest.o $(BUILD_DIR)/oci/digest-set.o $(BUILD_DIR)/oci/manifest.o $(BUILD_DIR)/oci/media-type.o $(BUILD_DIR)/oci/origin-meta.o $(BUILD_DIR)/oci/volume-list.o $(BUILD_DIR)/oci/ref.o $(CJSON_OBJ) | $(BUILD_DIR) + @echo " LD $@" + $(Q)$(CC) $(CFLAGS) -o $@ $^ + +## Build the OCI policy.json schema and loader unit test (native macOS, no HVF). +## Pure C; links against the policy translation unit plus cJSON for the JSON +## parser. Drives oci_policy_load against per-test scratch HOME / XDG / override +## trees under /tmp. +$(BUILD_DIR)/test-oci-policy: $(BUILD_DIR)/test-oci-policy.o $(BUILD_DIR)/oci/policy.o $(CJSON_OBJ) | $(BUILD_DIR) + @echo " LD $@" + $(Q)$(CC) $(CFLAGS) -o $@ $^ + +## Build the OCI tar reader unit test (native macOS, no HVF). Pure C; the +## test constructs ustar / GNU long-name streams in memory and drives them +## through the reader via a callback that exercises short-read chunking. +$(BUILD_DIR)/test-oci-tar: $(BUILD_DIR)/test-oci-tar.o $(BUILD_DIR)/oci/tar.o | $(BUILD_DIR) + @echo " LD $@" + $(Q)$(CC) $(CFLAGS) -o $@ $^ + +## Build the OCI runspec unit test (native macOS, no HVF). Merges +## image-config runtime block + CLI overrides; the rootfs-driven +## symbolic-User cases write /etc/passwd and /etc/group fixtures under +## /tmp, so the link island pulls in oci/user-lookup.o. +$(BUILD_DIR)/test-oci-runspec: $(BUILD_DIR)/test-oci-runspec.o $(BUILD_DIR)/oci/runspec.o $(BUILD_DIR)/oci/user-lookup.o | $(BUILD_DIR) + @echo " LD $@" + $(Q)$(CC) $(CFLAGS) -o $@ $^ + +## Build the OCI User-field resolver unit test (native macOS, no HVF). +## Pure C; the test builds scratch /tmp rootfses with synthetic +## /etc/passwd / /etc/group and drives oci_user_lookup across the seven +## OCI image-spec User shapes plus the policy edges. +$(BUILD_DIR)/test-oci-user: $(BUILD_DIR)/test-oci-user.o $(BUILD_DIR)/oci/user-lookup.o | $(BUILD_DIR) + @echo " LD $@" + $(Q)$(CC) $(CFLAGS) -o $@ $^ + +## Build the OCI path-resolve unit test (native macOS, no HVF). Touches +## the host filesystem to build a small fake sysroot tree and drives +## oci_path_resolve through realpath / stat / symlink-follow scenarios. +## Pure C; no libcurl, no zstd, no HVF. +$(BUILD_DIR)/test-oci-path-resolve: $(BUILD_DIR)/test-oci-path-resolve.o $(BUILD_DIR)/oci/path-resolve.o | $(BUILD_DIR) + @echo " LD $@" + $(Q)$(CC) $(CFLAGS) -o $@ $^ + +## Build the OCI run orchestrator unit test (native macOS, no HVF). Links +## the same OCI graph the unpack test pulls in, plus oci/run.o, +## oci/runspec.o, and oci/path-resolve.o. Does NOT link core/launch.o: +## the test ships an in-file elfuse_launch stub that aborts when called, +## and every case installs a launch hook via oci_run_set_launch_for_testing +## before invoking oci_run, so the real VM bring-up never runs from a test. +$(BUILD_DIR)/test-oci-run: $(BUILD_DIR)/test-oci-run.o $(BUILD_DIR)/oci/run.o $(BUILD_DIR)/oci/runspec.o $(BUILD_DIR)/oci/user-lookup.o $(BUILD_DIR)/oci/path-resolve.o $(BUILD_DIR)/oci/runtime-files.o $(BUILD_DIR)/oci/unpack.o $(BUILD_DIR)/oci/volume.o $(BUILD_DIR)/oci/volume-list.o $(BUILD_DIR)/oci/clone-rootfs.o $(BUILD_DIR)/oci/layer-apply.o $(BUILD_DIR)/oci/layer-meta.o $(BUILD_DIR)/oci/origin-meta.o $(BUILD_DIR)/oci/decompress.o $(BUILD_DIR)/oci/tar.o $(BUILD_DIR)/oci/store.o $(BUILD_DIR)/oci/blob-store.o $(BUILD_DIR)/oci/digest.o $(BUILD_DIR)/oci/digest-set.o $(BUILD_DIR)/oci/manifest.o $(BUILD_DIR)/oci/media-type.o $(BUILD_DIR)/oci/ref.o $(BUILD_DIR)/core/sysroot.o $(BUILD_DIR)/debug/log.o $(CJSON_OBJ) $(ZSTD_OBJS) | $(BUILD_DIR) + @echo " LD $@" + $(Q)$(CC) $(CFLAGS) -o $@ $^ -lz + +## Build the OCI runtime-files injection unit test (native macOS, no HVF). +## Pure C; the test drives oci_runtime_files_inject against scratch +## /tmp/elfuse-rf-* run directories and verifies the synthesised +## /etc/{resolv.conf,hosts,hostname} content. +$(BUILD_DIR)/test-oci-runtime-files: $(BUILD_DIR)/test-oci-runtime-files.o $(BUILD_DIR)/oci/runtime-files.o | $(BUILD_DIR) + @echo " LD $@" + $(Q)$(CC) $(CFLAGS) -o $@ $^ + +## Build the OCI fixture builder (Phase 3 compat tests). Standalone tool +## that synthesises a complete OCI store from uncompressed-tar layers +## plus image-config flags. Used by tests/test-oci-compat.sh and +## available standalone for one-off "shape an image from local files" +## experiments. +$(BUILD_DIR)/oci-fixture-builder: $(BUILD_DIR)/lib/oci-fixture-builder.o $(BUILD_DIR)/oci/store.o $(BUILD_DIR)/oci/blob-store.o $(BUILD_DIR)/oci/digest.o $(BUILD_DIR)/oci/digest-set.o $(BUILD_DIR)/oci/manifest.o $(BUILD_DIR)/oci/media-type.o $(BUILD_DIR)/oci/origin-meta.o $(BUILD_DIR)/oci/volume-list.o $(BUILD_DIR)/oci/ref.o $(CJSON_OBJ) | $(BUILD_DIR) + @echo " LD $@" + $(Q)$(CC) $(CFLAGS) -o $@ $^ + +## decompress.c is the only translation unit in elfuse that includes +## externals/zstd/lib/zstd.h. Attach the zstd include path as a target- +## specific CFLAG so the rest of the codebase never sees zstd headers. +$(BUILD_DIR)/oci/decompress.o: CFLAGS += -I$(ZSTD_DIR)/lib + +## Build the OCI sidecar metadata unit test (native macOS, no HVF). Pure +## C; links against cJSON for the JSON round-trip plus the layer-meta +## translation unit. +$(BUILD_DIR)/test-oci-meta: $(BUILD_DIR)/test-oci-meta.o $(BUILD_DIR)/oci/layer-meta.o $(CJSON_OBJ) | $(BUILD_DIR) + @echo " LD $@" + $(Q)$(CC) $(CFLAGS) -o $@ $^ + +## Build the OCI origin sidecar unit test (native macOS, no HVF). Drives +## oci_origin_write against a tmpdir and verifies the resulting +## .elfuse-origin.json by parsing it back through cJSON. +$(BUILD_DIR)/test-oci-origin: $(BUILD_DIR)/test-oci-origin.o $(BUILD_DIR)/oci/origin-meta.o $(CJSON_OBJ) | $(BUILD_DIR) + @echo " LD $@" + $(Q)$(CC) $(CFLAGS) -o $@ $^ + +## Build the OCI layer applier unit test (native macOS, no HVF). Builds +## tar payloads in memory, drives them through oci_layer_apply into a +## tmp tree, and verifies filesystem state via lstat/readlink. +$(BUILD_DIR)/test-oci-layer-apply: $(BUILD_DIR)/test-oci-layer-apply.o $(BUILD_DIR)/oci/layer-apply.o $(BUILD_DIR)/oci/layer-meta.o $(BUILD_DIR)/oci/tar.o $(CJSON_OBJ) | $(BUILD_DIR) + @echo " LD $@" + $(Q)$(CC) $(CFLAGS) -o $@ $^ + +## Build the OCI volume bootstrap unit test (native macOS, no HVF). +## Default-volume test is gated behind OCI_VOLUME_TEST=1 because it +## costs ~150 ms of hdiutil orchestration on first run. Links +## src/core/sysroot.o for the hdiutil wrappers PR #33 introduced. +$(BUILD_DIR)/test-oci-volume: $(BUILD_DIR)/test-oci-volume.o $(BUILD_DIR)/oci/volume.o $(BUILD_DIR)/core/sysroot.o $(BUILD_DIR)/debug/log.o | $(BUILD_DIR) + @echo " LD $@" + $(Q)$(CC) $(CFLAGS) -o $@ $^ + +## Build the OCI clone-rootfs unit test (native macOS, no HVF). The +## test skips itself if clonefile returns ENOTSUP (non-APFS scratch). +$(BUILD_DIR)/test-oci-clone: $(BUILD_DIR)/test-oci-clone.o $(BUILD_DIR)/oci/clone-rootfs.o | $(BUILD_DIR) + @echo " LD $@" + $(Q)$(CC) $(CFLAGS) -o $@ $^ + +## Build the OCI unpack orchestrator integration smoke (native macOS, +## no HVF). Pulls in the full Phase 2 OCI stack so the dependency +## edges between modules are exercised at link time. +$(BUILD_DIR)/test-oci-unpack: $(BUILD_DIR)/test-oci-unpack.o $(BUILD_DIR)/oci/unpack.o $(BUILD_DIR)/oci/volume.o $(BUILD_DIR)/oci/volume-list.o $(BUILD_DIR)/oci/clone-rootfs.o $(BUILD_DIR)/oci/layer-apply.o $(BUILD_DIR)/oci/layer-meta.o $(BUILD_DIR)/oci/origin-meta.o $(BUILD_DIR)/oci/decompress.o $(BUILD_DIR)/oci/tar.o $(BUILD_DIR)/oci/store.o $(BUILD_DIR)/oci/blob-store.o $(BUILD_DIR)/oci/digest.o $(BUILD_DIR)/oci/digest-set.o $(BUILD_DIR)/oci/manifest.o $(BUILD_DIR)/oci/media-type.o $(BUILD_DIR)/oci/ref.o $(BUILD_DIR)/core/sysroot.o $(BUILD_DIR)/debug/log.o $(CJSON_OBJ) $(ZSTD_OBJS) | $(BUILD_DIR) + @echo " LD $@" + $(Q)$(CC) $(CFLAGS) -o $@ $^ -lz + +## Build the OCI decompression dispatch unit test (native macOS, no HVF). +## Links zstd objects + system zlib so gzip and zstd payloads both round- +## trip through oci_stream_t. The gzip fixture is generated at test time +## via zlib; the zstd fixture is an embedded byte array because the +## vendored libzstd is decode-only. +$(BUILD_DIR)/test-oci-decompress.o: CFLAGS += -I$(ZSTD_DIR)/lib +$(BUILD_DIR)/test-oci-decompress: $(BUILD_DIR)/test-oci-decompress.o $(BUILD_DIR)/oci/decompress.o $(ZSTD_OBJS) | $(BUILD_DIR) + @echo " LD $@" + $(Q)$(CC) $(CFLAGS) -o $@ $^ -lz + # Guest test binaries (cross-compiled, aarch64-linux) # Only used when GUEST_TEST_BINARIES is not set. diff --git a/docs/usage.md b/docs/usage.md index 4cf6946..80e94b2 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -99,6 +99,179 @@ and memory access, and per-thread inspection. Implementation details, including the snapshot protocol used to keep Hypervisor.framework register access on the owning thread, are documented in [internals.md](internals.md). +## Running OCI Images (`elfuse oci run`) + +Phase 3 adds a direct-execution path for pulled OCI images: + +```sh +elfuse oci run [OPTIONS] IMAGE [ARG...] +``` + +The subcommand reads the image's runtime block (Entrypoint, Cmd, Env, +WorkingDir, User) and folds in any CLI overrides, then unpacks the image +into the local APFS sysroot volume, clones a per-run rootfs via APFS +`clonefile(2)`, resolves argv[0] against PATH inside the rootfs, and +hands off to the same VM bring-up the legacy positional-ELF `elfuse` +entry uses. + +The image must already be pulled. `oci run` does not auto-pull on miss. +The usual workflow is: + +```sh +elfuse oci pull alpine:3 +elfuse oci run alpine:3 /bin/sh -c 'echo hello from inside' +``` + +### Options + +| Option | Meaning | +|--------|---------| +| `--store DIR` | Override the local store root | +| `--volume DIR` | Override the APFS sysroot volume mount point | +| `--entrypoint PROG` | Replace the image Entrypoint with `PROG` | +| `-e KEY=VAL`, `--env KEY=VAL` | Set or replace one env var (repeatable) | +| `-e KEY`, `--env KEY` | Import `KEY` from the host environ (repeatable) | +| `-w DIR`, `--workdir DIR` | Override image WorkingDir | +| `-u USER[:GROUP]`, `--user USER[:GROUP]` | Override image User; numeric `UID[:GID]` or symbolic `name[:group]` resolved from the rootfs `/etc/passwd` and `/etc/group` (see [User and WorkingDir](#user-and-workingdir)) | +| `--keep` | Keep the per-run cloned rootfs after exit | +| `--name NAME` | Reserved: deterministic clone-dir suffix (ignored today) | + +### Argv override matrix + +| Image Entrypoint | Image Cmd | CLI ARGV | `--entrypoint` | Result argv | +|--|--|--|--|--| +| set | set | none | none | Entrypoint ++ Cmd | +| set | set | provided | none | Entrypoint ++ CLI ARGV (Cmd dropped) | +| set | none | provided | none | Entrypoint ++ CLI ARGV | +| none | set | none | none | Cmd | +| none | set | provided | none | CLI ARGV (Cmd dropped) | +| set | set | optional | provided | [`--entrypoint`] ++ CLI ARGV | +| none | none | provided | none | CLI ARGV | +| none | none | none | none | `EINVAL` "image has no entrypoint or cmd; pass one on the CLI" | + +### Env merge policy + +The merged guest env is built in this order: + +1. Image `Env` (verbatim, in spec order) +2. Each CLI `-e KEY=VAL` set-or-replaces by key +3. Each CLI `-e KEY` (no `=`) imports the host's value when present, otherwise drops silently +4. `TERM` auto-imported from the host iff the merged env has no `TERM` +5. `PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin` injected iff the merged env has no `PATH` +6. `container=elfuse` injected unconditionally so systemd-style sandbox detection works + +CLI `-e DYLD_*=...` overrides are hard-rejected with `EINVAL`: `DYLD_*` is a +macOS-only loader contract with no meaning inside an aarch64-linux guest. +Image-provided `DYLD_*` entries pass through (the guest ignores them). + +### User and WorkingDir + +`User` accepts seven shapes: the empty string (no override), a numeric +`UID`, `UID:GID`, a symbolic `name`, `name:group`, `uid:group`, or +`name:gid`. Symbolic forms read `/etc/passwd` and `/etc/group` from +the cloned rootfs. A token made entirely of ASCII digits is always +parsed numerically, even when a same-named account ships in the image +(this matches runc semantics, so an image that happens to carry a +`1234` account does not capture `--user 1234`). When the symbolic +form names an account the unpacked layers do not actually carry, +lookup fails closed; `elfuse` never silently falls back to root. +`--user UID` alone defaults GID to the same value. + +`WorkingDir` must be absolute and free of `..` segments. If neither the +image nor the CLI sets it, the guest starts in `/`. The directory is +materialized under the cloned rootfs (`mkdir -p`, mode 0755, best- +effort chown to the resolved uid:gid when `--user` or image User +selects credentials). + +### Scope guardrails + +- Auto-pull on `run` miss -> never; `elfuse oci pull` must run first +- Network policy, `docker run -p`-style port mapping -> later phases +- Live `docker exec`-style attach -> never + +### Runtime host-truth surface + +`elfuse oci run` runs the guest against a freshly cloned per-run +rootfs and a small set of synthesized host-truth files. The rootfs +is produced by APFS `clonefile(2)` against the unpacked image +layers, so the first guest write to any path triggers copy-on-write +in APFS without touching the original image. The clone is removed at +guest exit unless `--keep` is set; nothing is ever pushed back to +the on-disk image, and concurrent `oci run` invocations against the +same image are isolated. + +Three `/etc` files are overwritten in the clone before the guest +starts. Any pre-existing symlink (the common case is +`/etc/resolv.conf -> /run/systemd/resolve/stub-resolv.conf`) is +unlinked first so it does not dangle inside the guest: + +| File | Source | +|--|--| +| `/etc/resolv.conf` | `nameserver` lines harvested from `scutil --dns`; falls back to `8.8.8.8` and `1.1.1.1` on any scutil failure | +| `/etc/hosts` | fixed 5-line block: `localhost`, the ip6-loopback aliases, ip6 link-local multicast, and `127.0.0.1 host.elfuse.internal` | +| `/etc/hostname` | literal string `elfuse` | + +The following pseudo-filesystem paths are synthesized by the host-side +openat interceptor and do not need to exist inside the rootfs: + +| Path | Behavior | +|--|--| +| `/dev/null`, `/dev/zero`, `/dev/random`, `/dev/urandom`, `/dev/tty` | redirected to the host device of the same name | +| `/dev/full` | reads zero-fill, writes of any non-zero length return `ENOSPC` | +| `/dev/console` | mirrored from the controlling tty when present (macOS reserves the real `/dev/console` for the kernel) | +| other `/dev/*` | `ENOENT` | +| `/proc/cpuinfo`, `/proc/meminfo`, `/proc/version` | derived from host sysctl | +| `/proc/self/{maps,exe,status,stat,comm,statm,cgroup}` | synthesized; `cgroup` reports the canonical `0::/` (elfuse runs outside any cgroup hierarchy) | +| `/proc/sys/kernel/{ostype,osrelease,hostname}` | tracks the cached `uname` fields (`Linux`, `6.17.0-20-generic`, `elfuse`) | + +### Libc-adjacent compatibility + +`elfuse` does not patch libc-adjacent payload (NSS modules, time-zone +data, locale data, character-set converters, dynamic-linker cache) +inside the guest. Each item below names the contract `elfuse` honors +and the failure mode an image hits when it does not ship the +matching files. + +- **`/etc/nsswitch.conf`** is read by the guest's libc, not by + `elfuse`. Only the `files` and `dns` backends actually function: + `files` resolves through `/etc/{passwd,group,hosts}` in the cloned + rootfs, and `dns` resolves through host `getaddrinfo` via the + synthesized `/etc/resolv.conf`. Backends such as `systemd`, `sss`, + or `ldap` need their NSS shared object plus a matching daemon, + neither of which `elfuse` provides. +- **NSS shared objects** (`libnss_systemd.so`, `libnss_sss.so`, + `libnss_ldap.so`, ...) are `dlopen`'d by guest libc against its own + loader. `elfuse` never injects NSS modules: they are aarch64-linux + ELF objects against guest libc, so the macOS host has no way to + load them, and the guest can only `dlopen` the modules its image + already carries. +- **tzdata** (`/usr/share/zoneinfo`, `/etc/localtime`, `/etc/timezone`) + ships with the image. `elfuse` does not transcode macOS + `/var/db/timezone/zoneinfo` into the tzdata format; if the image is + missing the needed zone, glibc / musl fall back to UTC. The `TZ` + environment variable is honored as-is and is not rewritten by the + Env merge policy. +- **`/usr/lib/locale/locale-archive`** is not regenerated. glibc + images without a built archive (or the matching `.UTF-8/` + directory) fall back to the `C` locale; locale-aware sort / printf + / strcoll outputs ASCII order. musl images do not use the archive + and are unaffected. +- **`/usr/lib//gconv/`** modules and the `gconv-modules` + index ship with the image. Missing modules surface as `EILSEQ` from + `iconv` / glibc's character-set conversion; this most often shows + up when an image ships a stripped glibc layer. +- **`ld.so.cache`** is not rebuilt. The guest dynamic linker reads + whatever cache the image carries; missing entries fall through to + the linker's library-path search, which is the normal slow path. + +Common workloads and the symptom-to-workaround mapping: + +| Symptom | Trigger | Workaround | +|--|--|--| +| `getaddrinfo` returns `EAI_AGAIN` or an empty result | `/etc/nsswitch.conf` lists a backend (`systemd`, `sss`, ...) that needs a daemon | use a distro whose `nsswitch.conf` is `files dns` (alpine ships this by default; debian needs the file edited) | +| `date`, `strftime` show UTC instead of the expected zone | the image does not contain `/usr/share/zoneinfo/` | install tzdata in the image (`apk add tzdata` / `apt install tzdata`), or pass `-e TZ=UTC` to acknowledge UTC | +| `sort`, `printf`, `strcoll` collate in ASCII order | the image is missing `/usr/lib/locale/locale-archive` or the matching `.UTF-8/` directory | accept the C-locale fallback, run `locale-gen` during the image build, or use a musl-based image (alpine), which does not depend on the archive | + ## Guest Compatibility Model `elfuse` is designed for Linux user-space workloads, not for booting a Linux diff --git a/externals/cjson/LICENSE b/externals/cjson/LICENSE new file mode 100644 index 0000000..78deb04 --- /dev/null +++ b/externals/cjson/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2009-2017 Dave Gamble and cJSON contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + diff --git a/externals/cjson/VENDORING.md b/externals/cjson/VENDORING.md new file mode 100644 index 0000000..7c19953 --- /dev/null +++ b/externals/cjson/VENDORING.md @@ -0,0 +1,35 @@ +# Vendored cJSON + +This directory contains a vendored copy of [cJSON](https://github.com/DaveGamble/cJSON), +the ultralightweight JSON parser written in ANSI C. cJSON ships as a single +`.c` / `.h` pair and is dual-licensed under the MIT license (see `LICENSE`). + +## Why vendored + +The OCI work stays hand-rolled C alongside the existing elfuse codebase: no +Go, no Rust, no `cargo` / `go` in the build matrix. cJSON is the smallest +credible JSON dependency that fits that contract; it is self-contained, has no +external dependencies, and compiles cleanly with `clang` and `gcc` on macOS +and Linux. + +## Version + +Pinned to upstream tag `v1.7.18` (2024-05-13). Fetched with: + +``` +curl -fsSL -o cJSON.h https://raw.githubusercontent.com/DaveGamble/cJSON/v1.7.18/cJSON.h +curl -fsSL -o cJSON.c https://raw.githubusercontent.com/DaveGamble/cJSON/v1.7.18/cJSON.c +curl -fsSL -o LICENSE https://raw.githubusercontent.com/DaveGamble/cJSON/v1.7.18/LICENSE +``` + +## Local modifications + +None. The files are byte-identical to the upstream tag so future security +updates can be applied by re-running the curl commands above. + +## Build integration + +The Makefile compiles `cJSON.c` with project warning flags relaxed: cJSON is +third-party code and its style does not match elfuse's `-Wpedantic +-Wmissing-prototypes -Wshadow` posture. Only `src/oci/` translation units +include `externals/cjson/cJSON.h`; the rest of the codebase never sees it. diff --git a/externals/cjson/cJSON.c b/externals/cjson/cJSON.c new file mode 100644 index 0000000..61483d9 --- /dev/null +++ b/externals/cjson/cJSON.c @@ -0,0 +1,3143 @@ +/* + Copyright (c) 2009-2017 Dave Gamble and cJSON contributors + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +*/ + +/* cJSON */ +/* JSON parser in C. */ + +/* disable warnings about old C89 functions in MSVC */ +#if !defined(_CRT_SECURE_NO_DEPRECATE) && defined(_MSC_VER) +#define _CRT_SECURE_NO_DEPRECATE +#endif + +#ifdef __GNUC__ +#pragma GCC visibility push(default) +#endif +#if defined(_MSC_VER) +#pragma warning (push) +/* disable warning about single line comments in system headers */ +#pragma warning (disable : 4001) +#endif + +#include +#include +#include +#include +#include +#include +#include + +#ifdef ENABLE_LOCALES +#include +#endif + +#if defined(_MSC_VER) +#pragma warning (pop) +#endif +#ifdef __GNUC__ +#pragma GCC visibility pop +#endif + +#include "cJSON.h" + +/* define our own boolean type */ +#ifdef true +#undef true +#endif +#define true ((cJSON_bool)1) + +#ifdef false +#undef false +#endif +#define false ((cJSON_bool)0) + +/* define isnan and isinf for ANSI C, if in C99 or above, isnan and isinf has been defined in math.h */ +#ifndef isinf +#define isinf(d) (isnan((d - d)) && !isnan(d)) +#endif +#ifndef isnan +#define isnan(d) (d != d) +#endif + +#ifndef NAN +#ifdef _WIN32 +#define NAN sqrt(-1.0) +#else +#define NAN 0.0/0.0 +#endif +#endif + +typedef struct { + const unsigned char *json; + size_t position; +} error; +static error global_error = { NULL, 0 }; + +CJSON_PUBLIC(const char *) cJSON_GetErrorPtr(void) +{ + return (const char*) (global_error.json + global_error.position); +} + +CJSON_PUBLIC(char *) cJSON_GetStringValue(const cJSON * const item) +{ + if (!cJSON_IsString(item)) + { + return NULL; + } + + return item->valuestring; +} + +CJSON_PUBLIC(double) cJSON_GetNumberValue(const cJSON * const item) +{ + if (!cJSON_IsNumber(item)) + { + return (double) NAN; + } + + return item->valuedouble; +} + +/* This is a safeguard to prevent copy-pasters from using incompatible C and header files */ +#if (CJSON_VERSION_MAJOR != 1) || (CJSON_VERSION_MINOR != 7) || (CJSON_VERSION_PATCH != 18) + #error cJSON.h and cJSON.c have different versions. Make sure that both have the same. +#endif + +CJSON_PUBLIC(const char*) cJSON_Version(void) +{ + static char version[15]; + sprintf(version, "%i.%i.%i", CJSON_VERSION_MAJOR, CJSON_VERSION_MINOR, CJSON_VERSION_PATCH); + + return version; +} + +/* Case insensitive string comparison, doesn't consider two NULL pointers equal though */ +static int case_insensitive_strcmp(const unsigned char *string1, const unsigned char *string2) +{ + if ((string1 == NULL) || (string2 == NULL)) + { + return 1; + } + + if (string1 == string2) + { + return 0; + } + + for(; tolower(*string1) == tolower(*string2); (void)string1++, string2++) + { + if (*string1 == '\0') + { + return 0; + } + } + + return tolower(*string1) - tolower(*string2); +} + +typedef struct internal_hooks +{ + void *(CJSON_CDECL *allocate)(size_t size); + void (CJSON_CDECL *deallocate)(void *pointer); + void *(CJSON_CDECL *reallocate)(void *pointer, size_t size); +} internal_hooks; + +#if defined(_MSC_VER) +/* work around MSVC error C2322: '...' address of dllimport '...' is not static */ +static void * CJSON_CDECL internal_malloc(size_t size) +{ + return malloc(size); +} +static void CJSON_CDECL internal_free(void *pointer) +{ + free(pointer); +} +static void * CJSON_CDECL internal_realloc(void *pointer, size_t size) +{ + return realloc(pointer, size); +} +#else +#define internal_malloc malloc +#define internal_free free +#define internal_realloc realloc +#endif + +/* strlen of character literals resolved at compile time */ +#define static_strlen(string_literal) (sizeof(string_literal) - sizeof("")) + +static internal_hooks global_hooks = { internal_malloc, internal_free, internal_realloc }; + +static unsigned char* cJSON_strdup(const unsigned char* string, const internal_hooks * const hooks) +{ + size_t length = 0; + unsigned char *copy = NULL; + + if (string == NULL) + { + return NULL; + } + + length = strlen((const char*)string) + sizeof(""); + copy = (unsigned char*)hooks->allocate(length); + if (copy == NULL) + { + return NULL; + } + memcpy(copy, string, length); + + return copy; +} + +CJSON_PUBLIC(void) cJSON_InitHooks(cJSON_Hooks* hooks) +{ + if (hooks == NULL) + { + /* Reset hooks */ + global_hooks.allocate = malloc; + global_hooks.deallocate = free; + global_hooks.reallocate = realloc; + return; + } + + global_hooks.allocate = malloc; + if (hooks->malloc_fn != NULL) + { + global_hooks.allocate = hooks->malloc_fn; + } + + global_hooks.deallocate = free; + if (hooks->free_fn != NULL) + { + global_hooks.deallocate = hooks->free_fn; + } + + /* use realloc only if both free and malloc are used */ + global_hooks.reallocate = NULL; + if ((global_hooks.allocate == malloc) && (global_hooks.deallocate == free)) + { + global_hooks.reallocate = realloc; + } +} + +/* Internal constructor. */ +static cJSON *cJSON_New_Item(const internal_hooks * const hooks) +{ + cJSON* node = (cJSON*)hooks->allocate(sizeof(cJSON)); + if (node) + { + memset(node, '\0', sizeof(cJSON)); + } + + return node; +} + +/* Delete a cJSON structure. */ +CJSON_PUBLIC(void) cJSON_Delete(cJSON *item) +{ + cJSON *next = NULL; + while (item != NULL) + { + next = item->next; + if (!(item->type & cJSON_IsReference) && (item->child != NULL)) + { + cJSON_Delete(item->child); + } + if (!(item->type & cJSON_IsReference) && (item->valuestring != NULL)) + { + global_hooks.deallocate(item->valuestring); + item->valuestring = NULL; + } + if (!(item->type & cJSON_StringIsConst) && (item->string != NULL)) + { + global_hooks.deallocate(item->string); + item->string = NULL; + } + global_hooks.deallocate(item); + item = next; + } +} + +/* get the decimal point character of the current locale */ +static unsigned char get_decimal_point(void) +{ +#ifdef ENABLE_LOCALES + struct lconv *lconv = localeconv(); + return (unsigned char) lconv->decimal_point[0]; +#else + return '.'; +#endif +} + +typedef struct +{ + const unsigned char *content; + size_t length; + size_t offset; + size_t depth; /* How deeply nested (in arrays/objects) is the input at the current offset. */ + internal_hooks hooks; +} parse_buffer; + +/* check if the given size is left to read in a given parse buffer (starting with 1) */ +#define can_read(buffer, size) ((buffer != NULL) && (((buffer)->offset + size) <= (buffer)->length)) +/* check if the buffer can be accessed at the given index (starting with 0) */ +#define can_access_at_index(buffer, index) ((buffer != NULL) && (((buffer)->offset + index) < (buffer)->length)) +#define cannot_access_at_index(buffer, index) (!can_access_at_index(buffer, index)) +/* get a pointer to the buffer at the position */ +#define buffer_at_offset(buffer) ((buffer)->content + (buffer)->offset) + +/* Parse the input text to generate a number, and populate the result into item. */ +static cJSON_bool parse_number(cJSON * const item, parse_buffer * const input_buffer) +{ + double number = 0; + unsigned char *after_end = NULL; + unsigned char number_c_string[64]; + unsigned char decimal_point = get_decimal_point(); + size_t i = 0; + + if ((input_buffer == NULL) || (input_buffer->content == NULL)) + { + return false; + } + + /* copy the number into a temporary buffer and replace '.' with the decimal point + * of the current locale (for strtod) + * This also takes care of '\0' not necessarily being available for marking the end of the input */ + for (i = 0; (i < (sizeof(number_c_string) - 1)) && can_access_at_index(input_buffer, i); i++) + { + switch (buffer_at_offset(input_buffer)[i]) + { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + case '+': + case '-': + case 'e': + case 'E': + number_c_string[i] = buffer_at_offset(input_buffer)[i]; + break; + + case '.': + number_c_string[i] = decimal_point; + break; + + default: + goto loop_end; + } + } +loop_end: + number_c_string[i] = '\0'; + + number = strtod((const char*)number_c_string, (char**)&after_end); + if (number_c_string == after_end) + { + return false; /* parse_error */ + } + + item->valuedouble = number; + + /* use saturation in case of overflow */ + if (number >= INT_MAX) + { + item->valueint = INT_MAX; + } + else if (number <= (double)INT_MIN) + { + item->valueint = INT_MIN; + } + else + { + item->valueint = (int)number; + } + + item->type = cJSON_Number; + + input_buffer->offset += (size_t)(after_end - number_c_string); + return true; +} + +/* don't ask me, but the original cJSON_SetNumberValue returns an integer or double */ +CJSON_PUBLIC(double) cJSON_SetNumberHelper(cJSON *object, double number) +{ + if (number >= INT_MAX) + { + object->valueint = INT_MAX; + } + else if (number <= (double)INT_MIN) + { + object->valueint = INT_MIN; + } + else + { + object->valueint = (int)number; + } + + return object->valuedouble = number; +} + +/* Note: when passing a NULL valuestring, cJSON_SetValuestring treats this as an error and return NULL */ +CJSON_PUBLIC(char*) cJSON_SetValuestring(cJSON *object, const char *valuestring) +{ + char *copy = NULL; + /* if object's type is not cJSON_String or is cJSON_IsReference, it should not set valuestring */ + if ((object == NULL) || !(object->type & cJSON_String) || (object->type & cJSON_IsReference)) + { + return NULL; + } + /* return NULL if the object is corrupted or valuestring is NULL */ + if (object->valuestring == NULL || valuestring == NULL) + { + return NULL; + } + if (strlen(valuestring) <= strlen(object->valuestring)) + { + strcpy(object->valuestring, valuestring); + return object->valuestring; + } + copy = (char*) cJSON_strdup((const unsigned char*)valuestring, &global_hooks); + if (copy == NULL) + { + return NULL; + } + if (object->valuestring != NULL) + { + cJSON_free(object->valuestring); + } + object->valuestring = copy; + + return copy; +} + +typedef struct +{ + unsigned char *buffer; + size_t length; + size_t offset; + size_t depth; /* current nesting depth (for formatted printing) */ + cJSON_bool noalloc; + cJSON_bool format; /* is this print a formatted print */ + internal_hooks hooks; +} printbuffer; + +/* realloc printbuffer if necessary to have at least "needed" bytes more */ +static unsigned char* ensure(printbuffer * const p, size_t needed) +{ + unsigned char *newbuffer = NULL; + size_t newsize = 0; + + if ((p == NULL) || (p->buffer == NULL)) + { + return NULL; + } + + if ((p->length > 0) && (p->offset >= p->length)) + { + /* make sure that offset is valid */ + return NULL; + } + + if (needed > INT_MAX) + { + /* sizes bigger than INT_MAX are currently not supported */ + return NULL; + } + + needed += p->offset + 1; + if (needed <= p->length) + { + return p->buffer + p->offset; + } + + if (p->noalloc) { + return NULL; + } + + /* calculate new buffer size */ + if (needed > (INT_MAX / 2)) + { + /* overflow of int, use INT_MAX if possible */ + if (needed <= INT_MAX) + { + newsize = INT_MAX; + } + else + { + return NULL; + } + } + else + { + newsize = needed * 2; + } + + if (p->hooks.reallocate != NULL) + { + /* reallocate with realloc if available */ + newbuffer = (unsigned char*)p->hooks.reallocate(p->buffer, newsize); + if (newbuffer == NULL) + { + p->hooks.deallocate(p->buffer); + p->length = 0; + p->buffer = NULL; + + return NULL; + } + } + else + { + /* otherwise reallocate manually */ + newbuffer = (unsigned char*)p->hooks.allocate(newsize); + if (!newbuffer) + { + p->hooks.deallocate(p->buffer); + p->length = 0; + p->buffer = NULL; + + return NULL; + } + + memcpy(newbuffer, p->buffer, p->offset + 1); + p->hooks.deallocate(p->buffer); + } + p->length = newsize; + p->buffer = newbuffer; + + return newbuffer + p->offset; +} + +/* calculate the new length of the string in a printbuffer and update the offset */ +static void update_offset(printbuffer * const buffer) +{ + const unsigned char *buffer_pointer = NULL; + if ((buffer == NULL) || (buffer->buffer == NULL)) + { + return; + } + buffer_pointer = buffer->buffer + buffer->offset; + + buffer->offset += strlen((const char*)buffer_pointer); +} + +/* securely comparison of floating-point variables */ +static cJSON_bool compare_double(double a, double b) +{ + double maxVal = fabs(a) > fabs(b) ? fabs(a) : fabs(b); + return (fabs(a - b) <= maxVal * DBL_EPSILON); +} + +/* Render the number nicely from the given item into a string. */ +static cJSON_bool print_number(const cJSON * const item, printbuffer * const output_buffer) +{ + unsigned char *output_pointer = NULL; + double d = item->valuedouble; + int length = 0; + size_t i = 0; + unsigned char number_buffer[26] = {0}; /* temporary buffer to print the number into */ + unsigned char decimal_point = get_decimal_point(); + double test = 0.0; + + if (output_buffer == NULL) + { + return false; + } + + /* This checks for NaN and Infinity */ + if (isnan(d) || isinf(d)) + { + length = sprintf((char*)number_buffer, "null"); + } + else if(d == (double)item->valueint) + { + length = sprintf((char*)number_buffer, "%d", item->valueint); + } + else + { + /* Try 15 decimal places of precision to avoid nonsignificant nonzero digits */ + length = sprintf((char*)number_buffer, "%1.15g", d); + + /* Check whether the original double can be recovered */ + if ((sscanf((char*)number_buffer, "%lg", &test) != 1) || !compare_double((double)test, d)) + { + /* If not, print with 17 decimal places of precision */ + length = sprintf((char*)number_buffer, "%1.17g", d); + } + } + + /* sprintf failed or buffer overrun occurred */ + if ((length < 0) || (length > (int)(sizeof(number_buffer) - 1))) + { + return false; + } + + /* reserve appropriate space in the output */ + output_pointer = ensure(output_buffer, (size_t)length + sizeof("")); + if (output_pointer == NULL) + { + return false; + } + + /* copy the printed number to the output and replace locale + * dependent decimal point with '.' */ + for (i = 0; i < ((size_t)length); i++) + { + if (number_buffer[i] == decimal_point) + { + output_pointer[i] = '.'; + continue; + } + + output_pointer[i] = number_buffer[i]; + } + output_pointer[i] = '\0'; + + output_buffer->offset += (size_t)length; + + return true; +} + +/* parse 4 digit hexadecimal number */ +static unsigned parse_hex4(const unsigned char * const input) +{ + unsigned int h = 0; + size_t i = 0; + + for (i = 0; i < 4; i++) + { + /* parse digit */ + if ((input[i] >= '0') && (input[i] <= '9')) + { + h += (unsigned int) input[i] - '0'; + } + else if ((input[i] >= 'A') && (input[i] <= 'F')) + { + h += (unsigned int) 10 + input[i] - 'A'; + } + else if ((input[i] >= 'a') && (input[i] <= 'f')) + { + h += (unsigned int) 10 + input[i] - 'a'; + } + else /* invalid */ + { + return 0; + } + + if (i < 3) + { + /* shift left to make place for the next nibble */ + h = h << 4; + } + } + + return h; +} + +/* converts a UTF-16 literal to UTF-8 + * A literal can be one or two sequences of the form \uXXXX */ +static unsigned char utf16_literal_to_utf8(const unsigned char * const input_pointer, const unsigned char * const input_end, unsigned char **output_pointer) +{ + long unsigned int codepoint = 0; + unsigned int first_code = 0; + const unsigned char *first_sequence = input_pointer; + unsigned char utf8_length = 0; + unsigned char utf8_position = 0; + unsigned char sequence_length = 0; + unsigned char first_byte_mark = 0; + + if ((input_end - first_sequence) < 6) + { + /* input ends unexpectedly */ + goto fail; + } + + /* get the first utf16 sequence */ + first_code = parse_hex4(first_sequence + 2); + + /* check that the code is valid */ + if (((first_code >= 0xDC00) && (first_code <= 0xDFFF))) + { + goto fail; + } + + /* UTF16 surrogate pair */ + if ((first_code >= 0xD800) && (first_code <= 0xDBFF)) + { + const unsigned char *second_sequence = first_sequence + 6; + unsigned int second_code = 0; + sequence_length = 12; /* \uXXXX\uXXXX */ + + if ((input_end - second_sequence) < 6) + { + /* input ends unexpectedly */ + goto fail; + } + + if ((second_sequence[0] != '\\') || (second_sequence[1] != 'u')) + { + /* missing second half of the surrogate pair */ + goto fail; + } + + /* get the second utf16 sequence */ + second_code = parse_hex4(second_sequence + 2); + /* check that the code is valid */ + if ((second_code < 0xDC00) || (second_code > 0xDFFF)) + { + /* invalid second half of the surrogate pair */ + goto fail; + } + + + /* calculate the unicode codepoint from the surrogate pair */ + codepoint = 0x10000 + (((first_code & 0x3FF) << 10) | (second_code & 0x3FF)); + } + else + { + sequence_length = 6; /* \uXXXX */ + codepoint = first_code; + } + + /* encode as UTF-8 + * takes at maximum 4 bytes to encode: + * 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx */ + if (codepoint < 0x80) + { + /* normal ascii, encoding 0xxxxxxx */ + utf8_length = 1; + } + else if (codepoint < 0x800) + { + /* two bytes, encoding 110xxxxx 10xxxxxx */ + utf8_length = 2; + first_byte_mark = 0xC0; /* 11000000 */ + } + else if (codepoint < 0x10000) + { + /* three bytes, encoding 1110xxxx 10xxxxxx 10xxxxxx */ + utf8_length = 3; + first_byte_mark = 0xE0; /* 11100000 */ + } + else if (codepoint <= 0x10FFFF) + { + /* four bytes, encoding 1110xxxx 10xxxxxx 10xxxxxx 10xxxxxx */ + utf8_length = 4; + first_byte_mark = 0xF0; /* 11110000 */ + } + else + { + /* invalid unicode codepoint */ + goto fail; + } + + /* encode as utf8 */ + for (utf8_position = (unsigned char)(utf8_length - 1); utf8_position > 0; utf8_position--) + { + /* 10xxxxxx */ + (*output_pointer)[utf8_position] = (unsigned char)((codepoint | 0x80) & 0xBF); + codepoint >>= 6; + } + /* encode first byte */ + if (utf8_length > 1) + { + (*output_pointer)[0] = (unsigned char)((codepoint | first_byte_mark) & 0xFF); + } + else + { + (*output_pointer)[0] = (unsigned char)(codepoint & 0x7F); + } + + *output_pointer += utf8_length; + + return sequence_length; + +fail: + return 0; +} + +/* Parse the input text into an unescaped cinput, and populate item. */ +static cJSON_bool parse_string(cJSON * const item, parse_buffer * const input_buffer) +{ + const unsigned char *input_pointer = buffer_at_offset(input_buffer) + 1; + const unsigned char *input_end = buffer_at_offset(input_buffer) + 1; + unsigned char *output_pointer = NULL; + unsigned char *output = NULL; + + /* not a string */ + if (buffer_at_offset(input_buffer)[0] != '\"') + { + goto fail; + } + + { + /* calculate approximate size of the output (overestimate) */ + size_t allocation_length = 0; + size_t skipped_bytes = 0; + while (((size_t)(input_end - input_buffer->content) < input_buffer->length) && (*input_end != '\"')) + { + /* is escape sequence */ + if (input_end[0] == '\\') + { + if ((size_t)(input_end + 1 - input_buffer->content) >= input_buffer->length) + { + /* prevent buffer overflow when last input character is a backslash */ + goto fail; + } + skipped_bytes++; + input_end++; + } + input_end++; + } + if (((size_t)(input_end - input_buffer->content) >= input_buffer->length) || (*input_end != '\"')) + { + goto fail; /* string ended unexpectedly */ + } + + /* This is at most how much we need for the output */ + allocation_length = (size_t) (input_end - buffer_at_offset(input_buffer)) - skipped_bytes; + output = (unsigned char*)input_buffer->hooks.allocate(allocation_length + sizeof("")); + if (output == NULL) + { + goto fail; /* allocation failure */ + } + } + + output_pointer = output; + /* loop through the string literal */ + while (input_pointer < input_end) + { + if (*input_pointer != '\\') + { + *output_pointer++ = *input_pointer++; + } + /* escape sequence */ + else + { + unsigned char sequence_length = 2; + if ((input_end - input_pointer) < 1) + { + goto fail; + } + + switch (input_pointer[1]) + { + case 'b': + *output_pointer++ = '\b'; + break; + case 'f': + *output_pointer++ = '\f'; + break; + case 'n': + *output_pointer++ = '\n'; + break; + case 'r': + *output_pointer++ = '\r'; + break; + case 't': + *output_pointer++ = '\t'; + break; + case '\"': + case '\\': + case '/': + *output_pointer++ = input_pointer[1]; + break; + + /* UTF-16 literal */ + case 'u': + sequence_length = utf16_literal_to_utf8(input_pointer, input_end, &output_pointer); + if (sequence_length == 0) + { + /* failed to convert UTF16-literal to UTF-8 */ + goto fail; + } + break; + + default: + goto fail; + } + input_pointer += sequence_length; + } + } + + /* zero terminate the output */ + *output_pointer = '\0'; + + item->type = cJSON_String; + item->valuestring = (char*)output; + + input_buffer->offset = (size_t) (input_end - input_buffer->content); + input_buffer->offset++; + + return true; + +fail: + if (output != NULL) + { + input_buffer->hooks.deallocate(output); + output = NULL; + } + + if (input_pointer != NULL) + { + input_buffer->offset = (size_t)(input_pointer - input_buffer->content); + } + + return false; +} + +/* Render the cstring provided to an escaped version that can be printed. */ +static cJSON_bool print_string_ptr(const unsigned char * const input, printbuffer * const output_buffer) +{ + const unsigned char *input_pointer = NULL; + unsigned char *output = NULL; + unsigned char *output_pointer = NULL; + size_t output_length = 0; + /* numbers of additional characters needed for escaping */ + size_t escape_characters = 0; + + if (output_buffer == NULL) + { + return false; + } + + /* empty string */ + if (input == NULL) + { + output = ensure(output_buffer, sizeof("\"\"")); + if (output == NULL) + { + return false; + } + strcpy((char*)output, "\"\""); + + return true; + } + + /* set "flag" to 1 if something needs to be escaped */ + for (input_pointer = input; *input_pointer; input_pointer++) + { + switch (*input_pointer) + { + case '\"': + case '\\': + case '\b': + case '\f': + case '\n': + case '\r': + case '\t': + /* one character escape sequence */ + escape_characters++; + break; + default: + if (*input_pointer < 32) + { + /* UTF-16 escape sequence uXXXX */ + escape_characters += 5; + } + break; + } + } + output_length = (size_t)(input_pointer - input) + escape_characters; + + output = ensure(output_buffer, output_length + sizeof("\"\"")); + if (output == NULL) + { + return false; + } + + /* no characters have to be escaped */ + if (escape_characters == 0) + { + output[0] = '\"'; + memcpy(output + 1, input, output_length); + output[output_length + 1] = '\"'; + output[output_length + 2] = '\0'; + + return true; + } + + output[0] = '\"'; + output_pointer = output + 1; + /* copy the string */ + for (input_pointer = input; *input_pointer != '\0'; (void)input_pointer++, output_pointer++) + { + if ((*input_pointer > 31) && (*input_pointer != '\"') && (*input_pointer != '\\')) + { + /* normal character, copy */ + *output_pointer = *input_pointer; + } + else + { + /* character needs to be escaped */ + *output_pointer++ = '\\'; + switch (*input_pointer) + { + case '\\': + *output_pointer = '\\'; + break; + case '\"': + *output_pointer = '\"'; + break; + case '\b': + *output_pointer = 'b'; + break; + case '\f': + *output_pointer = 'f'; + break; + case '\n': + *output_pointer = 'n'; + break; + case '\r': + *output_pointer = 'r'; + break; + case '\t': + *output_pointer = 't'; + break; + default: + /* escape and print as unicode codepoint */ + sprintf((char*)output_pointer, "u%04x", *input_pointer); + output_pointer += 4; + break; + } + } + } + output[output_length + 1] = '\"'; + output[output_length + 2] = '\0'; + + return true; +} + +/* Invoke print_string_ptr (which is useful) on an item. */ +static cJSON_bool print_string(const cJSON * const item, printbuffer * const p) +{ + return print_string_ptr((unsigned char*)item->valuestring, p); +} + +/* Predeclare these prototypes. */ +static cJSON_bool parse_value(cJSON * const item, parse_buffer * const input_buffer); +static cJSON_bool print_value(const cJSON * const item, printbuffer * const output_buffer); +static cJSON_bool parse_array(cJSON * const item, parse_buffer * const input_buffer); +static cJSON_bool print_array(const cJSON * const item, printbuffer * const output_buffer); +static cJSON_bool parse_object(cJSON * const item, parse_buffer * const input_buffer); +static cJSON_bool print_object(const cJSON * const item, printbuffer * const output_buffer); + +/* Utility to jump whitespace and cr/lf */ +static parse_buffer *buffer_skip_whitespace(parse_buffer * const buffer) +{ + if ((buffer == NULL) || (buffer->content == NULL)) + { + return NULL; + } + + if (cannot_access_at_index(buffer, 0)) + { + return buffer; + } + + while (can_access_at_index(buffer, 0) && (buffer_at_offset(buffer)[0] <= 32)) + { + buffer->offset++; + } + + if (buffer->offset == buffer->length) + { + buffer->offset--; + } + + return buffer; +} + +/* skip the UTF-8 BOM (byte order mark) if it is at the beginning of a buffer */ +static parse_buffer *skip_utf8_bom(parse_buffer * const buffer) +{ + if ((buffer == NULL) || (buffer->content == NULL) || (buffer->offset != 0)) + { + return NULL; + } + + if (can_access_at_index(buffer, 4) && (strncmp((const char*)buffer_at_offset(buffer), "\xEF\xBB\xBF", 3) == 0)) + { + buffer->offset += 3; + } + + return buffer; +} + +CJSON_PUBLIC(cJSON *) cJSON_ParseWithOpts(const char *value, const char **return_parse_end, cJSON_bool require_null_terminated) +{ + size_t buffer_length; + + if (NULL == value) + { + return NULL; + } + + /* Adding null character size due to require_null_terminated. */ + buffer_length = strlen(value) + sizeof(""); + + return cJSON_ParseWithLengthOpts(value, buffer_length, return_parse_end, require_null_terminated); +} + +/* Parse an object - create a new root, and populate. */ +CJSON_PUBLIC(cJSON *) cJSON_ParseWithLengthOpts(const char *value, size_t buffer_length, const char **return_parse_end, cJSON_bool require_null_terminated) +{ + parse_buffer buffer = { 0, 0, 0, 0, { 0, 0, 0 } }; + cJSON *item = NULL; + + /* reset error position */ + global_error.json = NULL; + global_error.position = 0; + + if (value == NULL || 0 == buffer_length) + { + goto fail; + } + + buffer.content = (const unsigned char*)value; + buffer.length = buffer_length; + buffer.offset = 0; + buffer.hooks = global_hooks; + + item = cJSON_New_Item(&global_hooks); + if (item == NULL) /* memory fail */ + { + goto fail; + } + + if (!parse_value(item, buffer_skip_whitespace(skip_utf8_bom(&buffer)))) + { + /* parse failure. ep is set. */ + goto fail; + } + + /* if we require null-terminated JSON without appended garbage, skip and then check for a null terminator */ + if (require_null_terminated) + { + buffer_skip_whitespace(&buffer); + if ((buffer.offset >= buffer.length) || buffer_at_offset(&buffer)[0] != '\0') + { + goto fail; + } + } + if (return_parse_end) + { + *return_parse_end = (const char*)buffer_at_offset(&buffer); + } + + return item; + +fail: + if (item != NULL) + { + cJSON_Delete(item); + } + + if (value != NULL) + { + error local_error; + local_error.json = (const unsigned char*)value; + local_error.position = 0; + + if (buffer.offset < buffer.length) + { + local_error.position = buffer.offset; + } + else if (buffer.length > 0) + { + local_error.position = buffer.length - 1; + } + + if (return_parse_end != NULL) + { + *return_parse_end = (const char*)local_error.json + local_error.position; + } + + global_error = local_error; + } + + return NULL; +} + +/* Default options for cJSON_Parse */ +CJSON_PUBLIC(cJSON *) cJSON_Parse(const char *value) +{ + return cJSON_ParseWithOpts(value, 0, 0); +} + +CJSON_PUBLIC(cJSON *) cJSON_ParseWithLength(const char *value, size_t buffer_length) +{ + return cJSON_ParseWithLengthOpts(value, buffer_length, 0, 0); +} + +#define cjson_min(a, b) (((a) < (b)) ? (a) : (b)) + +static unsigned char *print(const cJSON * const item, cJSON_bool format, const internal_hooks * const hooks) +{ + static const size_t default_buffer_size = 256; + printbuffer buffer[1]; + unsigned char *printed = NULL; + + memset(buffer, 0, sizeof(buffer)); + + /* create buffer */ + buffer->buffer = (unsigned char*) hooks->allocate(default_buffer_size); + buffer->length = default_buffer_size; + buffer->format = format; + buffer->hooks = *hooks; + if (buffer->buffer == NULL) + { + goto fail; + } + + /* print the value */ + if (!print_value(item, buffer)) + { + goto fail; + } + update_offset(buffer); + + /* check if reallocate is available */ + if (hooks->reallocate != NULL) + { + printed = (unsigned char*) hooks->reallocate(buffer->buffer, buffer->offset + 1); + if (printed == NULL) { + goto fail; + } + buffer->buffer = NULL; + } + else /* otherwise copy the JSON over to a new buffer */ + { + printed = (unsigned char*) hooks->allocate(buffer->offset + 1); + if (printed == NULL) + { + goto fail; + } + memcpy(printed, buffer->buffer, cjson_min(buffer->length, buffer->offset + 1)); + printed[buffer->offset] = '\0'; /* just to be sure */ + + /* free the buffer */ + hooks->deallocate(buffer->buffer); + buffer->buffer = NULL; + } + + return printed; + +fail: + if (buffer->buffer != NULL) + { + hooks->deallocate(buffer->buffer); + buffer->buffer = NULL; + } + + if (printed != NULL) + { + hooks->deallocate(printed); + printed = NULL; + } + + return NULL; +} + +/* Render a cJSON item/entity/structure to text. */ +CJSON_PUBLIC(char *) cJSON_Print(const cJSON *item) +{ + return (char*)print(item, true, &global_hooks); +} + +CJSON_PUBLIC(char *) cJSON_PrintUnformatted(const cJSON *item) +{ + return (char*)print(item, false, &global_hooks); +} + +CJSON_PUBLIC(char *) cJSON_PrintBuffered(const cJSON *item, int prebuffer, cJSON_bool fmt) +{ + printbuffer p = { 0, 0, 0, 0, 0, 0, { 0, 0, 0 } }; + + if (prebuffer < 0) + { + return NULL; + } + + p.buffer = (unsigned char*)global_hooks.allocate((size_t)prebuffer); + if (!p.buffer) + { + return NULL; + } + + p.length = (size_t)prebuffer; + p.offset = 0; + p.noalloc = false; + p.format = fmt; + p.hooks = global_hooks; + + if (!print_value(item, &p)) + { + global_hooks.deallocate(p.buffer); + p.buffer = NULL; + return NULL; + } + + return (char*)p.buffer; +} + +CJSON_PUBLIC(cJSON_bool) cJSON_PrintPreallocated(cJSON *item, char *buffer, const int length, const cJSON_bool format) +{ + printbuffer p = { 0, 0, 0, 0, 0, 0, { 0, 0, 0 } }; + + if ((length < 0) || (buffer == NULL)) + { + return false; + } + + p.buffer = (unsigned char*)buffer; + p.length = (size_t)length; + p.offset = 0; + p.noalloc = true; + p.format = format; + p.hooks = global_hooks; + + return print_value(item, &p); +} + +/* Parser core - when encountering text, process appropriately. */ +static cJSON_bool parse_value(cJSON * const item, parse_buffer * const input_buffer) +{ + if ((input_buffer == NULL) || (input_buffer->content == NULL)) + { + return false; /* no input */ + } + + /* parse the different types of values */ + /* null */ + if (can_read(input_buffer, 4) && (strncmp((const char*)buffer_at_offset(input_buffer), "null", 4) == 0)) + { + item->type = cJSON_NULL; + input_buffer->offset += 4; + return true; + } + /* false */ + if (can_read(input_buffer, 5) && (strncmp((const char*)buffer_at_offset(input_buffer), "false", 5) == 0)) + { + item->type = cJSON_False; + input_buffer->offset += 5; + return true; + } + /* true */ + if (can_read(input_buffer, 4) && (strncmp((const char*)buffer_at_offset(input_buffer), "true", 4) == 0)) + { + item->type = cJSON_True; + item->valueint = 1; + input_buffer->offset += 4; + return true; + } + /* string */ + if (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == '\"')) + { + return parse_string(item, input_buffer); + } + /* number */ + if (can_access_at_index(input_buffer, 0) && ((buffer_at_offset(input_buffer)[0] == '-') || ((buffer_at_offset(input_buffer)[0] >= '0') && (buffer_at_offset(input_buffer)[0] <= '9')))) + { + return parse_number(item, input_buffer); + } + /* array */ + if (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == '[')) + { + return parse_array(item, input_buffer); + } + /* object */ + if (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == '{')) + { + return parse_object(item, input_buffer); + } + + return false; +} + +/* Render a value to text. */ +static cJSON_bool print_value(const cJSON * const item, printbuffer * const output_buffer) +{ + unsigned char *output = NULL; + + if ((item == NULL) || (output_buffer == NULL)) + { + return false; + } + + switch ((item->type) & 0xFF) + { + case cJSON_NULL: + output = ensure(output_buffer, 5); + if (output == NULL) + { + return false; + } + strcpy((char*)output, "null"); + return true; + + case cJSON_False: + output = ensure(output_buffer, 6); + if (output == NULL) + { + return false; + } + strcpy((char*)output, "false"); + return true; + + case cJSON_True: + output = ensure(output_buffer, 5); + if (output == NULL) + { + return false; + } + strcpy((char*)output, "true"); + return true; + + case cJSON_Number: + return print_number(item, output_buffer); + + case cJSON_Raw: + { + size_t raw_length = 0; + if (item->valuestring == NULL) + { + return false; + } + + raw_length = strlen(item->valuestring) + sizeof(""); + output = ensure(output_buffer, raw_length); + if (output == NULL) + { + return false; + } + memcpy(output, item->valuestring, raw_length); + return true; + } + + case cJSON_String: + return print_string(item, output_buffer); + + case cJSON_Array: + return print_array(item, output_buffer); + + case cJSON_Object: + return print_object(item, output_buffer); + + default: + return false; + } +} + +/* Build an array from input text. */ +static cJSON_bool parse_array(cJSON * const item, parse_buffer * const input_buffer) +{ + cJSON *head = NULL; /* head of the linked list */ + cJSON *current_item = NULL; + + if (input_buffer->depth >= CJSON_NESTING_LIMIT) + { + return false; /* to deeply nested */ + } + input_buffer->depth++; + + if (buffer_at_offset(input_buffer)[0] != '[') + { + /* not an array */ + goto fail; + } + + input_buffer->offset++; + buffer_skip_whitespace(input_buffer); + if (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == ']')) + { + /* empty array */ + goto success; + } + + /* check if we skipped to the end of the buffer */ + if (cannot_access_at_index(input_buffer, 0)) + { + input_buffer->offset--; + goto fail; + } + + /* step back to character in front of the first element */ + input_buffer->offset--; + /* loop through the comma separated array elements */ + do + { + /* allocate next item */ + cJSON *new_item = cJSON_New_Item(&(input_buffer->hooks)); + if (new_item == NULL) + { + goto fail; /* allocation failure */ + } + + /* attach next item to list */ + if (head == NULL) + { + /* start the linked list */ + current_item = head = new_item; + } + else + { + /* add to the end and advance */ + current_item->next = new_item; + new_item->prev = current_item; + current_item = new_item; + } + + /* parse next value */ + input_buffer->offset++; + buffer_skip_whitespace(input_buffer); + if (!parse_value(current_item, input_buffer)) + { + goto fail; /* failed to parse value */ + } + buffer_skip_whitespace(input_buffer); + } + while (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == ',')); + + if (cannot_access_at_index(input_buffer, 0) || buffer_at_offset(input_buffer)[0] != ']') + { + goto fail; /* expected end of array */ + } + +success: + input_buffer->depth--; + + if (head != NULL) { + head->prev = current_item; + } + + item->type = cJSON_Array; + item->child = head; + + input_buffer->offset++; + + return true; + +fail: + if (head != NULL) + { + cJSON_Delete(head); + } + + return false; +} + +/* Render an array to text */ +static cJSON_bool print_array(const cJSON * const item, printbuffer * const output_buffer) +{ + unsigned char *output_pointer = NULL; + size_t length = 0; + cJSON *current_element = item->child; + + if (output_buffer == NULL) + { + return false; + } + + /* Compose the output array. */ + /* opening square bracket */ + output_pointer = ensure(output_buffer, 1); + if (output_pointer == NULL) + { + return false; + } + + *output_pointer = '['; + output_buffer->offset++; + output_buffer->depth++; + + while (current_element != NULL) + { + if (!print_value(current_element, output_buffer)) + { + return false; + } + update_offset(output_buffer); + if (current_element->next) + { + length = (size_t) (output_buffer->format ? 2 : 1); + output_pointer = ensure(output_buffer, length + 1); + if (output_pointer == NULL) + { + return false; + } + *output_pointer++ = ','; + if(output_buffer->format) + { + *output_pointer++ = ' '; + } + *output_pointer = '\0'; + output_buffer->offset += length; + } + current_element = current_element->next; + } + + output_pointer = ensure(output_buffer, 2); + if (output_pointer == NULL) + { + return false; + } + *output_pointer++ = ']'; + *output_pointer = '\0'; + output_buffer->depth--; + + return true; +} + +/* Build an object from the text. */ +static cJSON_bool parse_object(cJSON * const item, parse_buffer * const input_buffer) +{ + cJSON *head = NULL; /* linked list head */ + cJSON *current_item = NULL; + + if (input_buffer->depth >= CJSON_NESTING_LIMIT) + { + return false; /* to deeply nested */ + } + input_buffer->depth++; + + if (cannot_access_at_index(input_buffer, 0) || (buffer_at_offset(input_buffer)[0] != '{')) + { + goto fail; /* not an object */ + } + + input_buffer->offset++; + buffer_skip_whitespace(input_buffer); + if (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == '}')) + { + goto success; /* empty object */ + } + + /* check if we skipped to the end of the buffer */ + if (cannot_access_at_index(input_buffer, 0)) + { + input_buffer->offset--; + goto fail; + } + + /* step back to character in front of the first element */ + input_buffer->offset--; + /* loop through the comma separated array elements */ + do + { + /* allocate next item */ + cJSON *new_item = cJSON_New_Item(&(input_buffer->hooks)); + if (new_item == NULL) + { + goto fail; /* allocation failure */ + } + + /* attach next item to list */ + if (head == NULL) + { + /* start the linked list */ + current_item = head = new_item; + } + else + { + /* add to the end and advance */ + current_item->next = new_item; + new_item->prev = current_item; + current_item = new_item; + } + + if (cannot_access_at_index(input_buffer, 1)) + { + goto fail; /* nothing comes after the comma */ + } + + /* parse the name of the child */ + input_buffer->offset++; + buffer_skip_whitespace(input_buffer); + if (!parse_string(current_item, input_buffer)) + { + goto fail; /* failed to parse name */ + } + buffer_skip_whitespace(input_buffer); + + /* swap valuestring and string, because we parsed the name */ + current_item->string = current_item->valuestring; + current_item->valuestring = NULL; + + if (cannot_access_at_index(input_buffer, 0) || (buffer_at_offset(input_buffer)[0] != ':')) + { + goto fail; /* invalid object */ + } + + /* parse the value */ + input_buffer->offset++; + buffer_skip_whitespace(input_buffer); + if (!parse_value(current_item, input_buffer)) + { + goto fail; /* failed to parse value */ + } + buffer_skip_whitespace(input_buffer); + } + while (can_access_at_index(input_buffer, 0) && (buffer_at_offset(input_buffer)[0] == ',')); + + if (cannot_access_at_index(input_buffer, 0) || (buffer_at_offset(input_buffer)[0] != '}')) + { + goto fail; /* expected end of object */ + } + +success: + input_buffer->depth--; + + if (head != NULL) { + head->prev = current_item; + } + + item->type = cJSON_Object; + item->child = head; + + input_buffer->offset++; + return true; + +fail: + if (head != NULL) + { + cJSON_Delete(head); + } + + return false; +} + +/* Render an object to text. */ +static cJSON_bool print_object(const cJSON * const item, printbuffer * const output_buffer) +{ + unsigned char *output_pointer = NULL; + size_t length = 0; + cJSON *current_item = item->child; + + if (output_buffer == NULL) + { + return false; + } + + /* Compose the output: */ + length = (size_t) (output_buffer->format ? 2 : 1); /* fmt: {\n */ + output_pointer = ensure(output_buffer, length + 1); + if (output_pointer == NULL) + { + return false; + } + + *output_pointer++ = '{'; + output_buffer->depth++; + if (output_buffer->format) + { + *output_pointer++ = '\n'; + } + output_buffer->offset += length; + + while (current_item) + { + if (output_buffer->format) + { + size_t i; + output_pointer = ensure(output_buffer, output_buffer->depth); + if (output_pointer == NULL) + { + return false; + } + for (i = 0; i < output_buffer->depth; i++) + { + *output_pointer++ = '\t'; + } + output_buffer->offset += output_buffer->depth; + } + + /* print key */ + if (!print_string_ptr((unsigned char*)current_item->string, output_buffer)) + { + return false; + } + update_offset(output_buffer); + + length = (size_t) (output_buffer->format ? 2 : 1); + output_pointer = ensure(output_buffer, length); + if (output_pointer == NULL) + { + return false; + } + *output_pointer++ = ':'; + if (output_buffer->format) + { + *output_pointer++ = '\t'; + } + output_buffer->offset += length; + + /* print value */ + if (!print_value(current_item, output_buffer)) + { + return false; + } + update_offset(output_buffer); + + /* print comma if not last */ + length = ((size_t)(output_buffer->format ? 1 : 0) + (size_t)(current_item->next ? 1 : 0)); + output_pointer = ensure(output_buffer, length + 1); + if (output_pointer == NULL) + { + return false; + } + if (current_item->next) + { + *output_pointer++ = ','; + } + + if (output_buffer->format) + { + *output_pointer++ = '\n'; + } + *output_pointer = '\0'; + output_buffer->offset += length; + + current_item = current_item->next; + } + + output_pointer = ensure(output_buffer, output_buffer->format ? (output_buffer->depth + 1) : 2); + if (output_pointer == NULL) + { + return false; + } + if (output_buffer->format) + { + size_t i; + for (i = 0; i < (output_buffer->depth - 1); i++) + { + *output_pointer++ = '\t'; + } + } + *output_pointer++ = '}'; + *output_pointer = '\0'; + output_buffer->depth--; + + return true; +} + +/* Get Array size/item / object item. */ +CJSON_PUBLIC(int) cJSON_GetArraySize(const cJSON *array) +{ + cJSON *child = NULL; + size_t size = 0; + + if (array == NULL) + { + return 0; + } + + child = array->child; + + while(child != NULL) + { + size++; + child = child->next; + } + + /* FIXME: Can overflow here. Cannot be fixed without breaking the API */ + + return (int)size; +} + +static cJSON* get_array_item(const cJSON *array, size_t index) +{ + cJSON *current_child = NULL; + + if (array == NULL) + { + return NULL; + } + + current_child = array->child; + while ((current_child != NULL) && (index > 0)) + { + index--; + current_child = current_child->next; + } + + return current_child; +} + +CJSON_PUBLIC(cJSON *) cJSON_GetArrayItem(const cJSON *array, int index) +{ + if (index < 0) + { + return NULL; + } + + return get_array_item(array, (size_t)index); +} + +static cJSON *get_object_item(const cJSON * const object, const char * const name, const cJSON_bool case_sensitive) +{ + cJSON *current_element = NULL; + + if ((object == NULL) || (name == NULL)) + { + return NULL; + } + + current_element = object->child; + if (case_sensitive) + { + while ((current_element != NULL) && (current_element->string != NULL) && (strcmp(name, current_element->string) != 0)) + { + current_element = current_element->next; + } + } + else + { + while ((current_element != NULL) && (case_insensitive_strcmp((const unsigned char*)name, (const unsigned char*)(current_element->string)) != 0)) + { + current_element = current_element->next; + } + } + + if ((current_element == NULL) || (current_element->string == NULL)) { + return NULL; + } + + return current_element; +} + +CJSON_PUBLIC(cJSON *) cJSON_GetObjectItem(const cJSON * const object, const char * const string) +{ + return get_object_item(object, string, false); +} + +CJSON_PUBLIC(cJSON *) cJSON_GetObjectItemCaseSensitive(const cJSON * const object, const char * const string) +{ + return get_object_item(object, string, true); +} + +CJSON_PUBLIC(cJSON_bool) cJSON_HasObjectItem(const cJSON *object, const char *string) +{ + return cJSON_GetObjectItem(object, string) ? 1 : 0; +} + +/* Utility for array list handling. */ +static void suffix_object(cJSON *prev, cJSON *item) +{ + prev->next = item; + item->prev = prev; +} + +/* Utility for handling references. */ +static cJSON *create_reference(const cJSON *item, const internal_hooks * const hooks) +{ + cJSON *reference = NULL; + if (item == NULL) + { + return NULL; + } + + reference = cJSON_New_Item(hooks); + if (reference == NULL) + { + return NULL; + } + + memcpy(reference, item, sizeof(cJSON)); + reference->string = NULL; + reference->type |= cJSON_IsReference; + reference->next = reference->prev = NULL; + return reference; +} + +static cJSON_bool add_item_to_array(cJSON *array, cJSON *item) +{ + cJSON *child = NULL; + + if ((item == NULL) || (array == NULL) || (array == item)) + { + return false; + } + + child = array->child; + /* + * To find the last item in array quickly, we use prev in array + */ + if (child == NULL) + { + /* list is empty, start new one */ + array->child = item; + item->prev = item; + item->next = NULL; + } + else + { + /* append to the end */ + if (child->prev) + { + suffix_object(child->prev, item); + array->child->prev = item; + } + } + + return true; +} + +/* Add item to array/object. */ +CJSON_PUBLIC(cJSON_bool) cJSON_AddItemToArray(cJSON *array, cJSON *item) +{ + return add_item_to_array(array, item); +} + +#if defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ > 5)))) + #pragma GCC diagnostic push +#endif +#ifdef __GNUC__ +#pragma GCC diagnostic ignored "-Wcast-qual" +#endif +/* helper function to cast away const */ +static void* cast_away_const(const void* string) +{ + return (void*)string; +} +#if defined(__clang__) || (defined(__GNUC__) && ((__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ > 5)))) + #pragma GCC diagnostic pop +#endif + + +static cJSON_bool add_item_to_object(cJSON * const object, const char * const string, cJSON * const item, const internal_hooks * const hooks, const cJSON_bool constant_key) +{ + char *new_key = NULL; + int new_type = cJSON_Invalid; + + if ((object == NULL) || (string == NULL) || (item == NULL) || (object == item)) + { + return false; + } + + if (constant_key) + { + new_key = (char*)cast_away_const(string); + new_type = item->type | cJSON_StringIsConst; + } + else + { + new_key = (char*)cJSON_strdup((const unsigned char*)string, hooks); + if (new_key == NULL) + { + return false; + } + + new_type = item->type & ~cJSON_StringIsConst; + } + + if (!(item->type & cJSON_StringIsConst) && (item->string != NULL)) + { + hooks->deallocate(item->string); + } + + item->string = new_key; + item->type = new_type; + + return add_item_to_array(object, item); +} + +CJSON_PUBLIC(cJSON_bool) cJSON_AddItemToObject(cJSON *object, const char *string, cJSON *item) +{ + return add_item_to_object(object, string, item, &global_hooks, false); +} + +/* Add an item to an object with constant string as key */ +CJSON_PUBLIC(cJSON_bool) cJSON_AddItemToObjectCS(cJSON *object, const char *string, cJSON *item) +{ + return add_item_to_object(object, string, item, &global_hooks, true); +} + +CJSON_PUBLIC(cJSON_bool) cJSON_AddItemReferenceToArray(cJSON *array, cJSON *item) +{ + if (array == NULL) + { + return false; + } + + return add_item_to_array(array, create_reference(item, &global_hooks)); +} + +CJSON_PUBLIC(cJSON_bool) cJSON_AddItemReferenceToObject(cJSON *object, const char *string, cJSON *item) +{ + if ((object == NULL) || (string == NULL)) + { + return false; + } + + return add_item_to_object(object, string, create_reference(item, &global_hooks), &global_hooks, false); +} + +CJSON_PUBLIC(cJSON*) cJSON_AddNullToObject(cJSON * const object, const char * const name) +{ + cJSON *null = cJSON_CreateNull(); + if (add_item_to_object(object, name, null, &global_hooks, false)) + { + return null; + } + + cJSON_Delete(null); + return NULL; +} + +CJSON_PUBLIC(cJSON*) cJSON_AddTrueToObject(cJSON * const object, const char * const name) +{ + cJSON *true_item = cJSON_CreateTrue(); + if (add_item_to_object(object, name, true_item, &global_hooks, false)) + { + return true_item; + } + + cJSON_Delete(true_item); + return NULL; +} + +CJSON_PUBLIC(cJSON*) cJSON_AddFalseToObject(cJSON * const object, const char * const name) +{ + cJSON *false_item = cJSON_CreateFalse(); + if (add_item_to_object(object, name, false_item, &global_hooks, false)) + { + return false_item; + } + + cJSON_Delete(false_item); + return NULL; +} + +CJSON_PUBLIC(cJSON*) cJSON_AddBoolToObject(cJSON * const object, const char * const name, const cJSON_bool boolean) +{ + cJSON *bool_item = cJSON_CreateBool(boolean); + if (add_item_to_object(object, name, bool_item, &global_hooks, false)) + { + return bool_item; + } + + cJSON_Delete(bool_item); + return NULL; +} + +CJSON_PUBLIC(cJSON*) cJSON_AddNumberToObject(cJSON * const object, const char * const name, const double number) +{ + cJSON *number_item = cJSON_CreateNumber(number); + if (add_item_to_object(object, name, number_item, &global_hooks, false)) + { + return number_item; + } + + cJSON_Delete(number_item); + return NULL; +} + +CJSON_PUBLIC(cJSON*) cJSON_AddStringToObject(cJSON * const object, const char * const name, const char * const string) +{ + cJSON *string_item = cJSON_CreateString(string); + if (add_item_to_object(object, name, string_item, &global_hooks, false)) + { + return string_item; + } + + cJSON_Delete(string_item); + return NULL; +} + +CJSON_PUBLIC(cJSON*) cJSON_AddRawToObject(cJSON * const object, const char * const name, const char * const raw) +{ + cJSON *raw_item = cJSON_CreateRaw(raw); + if (add_item_to_object(object, name, raw_item, &global_hooks, false)) + { + return raw_item; + } + + cJSON_Delete(raw_item); + return NULL; +} + +CJSON_PUBLIC(cJSON*) cJSON_AddObjectToObject(cJSON * const object, const char * const name) +{ + cJSON *object_item = cJSON_CreateObject(); + if (add_item_to_object(object, name, object_item, &global_hooks, false)) + { + return object_item; + } + + cJSON_Delete(object_item); + return NULL; +} + +CJSON_PUBLIC(cJSON*) cJSON_AddArrayToObject(cJSON * const object, const char * const name) +{ + cJSON *array = cJSON_CreateArray(); + if (add_item_to_object(object, name, array, &global_hooks, false)) + { + return array; + } + + cJSON_Delete(array); + return NULL; +} + +CJSON_PUBLIC(cJSON *) cJSON_DetachItemViaPointer(cJSON *parent, cJSON * const item) +{ + if ((parent == NULL) || (item == NULL)) + { + return NULL; + } + + if (item != parent->child) + { + /* not the first element */ + item->prev->next = item->next; + } + if (item->next != NULL) + { + /* not the last element */ + item->next->prev = item->prev; + } + + if (item == parent->child) + { + /* first element */ + parent->child = item->next; + } + else if (item->next == NULL) + { + /* last element */ + parent->child->prev = item->prev; + } + + /* make sure the detached item doesn't point anywhere anymore */ + item->prev = NULL; + item->next = NULL; + + return item; +} + +CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromArray(cJSON *array, int which) +{ + if (which < 0) + { + return NULL; + } + + return cJSON_DetachItemViaPointer(array, get_array_item(array, (size_t)which)); +} + +CJSON_PUBLIC(void) cJSON_DeleteItemFromArray(cJSON *array, int which) +{ + cJSON_Delete(cJSON_DetachItemFromArray(array, which)); +} + +CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromObject(cJSON *object, const char *string) +{ + cJSON *to_detach = cJSON_GetObjectItem(object, string); + + return cJSON_DetachItemViaPointer(object, to_detach); +} + +CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromObjectCaseSensitive(cJSON *object, const char *string) +{ + cJSON *to_detach = cJSON_GetObjectItemCaseSensitive(object, string); + + return cJSON_DetachItemViaPointer(object, to_detach); +} + +CJSON_PUBLIC(void) cJSON_DeleteItemFromObject(cJSON *object, const char *string) +{ + cJSON_Delete(cJSON_DetachItemFromObject(object, string)); +} + +CJSON_PUBLIC(void) cJSON_DeleteItemFromObjectCaseSensitive(cJSON *object, const char *string) +{ + cJSON_Delete(cJSON_DetachItemFromObjectCaseSensitive(object, string)); +} + +/* Replace array/object items with new ones. */ +CJSON_PUBLIC(cJSON_bool) cJSON_InsertItemInArray(cJSON *array, int which, cJSON *newitem) +{ + cJSON *after_inserted = NULL; + + if (which < 0 || newitem == NULL) + { + return false; + } + + after_inserted = get_array_item(array, (size_t)which); + if (after_inserted == NULL) + { + return add_item_to_array(array, newitem); + } + + if (after_inserted != array->child && after_inserted->prev == NULL) { + /* return false if after_inserted is a corrupted array item */ + return false; + } + + newitem->next = after_inserted; + newitem->prev = after_inserted->prev; + after_inserted->prev = newitem; + if (after_inserted == array->child) + { + array->child = newitem; + } + else + { + newitem->prev->next = newitem; + } + return true; +} + +CJSON_PUBLIC(cJSON_bool) cJSON_ReplaceItemViaPointer(cJSON * const parent, cJSON * const item, cJSON * replacement) +{ + if ((parent == NULL) || (parent->child == NULL) || (replacement == NULL) || (item == NULL)) + { + return false; + } + + if (replacement == item) + { + return true; + } + + replacement->next = item->next; + replacement->prev = item->prev; + + if (replacement->next != NULL) + { + replacement->next->prev = replacement; + } + if (parent->child == item) + { + if (parent->child->prev == parent->child) + { + replacement->prev = replacement; + } + parent->child = replacement; + } + else + { /* + * To find the last item in array quickly, we use prev in array. + * We can't modify the last item's next pointer where this item was the parent's child + */ + if (replacement->prev != NULL) + { + replacement->prev->next = replacement; + } + if (replacement->next == NULL) + { + parent->child->prev = replacement; + } + } + + item->next = NULL; + item->prev = NULL; + cJSON_Delete(item); + + return true; +} + +CJSON_PUBLIC(cJSON_bool) cJSON_ReplaceItemInArray(cJSON *array, int which, cJSON *newitem) +{ + if (which < 0) + { + return false; + } + + return cJSON_ReplaceItemViaPointer(array, get_array_item(array, (size_t)which), newitem); +} + +static cJSON_bool replace_item_in_object(cJSON *object, const char *string, cJSON *replacement, cJSON_bool case_sensitive) +{ + if ((replacement == NULL) || (string == NULL)) + { + return false; + } + + /* replace the name in the replacement */ + if (!(replacement->type & cJSON_StringIsConst) && (replacement->string != NULL)) + { + cJSON_free(replacement->string); + } + replacement->string = (char*)cJSON_strdup((const unsigned char*)string, &global_hooks); + if (replacement->string == NULL) + { + return false; + } + + replacement->type &= ~cJSON_StringIsConst; + + return cJSON_ReplaceItemViaPointer(object, get_object_item(object, string, case_sensitive), replacement); +} + +CJSON_PUBLIC(cJSON_bool) cJSON_ReplaceItemInObject(cJSON *object, const char *string, cJSON *newitem) +{ + return replace_item_in_object(object, string, newitem, false); +} + +CJSON_PUBLIC(cJSON_bool) cJSON_ReplaceItemInObjectCaseSensitive(cJSON *object, const char *string, cJSON *newitem) +{ + return replace_item_in_object(object, string, newitem, true); +} + +/* Create basic types: */ +CJSON_PUBLIC(cJSON *) cJSON_CreateNull(void) +{ + cJSON *item = cJSON_New_Item(&global_hooks); + if(item) + { + item->type = cJSON_NULL; + } + + return item; +} + +CJSON_PUBLIC(cJSON *) cJSON_CreateTrue(void) +{ + cJSON *item = cJSON_New_Item(&global_hooks); + if(item) + { + item->type = cJSON_True; + } + + return item; +} + +CJSON_PUBLIC(cJSON *) cJSON_CreateFalse(void) +{ + cJSON *item = cJSON_New_Item(&global_hooks); + if(item) + { + item->type = cJSON_False; + } + + return item; +} + +CJSON_PUBLIC(cJSON *) cJSON_CreateBool(cJSON_bool boolean) +{ + cJSON *item = cJSON_New_Item(&global_hooks); + if(item) + { + item->type = boolean ? cJSON_True : cJSON_False; + } + + return item; +} + +CJSON_PUBLIC(cJSON *) cJSON_CreateNumber(double num) +{ + cJSON *item = cJSON_New_Item(&global_hooks); + if(item) + { + item->type = cJSON_Number; + item->valuedouble = num; + + /* use saturation in case of overflow */ + if (num >= INT_MAX) + { + item->valueint = INT_MAX; + } + else if (num <= (double)INT_MIN) + { + item->valueint = INT_MIN; + } + else + { + item->valueint = (int)num; + } + } + + return item; +} + +CJSON_PUBLIC(cJSON *) cJSON_CreateString(const char *string) +{ + cJSON *item = cJSON_New_Item(&global_hooks); + if(item) + { + item->type = cJSON_String; + item->valuestring = (char*)cJSON_strdup((const unsigned char*)string, &global_hooks); + if(!item->valuestring) + { + cJSON_Delete(item); + return NULL; + } + } + + return item; +} + +CJSON_PUBLIC(cJSON *) cJSON_CreateStringReference(const char *string) +{ + cJSON *item = cJSON_New_Item(&global_hooks); + if (item != NULL) + { + item->type = cJSON_String | cJSON_IsReference; + item->valuestring = (char*)cast_away_const(string); + } + + return item; +} + +CJSON_PUBLIC(cJSON *) cJSON_CreateObjectReference(const cJSON *child) +{ + cJSON *item = cJSON_New_Item(&global_hooks); + if (item != NULL) { + item->type = cJSON_Object | cJSON_IsReference; + item->child = (cJSON*)cast_away_const(child); + } + + return item; +} + +CJSON_PUBLIC(cJSON *) cJSON_CreateArrayReference(const cJSON *child) { + cJSON *item = cJSON_New_Item(&global_hooks); + if (item != NULL) { + item->type = cJSON_Array | cJSON_IsReference; + item->child = (cJSON*)cast_away_const(child); + } + + return item; +} + +CJSON_PUBLIC(cJSON *) cJSON_CreateRaw(const char *raw) +{ + cJSON *item = cJSON_New_Item(&global_hooks); + if(item) + { + item->type = cJSON_Raw; + item->valuestring = (char*)cJSON_strdup((const unsigned char*)raw, &global_hooks); + if(!item->valuestring) + { + cJSON_Delete(item); + return NULL; + } + } + + return item; +} + +CJSON_PUBLIC(cJSON *) cJSON_CreateArray(void) +{ + cJSON *item = cJSON_New_Item(&global_hooks); + if(item) + { + item->type=cJSON_Array; + } + + return item; +} + +CJSON_PUBLIC(cJSON *) cJSON_CreateObject(void) +{ + cJSON *item = cJSON_New_Item(&global_hooks); + if (item) + { + item->type = cJSON_Object; + } + + return item; +} + +/* Create Arrays: */ +CJSON_PUBLIC(cJSON *) cJSON_CreateIntArray(const int *numbers, int count) +{ + size_t i = 0; + cJSON *n = NULL; + cJSON *p = NULL; + cJSON *a = NULL; + + if ((count < 0) || (numbers == NULL)) + { + return NULL; + } + + a = cJSON_CreateArray(); + + for(i = 0; a && (i < (size_t)count); i++) + { + n = cJSON_CreateNumber(numbers[i]); + if (!n) + { + cJSON_Delete(a); + return NULL; + } + if(!i) + { + a->child = n; + } + else + { + suffix_object(p, n); + } + p = n; + } + + if (a && a->child) { + a->child->prev = n; + } + + return a; +} + +CJSON_PUBLIC(cJSON *) cJSON_CreateFloatArray(const float *numbers, int count) +{ + size_t i = 0; + cJSON *n = NULL; + cJSON *p = NULL; + cJSON *a = NULL; + + if ((count < 0) || (numbers == NULL)) + { + return NULL; + } + + a = cJSON_CreateArray(); + + for(i = 0; a && (i < (size_t)count); i++) + { + n = cJSON_CreateNumber((double)numbers[i]); + if(!n) + { + cJSON_Delete(a); + return NULL; + } + if(!i) + { + a->child = n; + } + else + { + suffix_object(p, n); + } + p = n; + } + + if (a && a->child) { + a->child->prev = n; + } + + return a; +} + +CJSON_PUBLIC(cJSON *) cJSON_CreateDoubleArray(const double *numbers, int count) +{ + size_t i = 0; + cJSON *n = NULL; + cJSON *p = NULL; + cJSON *a = NULL; + + if ((count < 0) || (numbers == NULL)) + { + return NULL; + } + + a = cJSON_CreateArray(); + + for(i = 0; a && (i < (size_t)count); i++) + { + n = cJSON_CreateNumber(numbers[i]); + if(!n) + { + cJSON_Delete(a); + return NULL; + } + if(!i) + { + a->child = n; + } + else + { + suffix_object(p, n); + } + p = n; + } + + if (a && a->child) { + a->child->prev = n; + } + + return a; +} + +CJSON_PUBLIC(cJSON *) cJSON_CreateStringArray(const char *const *strings, int count) +{ + size_t i = 0; + cJSON *n = NULL; + cJSON *p = NULL; + cJSON *a = NULL; + + if ((count < 0) || (strings == NULL)) + { + return NULL; + } + + a = cJSON_CreateArray(); + + for (i = 0; a && (i < (size_t)count); i++) + { + n = cJSON_CreateString(strings[i]); + if(!n) + { + cJSON_Delete(a); + return NULL; + } + if(!i) + { + a->child = n; + } + else + { + suffix_object(p,n); + } + p = n; + } + + if (a && a->child) { + a->child->prev = n; + } + + return a; +} + +/* Duplication */ +CJSON_PUBLIC(cJSON *) cJSON_Duplicate(const cJSON *item, cJSON_bool recurse) +{ + cJSON *newitem = NULL; + cJSON *child = NULL; + cJSON *next = NULL; + cJSON *newchild = NULL; + + /* Bail on bad ptr */ + if (!item) + { + goto fail; + } + /* Create new item */ + newitem = cJSON_New_Item(&global_hooks); + if (!newitem) + { + goto fail; + } + /* Copy over all vars */ + newitem->type = item->type & (~cJSON_IsReference); + newitem->valueint = item->valueint; + newitem->valuedouble = item->valuedouble; + if (item->valuestring) + { + newitem->valuestring = (char*)cJSON_strdup((unsigned char*)item->valuestring, &global_hooks); + if (!newitem->valuestring) + { + goto fail; + } + } + if (item->string) + { + newitem->string = (item->type&cJSON_StringIsConst) ? item->string : (char*)cJSON_strdup((unsigned char*)item->string, &global_hooks); + if (!newitem->string) + { + goto fail; + } + } + /* If non-recursive, then we're done! */ + if (!recurse) + { + return newitem; + } + /* Walk the ->next chain for the child. */ + child = item->child; + while (child != NULL) + { + newchild = cJSON_Duplicate(child, true); /* Duplicate (with recurse) each item in the ->next chain */ + if (!newchild) + { + goto fail; + } + if (next != NULL) + { + /* If newitem->child already set, then crosswire ->prev and ->next and move on */ + next->next = newchild; + newchild->prev = next; + next = newchild; + } + else + { + /* Set newitem->child and move to it */ + newitem->child = newchild; + next = newchild; + } + child = child->next; + } + if (newitem && newitem->child) + { + newitem->child->prev = newchild; + } + + return newitem; + +fail: + if (newitem != NULL) + { + cJSON_Delete(newitem); + } + + return NULL; +} + +static void skip_oneline_comment(char **input) +{ + *input += static_strlen("//"); + + for (; (*input)[0] != '\0'; ++(*input)) + { + if ((*input)[0] == '\n') { + *input += static_strlen("\n"); + return; + } + } +} + +static void skip_multiline_comment(char **input) +{ + *input += static_strlen("/*"); + + for (; (*input)[0] != '\0'; ++(*input)) + { + if (((*input)[0] == '*') && ((*input)[1] == '/')) + { + *input += static_strlen("*/"); + return; + } + } +} + +static void minify_string(char **input, char **output) { + (*output)[0] = (*input)[0]; + *input += static_strlen("\""); + *output += static_strlen("\""); + + + for (; (*input)[0] != '\0'; (void)++(*input), ++(*output)) { + (*output)[0] = (*input)[0]; + + if ((*input)[0] == '\"') { + (*output)[0] = '\"'; + *input += static_strlen("\""); + *output += static_strlen("\""); + return; + } else if (((*input)[0] == '\\') && ((*input)[1] == '\"')) { + (*output)[1] = (*input)[1]; + *input += static_strlen("\""); + *output += static_strlen("\""); + } + } +} + +CJSON_PUBLIC(void) cJSON_Minify(char *json) +{ + char *into = json; + + if (json == NULL) + { + return; + } + + while (json[0] != '\0') + { + switch (json[0]) + { + case ' ': + case '\t': + case '\r': + case '\n': + json++; + break; + + case '/': + if (json[1] == '/') + { + skip_oneline_comment(&json); + } + else if (json[1] == '*') + { + skip_multiline_comment(&json); + } else { + json++; + } + break; + + case '\"': + minify_string(&json, (char**)&into); + break; + + default: + into[0] = json[0]; + json++; + into++; + } + } + + /* and null-terminate. */ + *into = '\0'; +} + +CJSON_PUBLIC(cJSON_bool) cJSON_IsInvalid(const cJSON * const item) +{ + if (item == NULL) + { + return false; + } + + return (item->type & 0xFF) == cJSON_Invalid; +} + +CJSON_PUBLIC(cJSON_bool) cJSON_IsFalse(const cJSON * const item) +{ + if (item == NULL) + { + return false; + } + + return (item->type & 0xFF) == cJSON_False; +} + +CJSON_PUBLIC(cJSON_bool) cJSON_IsTrue(const cJSON * const item) +{ + if (item == NULL) + { + return false; + } + + return (item->type & 0xff) == cJSON_True; +} + + +CJSON_PUBLIC(cJSON_bool) cJSON_IsBool(const cJSON * const item) +{ + if (item == NULL) + { + return false; + } + + return (item->type & (cJSON_True | cJSON_False)) != 0; +} +CJSON_PUBLIC(cJSON_bool) cJSON_IsNull(const cJSON * const item) +{ + if (item == NULL) + { + return false; + } + + return (item->type & 0xFF) == cJSON_NULL; +} + +CJSON_PUBLIC(cJSON_bool) cJSON_IsNumber(const cJSON * const item) +{ + if (item == NULL) + { + return false; + } + + return (item->type & 0xFF) == cJSON_Number; +} + +CJSON_PUBLIC(cJSON_bool) cJSON_IsString(const cJSON * const item) +{ + if (item == NULL) + { + return false; + } + + return (item->type & 0xFF) == cJSON_String; +} + +CJSON_PUBLIC(cJSON_bool) cJSON_IsArray(const cJSON * const item) +{ + if (item == NULL) + { + return false; + } + + return (item->type & 0xFF) == cJSON_Array; +} + +CJSON_PUBLIC(cJSON_bool) cJSON_IsObject(const cJSON * const item) +{ + if (item == NULL) + { + return false; + } + + return (item->type & 0xFF) == cJSON_Object; +} + +CJSON_PUBLIC(cJSON_bool) cJSON_IsRaw(const cJSON * const item) +{ + if (item == NULL) + { + return false; + } + + return (item->type & 0xFF) == cJSON_Raw; +} + +CJSON_PUBLIC(cJSON_bool) cJSON_Compare(const cJSON * const a, const cJSON * const b, const cJSON_bool case_sensitive) +{ + if ((a == NULL) || (b == NULL) || ((a->type & 0xFF) != (b->type & 0xFF))) + { + return false; + } + + /* check if type is valid */ + switch (a->type & 0xFF) + { + case cJSON_False: + case cJSON_True: + case cJSON_NULL: + case cJSON_Number: + case cJSON_String: + case cJSON_Raw: + case cJSON_Array: + case cJSON_Object: + break; + + default: + return false; + } + + /* identical objects are equal */ + if (a == b) + { + return true; + } + + switch (a->type & 0xFF) + { + /* in these cases and equal type is enough */ + case cJSON_False: + case cJSON_True: + case cJSON_NULL: + return true; + + case cJSON_Number: + if (compare_double(a->valuedouble, b->valuedouble)) + { + return true; + } + return false; + + case cJSON_String: + case cJSON_Raw: + if ((a->valuestring == NULL) || (b->valuestring == NULL)) + { + return false; + } + if (strcmp(a->valuestring, b->valuestring) == 0) + { + return true; + } + + return false; + + case cJSON_Array: + { + cJSON *a_element = a->child; + cJSON *b_element = b->child; + + for (; (a_element != NULL) && (b_element != NULL);) + { + if (!cJSON_Compare(a_element, b_element, case_sensitive)) + { + return false; + } + + a_element = a_element->next; + b_element = b_element->next; + } + + /* one of the arrays is longer than the other */ + if (a_element != b_element) { + return false; + } + + return true; + } + + case cJSON_Object: + { + cJSON *a_element = NULL; + cJSON *b_element = NULL; + cJSON_ArrayForEach(a_element, a) + { + /* TODO This has O(n^2) runtime, which is horrible! */ + b_element = get_object_item(b, a_element->string, case_sensitive); + if (b_element == NULL) + { + return false; + } + + if (!cJSON_Compare(a_element, b_element, case_sensitive)) + { + return false; + } + } + + /* doing this twice, once on a and b to prevent true comparison if a subset of b + * TODO: Do this the proper way, this is just a fix for now */ + cJSON_ArrayForEach(b_element, b) + { + a_element = get_object_item(a, b_element->string, case_sensitive); + if (a_element == NULL) + { + return false; + } + + if (!cJSON_Compare(b_element, a_element, case_sensitive)) + { + return false; + } + } + + return true; + } + + default: + return false; + } +} + +CJSON_PUBLIC(void *) cJSON_malloc(size_t size) +{ + return global_hooks.allocate(size); +} + +CJSON_PUBLIC(void) cJSON_free(void *object) +{ + global_hooks.deallocate(object); + object = NULL; +} diff --git a/externals/cjson/cJSON.h b/externals/cjson/cJSON.h new file mode 100644 index 0000000..88cf0bc --- /dev/null +++ b/externals/cjson/cJSON.h @@ -0,0 +1,300 @@ +/* + Copyright (c) 2009-2017 Dave Gamble and cJSON contributors + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. +*/ + +#ifndef cJSON__h +#define cJSON__h + +#ifdef __cplusplus +extern "C" +{ +#endif + +#if !defined(__WINDOWS__) && (defined(WIN32) || defined(WIN64) || defined(_MSC_VER) || defined(_WIN32)) +#define __WINDOWS__ +#endif + +#ifdef __WINDOWS__ + +/* When compiling for windows, we specify a specific calling convention to avoid issues where we are being called from a project with a different default calling convention. For windows you have 3 define options: + +CJSON_HIDE_SYMBOLS - Define this in the case where you don't want to ever dllexport symbols +CJSON_EXPORT_SYMBOLS - Define this on library build when you want to dllexport symbols (default) +CJSON_IMPORT_SYMBOLS - Define this if you want to dllimport symbol + +For *nix builds that support visibility attribute, you can define similar behavior by + +setting default visibility to hidden by adding +-fvisibility=hidden (for gcc) +or +-xldscope=hidden (for sun cc) +to CFLAGS + +then using the CJSON_API_VISIBILITY flag to "export" the same symbols the way CJSON_EXPORT_SYMBOLS does + +*/ + +#define CJSON_CDECL __cdecl +#define CJSON_STDCALL __stdcall + +/* export symbols by default, this is necessary for copy pasting the C and header file */ +#if !defined(CJSON_HIDE_SYMBOLS) && !defined(CJSON_IMPORT_SYMBOLS) && !defined(CJSON_EXPORT_SYMBOLS) +#define CJSON_EXPORT_SYMBOLS +#endif + +#if defined(CJSON_HIDE_SYMBOLS) +#define CJSON_PUBLIC(type) type CJSON_STDCALL +#elif defined(CJSON_EXPORT_SYMBOLS) +#define CJSON_PUBLIC(type) __declspec(dllexport) type CJSON_STDCALL +#elif defined(CJSON_IMPORT_SYMBOLS) +#define CJSON_PUBLIC(type) __declspec(dllimport) type CJSON_STDCALL +#endif +#else /* !__WINDOWS__ */ +#define CJSON_CDECL +#define CJSON_STDCALL + +#if (defined(__GNUC__) || defined(__SUNPRO_CC) || defined (__SUNPRO_C)) && defined(CJSON_API_VISIBILITY) +#define CJSON_PUBLIC(type) __attribute__((visibility("default"))) type +#else +#define CJSON_PUBLIC(type) type +#endif +#endif + +/* project version */ +#define CJSON_VERSION_MAJOR 1 +#define CJSON_VERSION_MINOR 7 +#define CJSON_VERSION_PATCH 18 + +#include + +/* cJSON Types: */ +#define cJSON_Invalid (0) +#define cJSON_False (1 << 0) +#define cJSON_True (1 << 1) +#define cJSON_NULL (1 << 2) +#define cJSON_Number (1 << 3) +#define cJSON_String (1 << 4) +#define cJSON_Array (1 << 5) +#define cJSON_Object (1 << 6) +#define cJSON_Raw (1 << 7) /* raw json */ + +#define cJSON_IsReference 256 +#define cJSON_StringIsConst 512 + +/* The cJSON structure: */ +typedef struct cJSON +{ + /* next/prev allow you to walk array/object chains. Alternatively, use GetArraySize/GetArrayItem/GetObjectItem */ + struct cJSON *next; + struct cJSON *prev; + /* An array or object item will have a child pointer pointing to a chain of the items in the array/object. */ + struct cJSON *child; + + /* The type of the item, as above. */ + int type; + + /* The item's string, if type==cJSON_String and type == cJSON_Raw */ + char *valuestring; + /* writing to valueint is DEPRECATED, use cJSON_SetNumberValue instead */ + int valueint; + /* The item's number, if type==cJSON_Number */ + double valuedouble; + + /* The item's name string, if this item is the child of, or is in the list of subitems of an object. */ + char *string; +} cJSON; + +typedef struct cJSON_Hooks +{ + /* malloc/free are CDECL on Windows regardless of the default calling convention of the compiler, so ensure the hooks allow passing those functions directly. */ + void *(CJSON_CDECL *malloc_fn)(size_t sz); + void (CJSON_CDECL *free_fn)(void *ptr); +} cJSON_Hooks; + +typedef int cJSON_bool; + +/* Limits how deeply nested arrays/objects can be before cJSON rejects to parse them. + * This is to prevent stack overflows. */ +#ifndef CJSON_NESTING_LIMIT +#define CJSON_NESTING_LIMIT 1000 +#endif + +/* returns the version of cJSON as a string */ +CJSON_PUBLIC(const char*) cJSON_Version(void); + +/* Supply malloc, realloc and free functions to cJSON */ +CJSON_PUBLIC(void) cJSON_InitHooks(cJSON_Hooks* hooks); + +/* Memory Management: the caller is always responsible to free the results from all variants of cJSON_Parse (with cJSON_Delete) and cJSON_Print (with stdlib free, cJSON_Hooks.free_fn, or cJSON_free as appropriate). The exception is cJSON_PrintPreallocated, where the caller has full responsibility of the buffer. */ +/* Supply a block of JSON, and this returns a cJSON object you can interrogate. */ +CJSON_PUBLIC(cJSON *) cJSON_Parse(const char *value); +CJSON_PUBLIC(cJSON *) cJSON_ParseWithLength(const char *value, size_t buffer_length); +/* ParseWithOpts allows you to require (and check) that the JSON is null terminated, and to retrieve the pointer to the final byte parsed. */ +/* If you supply a ptr in return_parse_end and parsing fails, then return_parse_end will contain a pointer to the error so will match cJSON_GetErrorPtr(). */ +CJSON_PUBLIC(cJSON *) cJSON_ParseWithOpts(const char *value, const char **return_parse_end, cJSON_bool require_null_terminated); +CJSON_PUBLIC(cJSON *) cJSON_ParseWithLengthOpts(const char *value, size_t buffer_length, const char **return_parse_end, cJSON_bool require_null_terminated); + +/* Render a cJSON entity to text for transfer/storage. */ +CJSON_PUBLIC(char *) cJSON_Print(const cJSON *item); +/* Render a cJSON entity to text for transfer/storage without any formatting. */ +CJSON_PUBLIC(char *) cJSON_PrintUnformatted(const cJSON *item); +/* Render a cJSON entity to text using a buffered strategy. prebuffer is a guess at the final size. guessing well reduces reallocation. fmt=0 gives unformatted, =1 gives formatted */ +CJSON_PUBLIC(char *) cJSON_PrintBuffered(const cJSON *item, int prebuffer, cJSON_bool fmt); +/* Render a cJSON entity to text using a buffer already allocated in memory with given length. Returns 1 on success and 0 on failure. */ +/* NOTE: cJSON is not always 100% accurate in estimating how much memory it will use, so to be safe allocate 5 bytes more than you actually need */ +CJSON_PUBLIC(cJSON_bool) cJSON_PrintPreallocated(cJSON *item, char *buffer, const int length, const cJSON_bool format); +/* Delete a cJSON entity and all subentities. */ +CJSON_PUBLIC(void) cJSON_Delete(cJSON *item); + +/* Returns the number of items in an array (or object). */ +CJSON_PUBLIC(int) cJSON_GetArraySize(const cJSON *array); +/* Retrieve item number "index" from array "array". Returns NULL if unsuccessful. */ +CJSON_PUBLIC(cJSON *) cJSON_GetArrayItem(const cJSON *array, int index); +/* Get item "string" from object. Case insensitive. */ +CJSON_PUBLIC(cJSON *) cJSON_GetObjectItem(const cJSON * const object, const char * const string); +CJSON_PUBLIC(cJSON *) cJSON_GetObjectItemCaseSensitive(const cJSON * const object, const char * const string); +CJSON_PUBLIC(cJSON_bool) cJSON_HasObjectItem(const cJSON *object, const char *string); +/* For analysing failed parses. This returns a pointer to the parse error. You'll probably need to look a few chars back to make sense of it. Defined when cJSON_Parse() returns 0. 0 when cJSON_Parse() succeeds. */ +CJSON_PUBLIC(const char *) cJSON_GetErrorPtr(void); + +/* Check item type and return its value */ +CJSON_PUBLIC(char *) cJSON_GetStringValue(const cJSON * const item); +CJSON_PUBLIC(double) cJSON_GetNumberValue(const cJSON * const item); + +/* These functions check the type of an item */ +CJSON_PUBLIC(cJSON_bool) cJSON_IsInvalid(const cJSON * const item); +CJSON_PUBLIC(cJSON_bool) cJSON_IsFalse(const cJSON * const item); +CJSON_PUBLIC(cJSON_bool) cJSON_IsTrue(const cJSON * const item); +CJSON_PUBLIC(cJSON_bool) cJSON_IsBool(const cJSON * const item); +CJSON_PUBLIC(cJSON_bool) cJSON_IsNull(const cJSON * const item); +CJSON_PUBLIC(cJSON_bool) cJSON_IsNumber(const cJSON * const item); +CJSON_PUBLIC(cJSON_bool) cJSON_IsString(const cJSON * const item); +CJSON_PUBLIC(cJSON_bool) cJSON_IsArray(const cJSON * const item); +CJSON_PUBLIC(cJSON_bool) cJSON_IsObject(const cJSON * const item); +CJSON_PUBLIC(cJSON_bool) cJSON_IsRaw(const cJSON * const item); + +/* These calls create a cJSON item of the appropriate type. */ +CJSON_PUBLIC(cJSON *) cJSON_CreateNull(void); +CJSON_PUBLIC(cJSON *) cJSON_CreateTrue(void); +CJSON_PUBLIC(cJSON *) cJSON_CreateFalse(void); +CJSON_PUBLIC(cJSON *) cJSON_CreateBool(cJSON_bool boolean); +CJSON_PUBLIC(cJSON *) cJSON_CreateNumber(double num); +CJSON_PUBLIC(cJSON *) cJSON_CreateString(const char *string); +/* raw json */ +CJSON_PUBLIC(cJSON *) cJSON_CreateRaw(const char *raw); +CJSON_PUBLIC(cJSON *) cJSON_CreateArray(void); +CJSON_PUBLIC(cJSON *) cJSON_CreateObject(void); + +/* Create a string where valuestring references a string so + * it will not be freed by cJSON_Delete */ +CJSON_PUBLIC(cJSON *) cJSON_CreateStringReference(const char *string); +/* Create an object/array that only references it's elements so + * they will not be freed by cJSON_Delete */ +CJSON_PUBLIC(cJSON *) cJSON_CreateObjectReference(const cJSON *child); +CJSON_PUBLIC(cJSON *) cJSON_CreateArrayReference(const cJSON *child); + +/* These utilities create an Array of count items. + * The parameter count cannot be greater than the number of elements in the number array, otherwise array access will be out of bounds.*/ +CJSON_PUBLIC(cJSON *) cJSON_CreateIntArray(const int *numbers, int count); +CJSON_PUBLIC(cJSON *) cJSON_CreateFloatArray(const float *numbers, int count); +CJSON_PUBLIC(cJSON *) cJSON_CreateDoubleArray(const double *numbers, int count); +CJSON_PUBLIC(cJSON *) cJSON_CreateStringArray(const char *const *strings, int count); + +/* Append item to the specified array/object. */ +CJSON_PUBLIC(cJSON_bool) cJSON_AddItemToArray(cJSON *array, cJSON *item); +CJSON_PUBLIC(cJSON_bool) cJSON_AddItemToObject(cJSON *object, const char *string, cJSON *item); +/* Use this when string is definitely const (i.e. a literal, or as good as), and will definitely survive the cJSON object. + * WARNING: When this function was used, make sure to always check that (item->type & cJSON_StringIsConst) is zero before + * writing to `item->string` */ +CJSON_PUBLIC(cJSON_bool) cJSON_AddItemToObjectCS(cJSON *object, const char *string, cJSON *item); +/* Append reference to item to the specified array/object. Use this when you want to add an existing cJSON to a new cJSON, but don't want to corrupt your existing cJSON. */ +CJSON_PUBLIC(cJSON_bool) cJSON_AddItemReferenceToArray(cJSON *array, cJSON *item); +CJSON_PUBLIC(cJSON_bool) cJSON_AddItemReferenceToObject(cJSON *object, const char *string, cJSON *item); + +/* Remove/Detach items from Arrays/Objects. */ +CJSON_PUBLIC(cJSON *) cJSON_DetachItemViaPointer(cJSON *parent, cJSON * const item); +CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromArray(cJSON *array, int which); +CJSON_PUBLIC(void) cJSON_DeleteItemFromArray(cJSON *array, int which); +CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromObject(cJSON *object, const char *string); +CJSON_PUBLIC(cJSON *) cJSON_DetachItemFromObjectCaseSensitive(cJSON *object, const char *string); +CJSON_PUBLIC(void) cJSON_DeleteItemFromObject(cJSON *object, const char *string); +CJSON_PUBLIC(void) cJSON_DeleteItemFromObjectCaseSensitive(cJSON *object, const char *string); + +/* Update array items. */ +CJSON_PUBLIC(cJSON_bool) cJSON_InsertItemInArray(cJSON *array, int which, cJSON *newitem); /* Shifts pre-existing items to the right. */ +CJSON_PUBLIC(cJSON_bool) cJSON_ReplaceItemViaPointer(cJSON * const parent, cJSON * const item, cJSON * replacement); +CJSON_PUBLIC(cJSON_bool) cJSON_ReplaceItemInArray(cJSON *array, int which, cJSON *newitem); +CJSON_PUBLIC(cJSON_bool) cJSON_ReplaceItemInObject(cJSON *object,const char *string,cJSON *newitem); +CJSON_PUBLIC(cJSON_bool) cJSON_ReplaceItemInObjectCaseSensitive(cJSON *object,const char *string,cJSON *newitem); + +/* Duplicate a cJSON item */ +CJSON_PUBLIC(cJSON *) cJSON_Duplicate(const cJSON *item, cJSON_bool recurse); +/* Duplicate will create a new, identical cJSON item to the one you pass, in new memory that will + * need to be released. With recurse!=0, it will duplicate any children connected to the item. + * The item->next and ->prev pointers are always zero on return from Duplicate. */ +/* Recursively compare two cJSON items for equality. If either a or b is NULL or invalid, they will be considered unequal. + * case_sensitive determines if object keys are treated case sensitive (1) or case insensitive (0) */ +CJSON_PUBLIC(cJSON_bool) cJSON_Compare(const cJSON * const a, const cJSON * const b, const cJSON_bool case_sensitive); + +/* Minify a strings, remove blank characters(such as ' ', '\t', '\r', '\n') from strings. + * The input pointer json cannot point to a read-only address area, such as a string constant, + * but should point to a readable and writable address area. */ +CJSON_PUBLIC(void) cJSON_Minify(char *json); + +/* Helper functions for creating and adding items to an object at the same time. + * They return the added item or NULL on failure. */ +CJSON_PUBLIC(cJSON*) cJSON_AddNullToObject(cJSON * const object, const char * const name); +CJSON_PUBLIC(cJSON*) cJSON_AddTrueToObject(cJSON * const object, const char * const name); +CJSON_PUBLIC(cJSON*) cJSON_AddFalseToObject(cJSON * const object, const char * const name); +CJSON_PUBLIC(cJSON*) cJSON_AddBoolToObject(cJSON * const object, const char * const name, const cJSON_bool boolean); +CJSON_PUBLIC(cJSON*) cJSON_AddNumberToObject(cJSON * const object, const char * const name, const double number); +CJSON_PUBLIC(cJSON*) cJSON_AddStringToObject(cJSON * const object, const char * const name, const char * const string); +CJSON_PUBLIC(cJSON*) cJSON_AddRawToObject(cJSON * const object, const char * const name, const char * const raw); +CJSON_PUBLIC(cJSON*) cJSON_AddObjectToObject(cJSON * const object, const char * const name); +CJSON_PUBLIC(cJSON*) cJSON_AddArrayToObject(cJSON * const object, const char * const name); + +/* When assigning an integer value, it needs to be propagated to valuedouble too. */ +#define cJSON_SetIntValue(object, number) ((object) ? (object)->valueint = (object)->valuedouble = (number) : (number)) +/* helper for the cJSON_SetNumberValue macro */ +CJSON_PUBLIC(double) cJSON_SetNumberHelper(cJSON *object, double number); +#define cJSON_SetNumberValue(object, number) ((object != NULL) ? cJSON_SetNumberHelper(object, (double)number) : (number)) +/* Change the valuestring of a cJSON_String object, only takes effect when type of object is cJSON_String */ +CJSON_PUBLIC(char*) cJSON_SetValuestring(cJSON *object, const char *valuestring); + +/* If the object is not a boolean type this does nothing and returns cJSON_Invalid else it returns the new type*/ +#define cJSON_SetBoolValue(object, boolValue) ( \ + (object != NULL && ((object)->type & (cJSON_False|cJSON_True))) ? \ + (object)->type=((object)->type &(~(cJSON_False|cJSON_True)))|((boolValue)?cJSON_True:cJSON_False) : \ + cJSON_Invalid\ +) + +/* Macro for iterating over an array or object */ +#define cJSON_ArrayForEach(element, array) for(element = (array != NULL) ? (array)->child : NULL; element != NULL; element = element->next) + +/* malloc/free objects using the malloc/free functions that have been set with cJSON_InitHooks */ +CJSON_PUBLIC(void *) cJSON_malloc(size_t size); +CJSON_PUBLIC(void) cJSON_free(void *object); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/externals/zstd/LICENSE b/externals/zstd/LICENSE new file mode 100644 index 0000000..7580028 --- /dev/null +++ b/externals/zstd/LICENSE @@ -0,0 +1,30 @@ +BSD License + +For Zstandard software + +Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook, nor Meta, nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/externals/zstd/VENDORING.md b/externals/zstd/VENDORING.md new file mode 100644 index 0000000..e1e0f5b --- /dev/null +++ b/externals/zstd/VENDORING.md @@ -0,0 +1,72 @@ +# Vendored zstd (decode-only) + +This directory contains a decode-only subset of [zstd](https://github.com/facebook/zstd), +the streaming compression library from Facebook. Phase 2 needs zstd to +decompress OCI image layers that ship with `application/vnd.oci.image.layer.v1.tar+zstd` +or `application/vnd.docker.image.rootfs.diff.tar.zstd` media types. + +Licensed under BSD-3-Clause (see `LICENSE`). + +## Why vendored, decode-only + +The OCI work stays hand-rolled C: no Go, no Rust, no `cargo` / `go` in the +build matrix. zstd is the only OCI-spec layer +compression beyond gzip that has wide registry support, and the upstream +library cleanly separates decoder-only from the full encoder. Phase 2 only +reads layers, so the encoder, dictionary builder, and legacy v01-v06 +support are all excluded. + +Resulting footprint: + +- `lib/{zstd,zstd_errors}.h` +- `lib/common/*.{c,h}` (allocator, error tables, FSE/Huff decoders, + threading stubs, xxhash, portability shims) +- `lib/decompress/*.{c,h}` (the streaming decode state machine) + +Compression, dictBuilder, deprecated, and legacy paths are NOT vendored. +Do not call `ZSTD_compress*`, `ZDICT_*`, or any `ZSTD_v0X_*` symbol. + +## Version + +Pinned to upstream tag `v1.5.6` (2024-03-30). Fetched with: + +``` +curl -fsSL -o zstd-v1.5.6.tar.gz \ + https://github.com/facebook/zstd/archive/refs/tags/v1.5.6.tar.gz +tar -xzf zstd-v1.5.6.tar.gz +SRC=zstd-1.5.6 +cp $SRC/LICENSE externals/zstd/LICENSE +cp $SRC/lib/zstd.h $SRC/lib/zstd_errors.h externals/zstd/lib/ +cp $SRC/lib/common/*.[ch] externals/zstd/lib/common/ +cp $SRC/lib/decompress/*.[ch] externals/zstd/lib/decompress/ +``` + +Note: `lib/decompress/huf_decompress_amd64.S` is intentionally NOT +copied. The build sets `-DZSTD_DISABLE_ASM=1` so `huf_decompress.c` +references no AMD64 assembly symbols; the elfuse host is Apple Silicon +in any case. + +## Local modifications + +None. The files are byte-identical to the upstream tag so future +security updates can be applied by re-running the curl + cp commands +above. + +## Build integration + +The Makefile compiles each `externals/zstd/lib/**/*.c` translation unit +with project warning flags relaxed (`-Wno-pedantic -Wno-shadow +-Wno-strict-prototypes -Wno-missing-prototypes`) and with the +configuration macros: + +``` +-DZSTD_DISABLE_ASM=1 +-DZSTD_LEGACY_SUPPORT=0 +-DZSTD_MULTITHREAD=0 +-DZSTDLIB_VISIBILITY= +``` + +Only `src/oci/decompress.c` includes `externals/zstd/lib/zstd.h`; the +rest of the codebase never sees zstd headers. The zstd objects are +statically embedded into the `elfuse` binary, so no `-lzstd` link line +is needed. diff --git a/externals/zstd/lib/common/allocations.h b/externals/zstd/lib/common/allocations.h new file mode 100644 index 0000000..5e89955 --- /dev/null +++ b/externals/zstd/lib/common/allocations.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* This file provides custom allocation primitives + */ + +#define ZSTD_DEPS_NEED_MALLOC +#include "zstd_deps.h" /* ZSTD_malloc, ZSTD_calloc, ZSTD_free, ZSTD_memset */ + +#include "compiler.h" /* MEM_STATIC */ +#define ZSTD_STATIC_LINKING_ONLY +#include "../zstd.h" /* ZSTD_customMem */ + +#ifndef ZSTD_ALLOCATIONS_H +#define ZSTD_ALLOCATIONS_H + +/* custom memory allocation functions */ + +MEM_STATIC void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem) +{ + if (customMem.customAlloc) + return customMem.customAlloc(customMem.opaque, size); + return ZSTD_malloc(size); +} + +MEM_STATIC void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem) +{ + if (customMem.customAlloc) { + /* calloc implemented as malloc+memset; + * not as efficient as calloc, but next best guess for custom malloc */ + void* const ptr = customMem.customAlloc(customMem.opaque, size); + ZSTD_memset(ptr, 0, size); + return ptr; + } + return ZSTD_calloc(1, size); +} + +MEM_STATIC void ZSTD_customFree(void* ptr, ZSTD_customMem customMem) +{ + if (ptr!=NULL) { + if (customMem.customFree) + customMem.customFree(customMem.opaque, ptr); + else + ZSTD_free(ptr); + } +} + +#endif /* ZSTD_ALLOCATIONS_H */ diff --git a/externals/zstd/lib/common/bits.h b/externals/zstd/lib/common/bits.h new file mode 100644 index 0000000..def56c4 --- /dev/null +++ b/externals/zstd/lib/common/bits.h @@ -0,0 +1,200 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_BITS_H +#define ZSTD_BITS_H + +#include "mem.h" + +MEM_STATIC unsigned ZSTD_countTrailingZeros32_fallback(U32 val) +{ + assert(val != 0); + { + static const U32 DeBruijnBytePos[32] = {0, 1, 28, 2, 29, 14, 24, 3, + 30, 22, 20, 15, 25, 17, 4, 8, + 31, 27, 13, 23, 21, 19, 16, 7, + 26, 12, 18, 6, 11, 5, 10, 9}; + return DeBruijnBytePos[((U32) ((val & -(S32) val) * 0x077CB531U)) >> 27]; + } +} + +MEM_STATIC unsigned ZSTD_countTrailingZeros32(U32 val) +{ + assert(val != 0); +# if defined(_MSC_VER) +# if STATIC_BMI2 == 1 + return (unsigned)_tzcnt_u32(val); +# else + if (val != 0) { + unsigned long r; + _BitScanForward(&r, val); + return (unsigned)r; + } else { + /* Should not reach this code path */ + __assume(0); + } +# endif +# elif defined(__GNUC__) && (__GNUC__ >= 4) + return (unsigned)__builtin_ctz(val); +# else + return ZSTD_countTrailingZeros32_fallback(val); +# endif +} + +MEM_STATIC unsigned ZSTD_countLeadingZeros32_fallback(U32 val) { + assert(val != 0); + { + static const U32 DeBruijnClz[32] = {0, 9, 1, 10, 13, 21, 2, 29, + 11, 14, 16, 18, 22, 25, 3, 30, + 8, 12, 20, 28, 15, 17, 24, 7, + 19, 27, 23, 6, 26, 5, 4, 31}; + val |= val >> 1; + val |= val >> 2; + val |= val >> 4; + val |= val >> 8; + val |= val >> 16; + return 31 - DeBruijnClz[(val * 0x07C4ACDDU) >> 27]; + } +} + +MEM_STATIC unsigned ZSTD_countLeadingZeros32(U32 val) +{ + assert(val != 0); +# if defined(_MSC_VER) +# if STATIC_BMI2 == 1 + return (unsigned)_lzcnt_u32(val); +# else + if (val != 0) { + unsigned long r; + _BitScanReverse(&r, val); + return (unsigned)(31 - r); + } else { + /* Should not reach this code path */ + __assume(0); + } +# endif +# elif defined(__GNUC__) && (__GNUC__ >= 4) + return (unsigned)__builtin_clz(val); +# else + return ZSTD_countLeadingZeros32_fallback(val); +# endif +} + +MEM_STATIC unsigned ZSTD_countTrailingZeros64(U64 val) +{ + assert(val != 0); +# if defined(_MSC_VER) && defined(_WIN64) +# if STATIC_BMI2 == 1 + return (unsigned)_tzcnt_u64(val); +# else + if (val != 0) { + unsigned long r; + _BitScanForward64(&r, val); + return (unsigned)r; + } else { + /* Should not reach this code path */ + __assume(0); + } +# endif +# elif defined(__GNUC__) && (__GNUC__ >= 4) && defined(__LP64__) + return (unsigned)__builtin_ctzll(val); +# else + { + U32 mostSignificantWord = (U32)(val >> 32); + U32 leastSignificantWord = (U32)val; + if (leastSignificantWord == 0) { + return 32 + ZSTD_countTrailingZeros32(mostSignificantWord); + } else { + return ZSTD_countTrailingZeros32(leastSignificantWord); + } + } +# endif +} + +MEM_STATIC unsigned ZSTD_countLeadingZeros64(U64 val) +{ + assert(val != 0); +# if defined(_MSC_VER) && defined(_WIN64) +# if STATIC_BMI2 == 1 + return (unsigned)_lzcnt_u64(val); +# else + if (val != 0) { + unsigned long r; + _BitScanReverse64(&r, val); + return (unsigned)(63 - r); + } else { + /* Should not reach this code path */ + __assume(0); + } +# endif +# elif defined(__GNUC__) && (__GNUC__ >= 4) + return (unsigned)(__builtin_clzll(val)); +# else + { + U32 mostSignificantWord = (U32)(val >> 32); + U32 leastSignificantWord = (U32)val; + if (mostSignificantWord == 0) { + return 32 + ZSTD_countLeadingZeros32(leastSignificantWord); + } else { + return ZSTD_countLeadingZeros32(mostSignificantWord); + } + } +# endif +} + +MEM_STATIC unsigned ZSTD_NbCommonBytes(size_t val) +{ + if (MEM_isLittleEndian()) { + if (MEM_64bits()) { + return ZSTD_countTrailingZeros64((U64)val) >> 3; + } else { + return ZSTD_countTrailingZeros32((U32)val) >> 3; + } + } else { /* Big Endian CPU */ + if (MEM_64bits()) { + return ZSTD_countLeadingZeros64((U64)val) >> 3; + } else { + return ZSTD_countLeadingZeros32((U32)val) >> 3; + } + } +} + +MEM_STATIC unsigned ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus */ +{ + assert(val != 0); + return 31 - ZSTD_countLeadingZeros32(val); +} + +/* ZSTD_rotateRight_*(): + * Rotates a bitfield to the right by "count" bits. + * https://en.wikipedia.org/w/index.php?title=Circular_shift&oldid=991635599#Implementing_circular_shifts + */ +MEM_STATIC +U64 ZSTD_rotateRight_U64(U64 const value, U32 count) { + assert(count < 64); + count &= 0x3F; /* for fickle pattern recognition */ + return (value >> count) | (U64)(value << ((0U - count) & 0x3F)); +} + +MEM_STATIC +U32 ZSTD_rotateRight_U32(U32 const value, U32 count) { + assert(count < 32); + count &= 0x1F; /* for fickle pattern recognition */ + return (value >> count) | (U32)(value << ((0U - count) & 0x1F)); +} + +MEM_STATIC +U16 ZSTD_rotateRight_U16(U16 const value, U32 count) { + assert(count < 16); + count &= 0x0F; /* for fickle pattern recognition */ + return (value >> count) | (U16)(value << ((0U - count) & 0x0F)); +} + +#endif /* ZSTD_BITS_H */ diff --git a/externals/zstd/lib/common/bitstream.h b/externals/zstd/lib/common/bitstream.h new file mode 100644 index 0000000..6760449 --- /dev/null +++ b/externals/zstd/lib/common/bitstream.h @@ -0,0 +1,457 @@ +/* ****************************************************************** + * bitstream + * Part of FSE library + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ +#ifndef BITSTREAM_H_MODULE +#define BITSTREAM_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif +/* +* This API consists of small unitary functions, which must be inlined for best performance. +* Since link-time-optimization is not available for all compilers, +* these functions are defined into a .h to be included. +*/ + +/*-**************************************** +* Dependencies +******************************************/ +#include "mem.h" /* unaligned access routines */ +#include "compiler.h" /* UNLIKELY() */ +#include "debug.h" /* assert(), DEBUGLOG(), RAWLOG() */ +#include "error_private.h" /* error codes and messages */ +#include "bits.h" /* ZSTD_highbit32 */ + + +/*========================================= +* Target specific +=========================================*/ +#ifndef ZSTD_NO_INTRINSICS +# if (defined(__BMI__) || defined(__BMI2__)) && defined(__GNUC__) +# include /* support for bextr (experimental)/bzhi */ +# elif defined(__ICCARM__) +# include +# endif +#endif + +#define STREAM_ACCUMULATOR_MIN_32 25 +#define STREAM_ACCUMULATOR_MIN_64 57 +#define STREAM_ACCUMULATOR_MIN ((U32)(MEM_32bits() ? STREAM_ACCUMULATOR_MIN_32 : STREAM_ACCUMULATOR_MIN_64)) + + +/*-****************************************** +* bitStream encoding API (write forward) +********************************************/ +/* bitStream can mix input from multiple sources. + * A critical property of these streams is that they encode and decode in **reverse** direction. + * So the first bit sequence you add will be the last to be read, like a LIFO stack. + */ +typedef struct { + size_t bitContainer; + unsigned bitPos; + char* startPtr; + char* ptr; + char* endPtr; +} BIT_CStream_t; + +MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* dstBuffer, size_t dstCapacity); +MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits); +MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC); +MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC); + +/* Start with initCStream, providing the size of buffer to write into. +* bitStream will never write outside of this buffer. +* `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code. +* +* bits are first added to a local register. +* Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems. +* Writing data into memory is an explicit operation, performed by the flushBits function. +* Hence keep track how many bits are potentially stored into local register to avoid register overflow. +* After a flushBits, a maximum of 7 bits might still be stored into local register. +* +* Avoid storing elements of more than 24 bits if you want compatibility with 32-bits bitstream readers. +* +* Last operation is to close the bitStream. +* The function returns the final size of CStream in bytes. +* If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable) +*/ + + +/*-******************************************** +* bitStream decoding API (read backward) +**********************************************/ +typedef size_t BitContainerType; +typedef struct { + BitContainerType bitContainer; + unsigned bitsConsumed; + const char* ptr; + const char* start; + const char* limitPtr; +} BIT_DStream_t; + +typedef enum { BIT_DStream_unfinished = 0, /* fully refilled */ + BIT_DStream_endOfBuffer = 1, /* still some bits left in bitstream */ + BIT_DStream_completed = 2, /* bitstream entirely consumed, bit-exact */ + BIT_DStream_overflow = 3 /* user requested more bits than present in bitstream */ + } BIT_DStream_status; /* result of BIT_reloadDStream() */ + +MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize); +MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits); +MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD); +MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD); + + +/* Start by invoking BIT_initDStream(). +* A chunk of the bitStream is then stored into a local register. +* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (BitContainerType). +* You can then retrieve bitFields stored into the local register, **in reverse order**. +* Local register is explicitly reloaded from memory by the BIT_reloadDStream() method. +* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished. +* Otherwise, it can be less than that, so proceed accordingly. +* Checking if DStream has reached its end can be performed with BIT_endOfDStream(). +*/ + + +/*-**************************************** +* unsafe API +******************************************/ +MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits); +/* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */ + +MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC); +/* unsafe version; does not check buffer overflow */ + +MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits); +/* faster, but works only if nbBits >= 1 */ + +/*===== Local Constants =====*/ +static const unsigned BIT_mask[] = { + 0, 1, 3, 7, 0xF, 0x1F, + 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, + 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, + 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, + 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF, 0x7FFFFFF, 0xFFFFFFF, 0x1FFFFFFF, + 0x3FFFFFFF, 0x7FFFFFFF}; /* up to 31 bits */ +#define BIT_MASK_SIZE (sizeof(BIT_mask) / sizeof(BIT_mask[0])) + +/*-************************************************************** +* bitStream encoding +****************************************************************/ +/*! BIT_initCStream() : + * `dstCapacity` must be > sizeof(size_t) + * @return : 0 if success, + * otherwise an error code (can be tested using ERR_isError()) */ +MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, + void* startPtr, size_t dstCapacity) +{ + bitC->bitContainer = 0; + bitC->bitPos = 0; + bitC->startPtr = (char*)startPtr; + bitC->ptr = bitC->startPtr; + bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->bitContainer); + if (dstCapacity <= sizeof(bitC->bitContainer)) return ERROR(dstSize_tooSmall); + return 0; +} + +FORCE_INLINE_TEMPLATE size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) +{ +#if defined(STATIC_BMI2) && STATIC_BMI2 == 1 && !defined(ZSTD_NO_INTRINSICS) + return _bzhi_u64(bitContainer, nbBits); +#else + assert(nbBits < BIT_MASK_SIZE); + return bitContainer & BIT_mask[nbBits]; +#endif +} + +/*! BIT_addBits() : + * can add up to 31 bits into `bitC`. + * Note : does not check for register overflow ! */ +MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, + size_t value, unsigned nbBits) +{ + DEBUG_STATIC_ASSERT(BIT_MASK_SIZE == 32); + assert(nbBits < BIT_MASK_SIZE); + assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8); + bitC->bitContainer |= BIT_getLowerBits(value, nbBits) << bitC->bitPos; + bitC->bitPos += nbBits; +} + +/*! BIT_addBitsFast() : + * works only if `value` is _clean_, + * meaning all high bits above nbBits are 0 */ +MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, + size_t value, unsigned nbBits) +{ + assert((value>>nbBits) == 0); + assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8); + bitC->bitContainer |= value << bitC->bitPos; + bitC->bitPos += nbBits; +} + +/*! BIT_flushBitsFast() : + * assumption : bitContainer has not overflowed + * unsafe version; does not check buffer overflow */ +MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC) +{ + size_t const nbBytes = bitC->bitPos >> 3; + assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8); + assert(bitC->ptr <= bitC->endPtr); + MEM_writeLEST(bitC->ptr, bitC->bitContainer); + bitC->ptr += nbBytes; + bitC->bitPos &= 7; + bitC->bitContainer >>= nbBytes*8; +} + +/*! BIT_flushBits() : + * assumption : bitContainer has not overflowed + * safe version; check for buffer overflow, and prevents it. + * note : does not signal buffer overflow. + * overflow will be revealed later on using BIT_closeCStream() */ +MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC) +{ + size_t const nbBytes = bitC->bitPos >> 3; + assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8); + assert(bitC->ptr <= bitC->endPtr); + MEM_writeLEST(bitC->ptr, bitC->bitContainer); + bitC->ptr += nbBytes; + if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr; + bitC->bitPos &= 7; + bitC->bitContainer >>= nbBytes*8; +} + +/*! BIT_closeCStream() : + * @return : size of CStream, in bytes, + * or 0 if it could not fit into dstBuffer */ +MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC) +{ + BIT_addBitsFast(bitC, 1, 1); /* endMark */ + BIT_flushBits(bitC); + if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */ + return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0); +} + + +/*-******************************************************** +* bitStream decoding +**********************************************************/ +/*! BIT_initDStream() : + * Initialize a BIT_DStream_t. + * `bitD` : a pointer to an already allocated BIT_DStream_t structure. + * `srcSize` must be the *exact* size of the bitStream, in bytes. + * @return : size of stream (== srcSize), or an errorCode if a problem is detected + */ +MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize) +{ + if (srcSize < 1) { ZSTD_memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); } + + bitD->start = (const char*)srcBuffer; + bitD->limitPtr = bitD->start + sizeof(bitD->bitContainer); + + if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */ + bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer); + bitD->bitContainer = MEM_readLEST(bitD->ptr); + { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; + bitD->bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */ + if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ } + } else { + bitD->ptr = bitD->start; + bitD->bitContainer = *(const BYTE*)(bitD->start); + switch(srcSize) + { + case 7: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16); + ZSTD_FALLTHROUGH; + + case 6: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24); + ZSTD_FALLTHROUGH; + + case 5: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32); + ZSTD_FALLTHROUGH; + + case 4: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[3]) << 24; + ZSTD_FALLTHROUGH; + + case 3: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[2]) << 16; + ZSTD_FALLTHROUGH; + + case 2: bitD->bitContainer += (BitContainerType)(((const BYTE*)(srcBuffer))[1]) << 8; + ZSTD_FALLTHROUGH; + + default: break; + } + { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; + bitD->bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0; + if (lastByte == 0) return ERROR(corruption_detected); /* endMark not present */ + } + bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8; + } + + return srcSize; +} + +FORCE_INLINE_TEMPLATE size_t BIT_getUpperBits(BitContainerType bitContainer, U32 const start) +{ + return bitContainer >> start; +} + +FORCE_INLINE_TEMPLATE size_t BIT_getMiddleBits(BitContainerType bitContainer, U32 const start, U32 const nbBits) +{ + U32 const regMask = sizeof(bitContainer)*8 - 1; + /* if start > regMask, bitstream is corrupted, and result is undefined */ + assert(nbBits < BIT_MASK_SIZE); + /* x86 transform & ((1 << nbBits) - 1) to bzhi instruction, it is better + * than accessing memory. When bmi2 instruction is not present, we consider + * such cpus old (pre-Haswell, 2013) and their performance is not of that + * importance. + */ +#if defined(__x86_64__) || defined(_M_X86) + return (bitContainer >> (start & regMask)) & ((((U64)1) << nbBits) - 1); +#else + return (bitContainer >> (start & regMask)) & BIT_mask[nbBits]; +#endif +} + +/*! BIT_lookBits() : + * Provides next n bits from local register. + * local register is not modified. + * On 32-bits, maxNbBits==24. + * On 64-bits, maxNbBits==56. + * @return : value extracted */ +FORCE_INLINE_TEMPLATE size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits) +{ + /* arbitrate between double-shift and shift+mask */ +#if 1 + /* if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8, + * bitstream is likely corrupted, and result is undefined */ + return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits); +#else + /* this code path is slower on my os-x laptop */ + U32 const regMask = sizeof(bitD->bitContainer)*8 - 1; + return ((bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> 1) >> ((regMask-nbBits) & regMask); +#endif +} + +/*! BIT_lookBitsFast() : + * unsafe version; only works if nbBits >= 1 */ +MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits) +{ + U32 const regMask = sizeof(bitD->bitContainer)*8 - 1; + assert(nbBits >= 1); + return (bitD->bitContainer << (bitD->bitsConsumed & regMask)) >> (((regMask+1)-nbBits) & regMask); +} + +FORCE_INLINE_TEMPLATE void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) +{ + bitD->bitsConsumed += nbBits; +} + +/*! BIT_readBits() : + * Read (consume) next n bits from local register and update. + * Pay attention to not read more than nbBits contained into local register. + * @return : extracted value. */ +FORCE_INLINE_TEMPLATE size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits) +{ + size_t const value = BIT_lookBits(bitD, nbBits); + BIT_skipBits(bitD, nbBits); + return value; +} + +/*! BIT_readBitsFast() : + * unsafe version; only works if nbBits >= 1 */ +MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits) +{ + size_t const value = BIT_lookBitsFast(bitD, nbBits); + assert(nbBits >= 1); + BIT_skipBits(bitD, nbBits); + return value; +} + +/*! BIT_reloadDStream_internal() : + * Simple variant of BIT_reloadDStream(), with two conditions: + * 1. bitstream is valid : bitsConsumed <= sizeof(bitD->bitContainer)*8 + * 2. look window is valid after shifted down : bitD->ptr >= bitD->start + */ +MEM_STATIC BIT_DStream_status BIT_reloadDStream_internal(BIT_DStream_t* bitD) +{ + assert(bitD->bitsConsumed <= sizeof(bitD->bitContainer)*8); + bitD->ptr -= bitD->bitsConsumed >> 3; + assert(bitD->ptr >= bitD->start); + bitD->bitsConsumed &= 7; + bitD->bitContainer = MEM_readLEST(bitD->ptr); + return BIT_DStream_unfinished; +} + +/*! BIT_reloadDStreamFast() : + * Similar to BIT_reloadDStream(), but with two differences: + * 1. bitsConsumed <= sizeof(bitD->bitContainer)*8 must hold! + * 2. Returns BIT_DStream_overflow when bitD->ptr < bitD->limitPtr, at this + * point you must use BIT_reloadDStream() to reload. + */ +MEM_STATIC BIT_DStream_status BIT_reloadDStreamFast(BIT_DStream_t* bitD) +{ + if (UNLIKELY(bitD->ptr < bitD->limitPtr)) + return BIT_DStream_overflow; + return BIT_reloadDStream_internal(bitD); +} + +/*! BIT_reloadDStream() : + * Refill `bitD` from buffer previously set in BIT_initDStream() . + * This function is safe, it guarantees it will not never beyond src buffer. + * @return : status of `BIT_DStream_t` internal register. + * when status == BIT_DStream_unfinished, internal register is filled with at least 25 or 57 bits */ +FORCE_INLINE_TEMPLATE BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) +{ + /* note : once in overflow mode, a bitstream remains in this mode until it's reset */ + if (UNLIKELY(bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8))) { + static const BitContainerType zeroFilled = 0; + bitD->ptr = (const char*)&zeroFilled; /* aliasing is allowed for char */ + /* overflow detected, erroneous scenario or end of stream: no update */ + return BIT_DStream_overflow; + } + + assert(bitD->ptr >= bitD->start); + + if (bitD->ptr >= bitD->limitPtr) { + return BIT_reloadDStream_internal(bitD); + } + if (bitD->ptr == bitD->start) { + /* reached end of bitStream => no update */ + if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer; + return BIT_DStream_completed; + } + /* start < ptr < limitPtr => cautious update */ + { U32 nbBytes = bitD->bitsConsumed >> 3; + BIT_DStream_status result = BIT_DStream_unfinished; + if (bitD->ptr - nbBytes < bitD->start) { + nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */ + result = BIT_DStream_endOfBuffer; + } + bitD->ptr -= nbBytes; + bitD->bitsConsumed -= nbBytes*8; + bitD->bitContainer = MEM_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD->bitContainer), otherwise bitD->ptr == bitD->start */ + return result; + } +} + +/*! BIT_endOfDStream() : + * @return : 1 if DStream has _exactly_ reached its end (all bits consumed). + */ +MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream) +{ + return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8)); +} + +#if defined (__cplusplus) +} +#endif + +#endif /* BITSTREAM_H_MODULE */ diff --git a/externals/zstd/lib/common/compiler.h b/externals/zstd/lib/common/compiler.h new file mode 100644 index 0000000..31880ec --- /dev/null +++ b/externals/zstd/lib/common/compiler.h @@ -0,0 +1,450 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMPILER_H +#define ZSTD_COMPILER_H + +#include + +#include "portability_macros.h" + +/*-******************************************************* +* Compiler specifics +*********************************************************/ +/* force inlining */ + +#if !defined(ZSTD_NO_INLINE) +#if (defined(__GNUC__) && !defined(__STRICT_ANSI__)) || defined(__cplusplus) || defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# define INLINE_KEYWORD inline +#else +# define INLINE_KEYWORD +#endif + +#if defined(__GNUC__) || defined(__ICCARM__) +# define FORCE_INLINE_ATTR __attribute__((always_inline)) +#elif defined(_MSC_VER) +# define FORCE_INLINE_ATTR __forceinline +#else +# define FORCE_INLINE_ATTR +#endif + +#else + +#define INLINE_KEYWORD +#define FORCE_INLINE_ATTR + +#endif + +/** + On MSVC qsort requires that functions passed into it use the __cdecl calling conversion(CC). + This explicitly marks such functions as __cdecl so that the code will still compile + if a CC other than __cdecl has been made the default. +*/ +#if defined(_MSC_VER) +# define WIN_CDECL __cdecl +#else +# define WIN_CDECL +#endif + +/* UNUSED_ATTR tells the compiler it is okay if the function is unused. */ +#if defined(__GNUC__) +# define UNUSED_ATTR __attribute__((unused)) +#else +# define UNUSED_ATTR +#endif + +/** + * FORCE_INLINE_TEMPLATE is used to define C "templates", which take constant + * parameters. They must be inlined for the compiler to eliminate the constant + * branches. + */ +#define FORCE_INLINE_TEMPLATE static INLINE_KEYWORD FORCE_INLINE_ATTR UNUSED_ATTR +/** + * HINT_INLINE is used to help the compiler generate better code. It is *not* + * used for "templates", so it can be tweaked based on the compilers + * performance. + * + * gcc-4.8 and gcc-4.9 have been shown to benefit from leaving off the + * always_inline attribute. + * + * clang up to 5.0.0 (trunk) benefit tremendously from the always_inline + * attribute. + */ +#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ >= 4 && __GNUC_MINOR__ >= 8 && __GNUC__ < 5 +# define HINT_INLINE static INLINE_KEYWORD +#else +# define HINT_INLINE FORCE_INLINE_TEMPLATE +#endif + +/* "soft" inline : + * The compiler is free to select if it's a good idea to inline or not. + * The main objective is to silence compiler warnings + * when a defined function in included but not used. + * + * Note : this macro is prefixed `MEM_` because it used to be provided by `mem.h` unit. + * Updating the prefix is probably preferable, but requires a fairly large codemod, + * since this name is used everywhere. + */ +#ifndef MEM_STATIC /* already defined in Linux Kernel mem.h */ +#if defined(__GNUC__) +# define MEM_STATIC static __inline UNUSED_ATTR +#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define MEM_STATIC static inline +#elif defined(_MSC_VER) +# define MEM_STATIC static __inline +#else +# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif +#endif + +/* force no inlining */ +#ifdef _MSC_VER +# define FORCE_NOINLINE static __declspec(noinline) +#else +# if defined(__GNUC__) || defined(__ICCARM__) +# define FORCE_NOINLINE static __attribute__((__noinline__)) +# else +# define FORCE_NOINLINE static +# endif +#endif + + +/* target attribute */ +#if defined(__GNUC__) || defined(__ICCARM__) +# define TARGET_ATTRIBUTE(target) __attribute__((__target__(target))) +#else +# define TARGET_ATTRIBUTE(target) +#endif + +/* Target attribute for BMI2 dynamic dispatch. + * Enable lzcnt, bmi, and bmi2. + * We test for bmi1 & bmi2. lzcnt is included in bmi1. + */ +#define BMI2_TARGET_ATTRIBUTE TARGET_ATTRIBUTE("lzcnt,bmi,bmi2") + +/* prefetch + * can be disabled, by declaring NO_PREFETCH build macro */ +#if defined(NO_PREFETCH) +# define PREFETCH_L1(ptr) do { (void)(ptr); } while (0) /* disabled */ +# define PREFETCH_L2(ptr) do { (void)(ptr); } while (0) /* disabled */ +#else +# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) && !defined(_M_ARM64EC) /* _mm_prefetch() is not defined outside of x86/x64 */ +# include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ +# define PREFETCH_L1(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) +# define PREFETCH_L2(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T1) +# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) +# define PREFETCH_L1(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) +# define PREFETCH_L2(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 2 /* locality */) +# elif defined(__aarch64__) +# define PREFETCH_L1(ptr) do { __asm__ __volatile__("prfm pldl1keep, %0" ::"Q"(*(ptr))); } while (0) +# define PREFETCH_L2(ptr) do { __asm__ __volatile__("prfm pldl2keep, %0" ::"Q"(*(ptr))); } while (0) +# else +# define PREFETCH_L1(ptr) do { (void)(ptr); } while (0) /* disabled */ +# define PREFETCH_L2(ptr) do { (void)(ptr); } while (0) /* disabled */ +# endif +#endif /* NO_PREFETCH */ + +#define CACHELINE_SIZE 64 + +#define PREFETCH_AREA(p, s) \ + do { \ + const char* const _ptr = (const char*)(p); \ + size_t const _size = (size_t)(s); \ + size_t _pos; \ + for (_pos=0; _pos<_size; _pos+=CACHELINE_SIZE) { \ + PREFETCH_L2(_ptr + _pos); \ + } \ + } while (0) + +/* vectorization + * older GCC (pre gcc-4.3 picked as the cutoff) uses a different syntax, + * and some compilers, like Intel ICC and MCST LCC, do not support it at all. */ +#if !defined(__INTEL_COMPILER) && !defined(__clang__) && defined(__GNUC__) && !defined(__LCC__) +# if (__GNUC__ == 4 && __GNUC_MINOR__ > 3) || (__GNUC__ >= 5) +# define DONT_VECTORIZE __attribute__((optimize("no-tree-vectorize"))) +# else +# define DONT_VECTORIZE _Pragma("GCC optimize(\"no-tree-vectorize\")") +# endif +#else +# define DONT_VECTORIZE +#endif + +/* Tell the compiler that a branch is likely or unlikely. + * Only use these macros if it causes the compiler to generate better code. + * If you can remove a LIKELY/UNLIKELY annotation without speed changes in gcc + * and clang, please do. + */ +#if defined(__GNUC__) +#define LIKELY(x) (__builtin_expect((x), 1)) +#define UNLIKELY(x) (__builtin_expect((x), 0)) +#else +#define LIKELY(x) (x) +#define UNLIKELY(x) (x) +#endif + +#if __has_builtin(__builtin_unreachable) || (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 5))) +# define ZSTD_UNREACHABLE do { assert(0), __builtin_unreachable(); } while (0) +#else +# define ZSTD_UNREACHABLE do { assert(0); } while (0) +#endif + +/* disable warnings */ +#ifdef _MSC_VER /* Visual Studio */ +# include /* For Visual 2005 */ +# pragma warning(disable : 4100) /* disable: C4100: unreferenced formal parameter */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ +# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ +# pragma warning(disable : 4324) /* disable: C4324: padded structure */ +#endif + +/*Like DYNAMIC_BMI2 but for compile time determination of BMI2 support*/ +#ifndef STATIC_BMI2 +# if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_I86)) +# ifdef __AVX2__ //MSVC does not have a BMI2 specific flag, but every CPU that supports AVX2 also supports BMI2 +# define STATIC_BMI2 1 +# endif +# elif defined(__BMI2__) && defined(__x86_64__) && defined(__GNUC__) +# define STATIC_BMI2 1 +# endif +#endif + +#ifndef STATIC_BMI2 + #define STATIC_BMI2 0 +#endif + +/* compile time determination of SIMD support */ +#if !defined(ZSTD_NO_INTRINSICS) +# if defined(__SSE2__) || defined(_M_AMD64) || (defined (_M_IX86) && defined(_M_IX86_FP) && (_M_IX86_FP >= 2)) +# define ZSTD_ARCH_X86_SSE2 +# endif +# if defined(__ARM_NEON) || defined(_M_ARM64) +# define ZSTD_ARCH_ARM_NEON +# endif +# +# if defined(ZSTD_ARCH_X86_SSE2) +# include +# elif defined(ZSTD_ARCH_ARM_NEON) +# include +# endif +#endif + +/* C-language Attributes are added in C23. */ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ > 201710L) && defined(__has_c_attribute) +# define ZSTD_HAS_C_ATTRIBUTE(x) __has_c_attribute(x) +#else +# define ZSTD_HAS_C_ATTRIBUTE(x) 0 +#endif + +/* Only use C++ attributes in C++. Some compilers report support for C++ + * attributes when compiling with C. + */ +#if defined(__cplusplus) && defined(__has_cpp_attribute) +# define ZSTD_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) +#else +# define ZSTD_HAS_CPP_ATTRIBUTE(x) 0 +#endif + +/* Define ZSTD_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute. + * - C23: https://en.cppreference.com/w/c/language/attributes/fallthrough + * - CPP17: https://en.cppreference.com/w/cpp/language/attributes/fallthrough + * - Else: __attribute__((__fallthrough__)) + */ +#ifndef ZSTD_FALLTHROUGH +# if ZSTD_HAS_C_ATTRIBUTE(fallthrough) +# define ZSTD_FALLTHROUGH [[fallthrough]] +# elif ZSTD_HAS_CPP_ATTRIBUTE(fallthrough) +# define ZSTD_FALLTHROUGH [[fallthrough]] +# elif __has_attribute(__fallthrough__) +/* Leading semicolon is to satisfy gcc-11 with -pedantic. Without the semicolon + * gcc complains about: a label can only be part of a statement and a declaration is not a statement. + */ +# define ZSTD_FALLTHROUGH ; __attribute__((__fallthrough__)) +# else +# define ZSTD_FALLTHROUGH +# endif +#endif + +/*-************************************************************** +* Alignment check +*****************************************************************/ + +/* this test was initially positioned in mem.h, + * but this file is removed (or replaced) for linux kernel + * so it's now hosted in compiler.h, + * which remains valid for both user & kernel spaces. + */ + +#ifndef ZSTD_ALIGNOF +# if defined(__GNUC__) || defined(_MSC_VER) +/* covers gcc, clang & MSVC */ +/* note : this section must come first, before C11, + * due to a limitation in the kernel source generator */ +# define ZSTD_ALIGNOF(T) __alignof(T) + +# elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) +/* C11 support */ +# include +# define ZSTD_ALIGNOF(T) alignof(T) + +# else +/* No known support for alignof() - imperfect backup */ +# define ZSTD_ALIGNOF(T) (sizeof(void*) < sizeof(T) ? sizeof(void*) : sizeof(T)) + +# endif +#endif /* ZSTD_ALIGNOF */ + +/*-************************************************************** +* Sanitizer +*****************************************************************/ + +/** + * Zstd relies on pointer overflow in its decompressor. + * We add this attribute to functions that rely on pointer overflow. + */ +#ifndef ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +# if __has_attribute(no_sanitize) +# if !defined(__clang__) && defined(__GNUC__) && __GNUC__ < 8 + /* gcc < 8 only has signed-integer-overlow which triggers on pointer overflow */ +# define ZSTD_ALLOW_POINTER_OVERFLOW_ATTR __attribute__((no_sanitize("signed-integer-overflow"))) +# else + /* older versions of clang [3.7, 5.0) will warn that pointer-overflow is ignored. */ +# define ZSTD_ALLOW_POINTER_OVERFLOW_ATTR __attribute__((no_sanitize("pointer-overflow"))) +# endif +# else +# define ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +# endif +#endif + +/** + * Helper function to perform a wrapped pointer difference without trigging + * UBSAN. + * + * @returns lhs - rhs with wrapping + */ +MEM_STATIC +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +ptrdiff_t ZSTD_wrappedPtrDiff(unsigned char const* lhs, unsigned char const* rhs) +{ + return lhs - rhs; +} + +/** + * Helper function to perform a wrapped pointer add without triggering UBSAN. + * + * @return ptr + add with wrapping + */ +MEM_STATIC +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +unsigned char const* ZSTD_wrappedPtrAdd(unsigned char const* ptr, ptrdiff_t add) +{ + return ptr + add; +} + +/** + * Helper function to perform a wrapped pointer subtraction without triggering + * UBSAN. + * + * @return ptr - sub with wrapping + */ +MEM_STATIC +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +unsigned char const* ZSTD_wrappedPtrSub(unsigned char const* ptr, ptrdiff_t sub) +{ + return ptr - sub; +} + +/** + * Helper function to add to a pointer that works around C's undefined behavior + * of adding 0 to NULL. + * + * @returns `ptr + add` except it defines `NULL + 0 == NULL`. + */ +MEM_STATIC +unsigned char* ZSTD_maybeNullPtrAdd(unsigned char* ptr, ptrdiff_t add) +{ + return add > 0 ? ptr + add : ptr; +} + +/* Issue #3240 reports an ASAN failure on an llvm-mingw build. Out of an + * abundance of caution, disable our custom poisoning on mingw. */ +#ifdef __MINGW32__ +#ifndef ZSTD_ASAN_DONT_POISON_WORKSPACE +#define ZSTD_ASAN_DONT_POISON_WORKSPACE 1 +#endif +#ifndef ZSTD_MSAN_DONT_POISON_WORKSPACE +#define ZSTD_MSAN_DONT_POISON_WORKSPACE 1 +#endif +#endif + +#if ZSTD_MEMORY_SANITIZER && !defined(ZSTD_MSAN_DONT_POISON_WORKSPACE) +/* Not all platforms that support msan provide sanitizers/msan_interface.h. + * We therefore declare the functions we need ourselves, rather than trying to + * include the header file... */ +#include /* size_t */ +#define ZSTD_DEPS_NEED_STDINT +#include "zstd_deps.h" /* intptr_t */ + +/* Make memory region fully initialized (without changing its contents). */ +void __msan_unpoison(const volatile void *a, size_t size); + +/* Make memory region fully uninitialized (without changing its contents). + This is a legacy interface that does not update origin information. Use + __msan_allocated_memory() instead. */ +void __msan_poison(const volatile void *a, size_t size); + +/* Returns the offset of the first (at least partially) poisoned byte in the + memory range, or -1 if the whole range is good. */ +intptr_t __msan_test_shadow(const volatile void *x, size_t size); + +/* Print shadow and origin for the memory range to stderr in a human-readable + format. */ +void __msan_print_shadow(const volatile void *x, size_t size); +#endif + +#if ZSTD_ADDRESS_SANITIZER && !defined(ZSTD_ASAN_DONT_POISON_WORKSPACE) +/* Not all platforms that support asan provide sanitizers/asan_interface.h. + * We therefore declare the functions we need ourselves, rather than trying to + * include the header file... */ +#include /* size_t */ + +/** + * Marks a memory region ([addr, addr+size)) as unaddressable. + * + * This memory must be previously allocated by your program. Instrumented + * code is forbidden from accessing addresses in this region until it is + * unpoisoned. This function is not guaranteed to poison the entire region - + * it could poison only a subregion of [addr, addr+size) due to ASan + * alignment restrictions. + * + * \note This function is not thread-safe because no two threads can poison or + * unpoison memory in the same memory region simultaneously. + * + * \param addr Start of memory region. + * \param size Size of memory region. */ +void __asan_poison_memory_region(void const volatile *addr, size_t size); + +/** + * Marks a memory region ([addr, addr+size)) as addressable. + * + * This memory must be previously allocated by your program. Accessing + * addresses in this region is allowed until this region is poisoned again. + * This function could unpoison a super-region of [addr, addr+size) due + * to ASan alignment restrictions. + * + * \note This function is not thread-safe because no two threads can + * poison or unpoison memory in the same memory region simultaneously. + * + * \param addr Start of memory region. + * \param size Size of memory region. */ +void __asan_unpoison_memory_region(void const volatile *addr, size_t size); +#endif + +#endif /* ZSTD_COMPILER_H */ diff --git a/externals/zstd/lib/common/cpu.h b/externals/zstd/lib/common/cpu.h new file mode 100644 index 0000000..0e684d9 --- /dev/null +++ b/externals/zstd/lib/common/cpu.h @@ -0,0 +1,249 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_COMMON_CPU_H +#define ZSTD_COMMON_CPU_H + +/** + * Implementation taken from folly/CpuId.h + * https://github.com/facebook/folly/blob/master/folly/CpuId.h + */ + +#include "mem.h" + +#ifdef _MSC_VER +#include +#endif + +typedef struct { + U32 f1c; + U32 f1d; + U32 f7b; + U32 f7c; +} ZSTD_cpuid_t; + +MEM_STATIC ZSTD_cpuid_t ZSTD_cpuid(void) { + U32 f1c = 0; + U32 f1d = 0; + U32 f7b = 0; + U32 f7c = 0; +#if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) +#if !defined(__clang__) + int reg[4]; + __cpuid((int*)reg, 0); + { + int const n = reg[0]; + if (n >= 1) { + __cpuid((int*)reg, 1); + f1c = (U32)reg[2]; + f1d = (U32)reg[3]; + } + if (n >= 7) { + __cpuidex((int*)reg, 7, 0); + f7b = (U32)reg[1]; + f7c = (U32)reg[2]; + } + } +#else + /* Clang compiler has a bug (fixed in https://reviews.llvm.org/D101338) in + * which the `__cpuid` intrinsic does not save and restore `rbx` as it needs + * to due to being a reserved register. So in that case, do the `cpuid` + * ourselves. Clang supports inline assembly anyway. + */ + U32 n; + __asm__( + "pushq %%rbx\n\t" + "cpuid\n\t" + "popq %%rbx\n\t" + : "=a"(n) + : "a"(0) + : "rcx", "rdx"); + if (n >= 1) { + U32 f1a; + __asm__( + "pushq %%rbx\n\t" + "cpuid\n\t" + "popq %%rbx\n\t" + : "=a"(f1a), "=c"(f1c), "=d"(f1d) + : "a"(1) + :); + } + if (n >= 7) { + __asm__( + "pushq %%rbx\n\t" + "cpuid\n\t" + "movq %%rbx, %%rax\n\t" + "popq %%rbx" + : "=a"(f7b), "=c"(f7c) + : "a"(7), "c"(0) + : "rdx"); + } +#endif +#elif defined(__i386__) && defined(__PIC__) && !defined(__clang__) && defined(__GNUC__) + /* The following block like the normal cpuid branch below, but gcc + * reserves ebx for use of its pic register so we must specially + * handle the save and restore to avoid clobbering the register + */ + U32 n; + __asm__( + "pushl %%ebx\n\t" + "cpuid\n\t" + "popl %%ebx\n\t" + : "=a"(n) + : "a"(0) + : "ecx", "edx"); + if (n >= 1) { + U32 f1a; + __asm__( + "pushl %%ebx\n\t" + "cpuid\n\t" + "popl %%ebx\n\t" + : "=a"(f1a), "=c"(f1c), "=d"(f1d) + : "a"(1)); + } + if (n >= 7) { + __asm__( + "pushl %%ebx\n\t" + "cpuid\n\t" + "movl %%ebx, %%eax\n\t" + "popl %%ebx" + : "=a"(f7b), "=c"(f7c) + : "a"(7), "c"(0) + : "edx"); + } +#elif defined(__x86_64__) || defined(_M_X64) || defined(__i386__) + U32 n; + __asm__("cpuid" : "=a"(n) : "a"(0) : "ebx", "ecx", "edx"); + if (n >= 1) { + U32 f1a; + __asm__("cpuid" : "=a"(f1a), "=c"(f1c), "=d"(f1d) : "a"(1) : "ebx"); + } + if (n >= 7) { + U32 f7a; + __asm__("cpuid" + : "=a"(f7a), "=b"(f7b), "=c"(f7c) + : "a"(7), "c"(0) + : "edx"); + } +#endif + { + ZSTD_cpuid_t cpuid; + cpuid.f1c = f1c; + cpuid.f1d = f1d; + cpuid.f7b = f7b; + cpuid.f7c = f7c; + return cpuid; + } +} + +#define X(name, r, bit) \ + MEM_STATIC int ZSTD_cpuid_##name(ZSTD_cpuid_t const cpuid) { \ + return ((cpuid.r) & (1U << bit)) != 0; \ + } + +/* cpuid(1): Processor Info and Feature Bits. */ +#define C(name, bit) X(name, f1c, bit) + C(sse3, 0) + C(pclmuldq, 1) + C(dtes64, 2) + C(monitor, 3) + C(dscpl, 4) + C(vmx, 5) + C(smx, 6) + C(eist, 7) + C(tm2, 8) + C(ssse3, 9) + C(cnxtid, 10) + C(fma, 12) + C(cx16, 13) + C(xtpr, 14) + C(pdcm, 15) + C(pcid, 17) + C(dca, 18) + C(sse41, 19) + C(sse42, 20) + C(x2apic, 21) + C(movbe, 22) + C(popcnt, 23) + C(tscdeadline, 24) + C(aes, 25) + C(xsave, 26) + C(osxsave, 27) + C(avx, 28) + C(f16c, 29) + C(rdrand, 30) +#undef C +#define D(name, bit) X(name, f1d, bit) + D(fpu, 0) + D(vme, 1) + D(de, 2) + D(pse, 3) + D(tsc, 4) + D(msr, 5) + D(pae, 6) + D(mce, 7) + D(cx8, 8) + D(apic, 9) + D(sep, 11) + D(mtrr, 12) + D(pge, 13) + D(mca, 14) + D(cmov, 15) + D(pat, 16) + D(pse36, 17) + D(psn, 18) + D(clfsh, 19) + D(ds, 21) + D(acpi, 22) + D(mmx, 23) + D(fxsr, 24) + D(sse, 25) + D(sse2, 26) + D(ss, 27) + D(htt, 28) + D(tm, 29) + D(pbe, 31) +#undef D + +/* cpuid(7): Extended Features. */ +#define B(name, bit) X(name, f7b, bit) + B(bmi1, 3) + B(hle, 4) + B(avx2, 5) + B(smep, 7) + B(bmi2, 8) + B(erms, 9) + B(invpcid, 10) + B(rtm, 11) + B(mpx, 14) + B(avx512f, 16) + B(avx512dq, 17) + B(rdseed, 18) + B(adx, 19) + B(smap, 20) + B(avx512ifma, 21) + B(pcommit, 22) + B(clflushopt, 23) + B(clwb, 24) + B(avx512pf, 26) + B(avx512er, 27) + B(avx512cd, 28) + B(sha, 29) + B(avx512bw, 30) + B(avx512vl, 31) +#undef B +#define C(name, bit) X(name, f7c, bit) + C(prefetchwt1, 0) + C(avx512vbmi, 1) +#undef C + +#undef X + +#endif /* ZSTD_COMMON_CPU_H */ diff --git a/externals/zstd/lib/common/debug.c b/externals/zstd/lib/common/debug.c new file mode 100644 index 0000000..9d0b7d2 --- /dev/null +++ b/externals/zstd/lib/common/debug.c @@ -0,0 +1,30 @@ +/* ****************************************************************** + * debug + * Part of FSE library + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + + +/* + * This module only hosts one global variable + * which can be used to dynamically influence the verbosity of traces, + * such as DEBUGLOG and RAWLOG + */ + +#include "debug.h" + +#if !defined(ZSTD_LINUX_KERNEL) || (DEBUGLEVEL>=2) +/* We only use this when DEBUGLEVEL>=2, but we get -Werror=pedantic errors if a + * translation unit is empty. So remove this from Linux kernel builds, but + * otherwise just leave it in. + */ +int g_debuglevel = DEBUGLEVEL; +#endif diff --git a/externals/zstd/lib/common/debug.h b/externals/zstd/lib/common/debug.h new file mode 100644 index 0000000..a16b69e --- /dev/null +++ b/externals/zstd/lib/common/debug.h @@ -0,0 +1,116 @@ +/* ****************************************************************** + * debug + * Part of FSE library + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + + +/* + * The purpose of this header is to enable debug functions. + * They regroup assert(), DEBUGLOG() and RAWLOG() for run-time, + * and DEBUG_STATIC_ASSERT() for compile-time. + * + * By default, DEBUGLEVEL==0, which means run-time debug is disabled. + * + * Level 1 enables assert() only. + * Starting level 2, traces can be generated and pushed to stderr. + * The higher the level, the more verbose the traces. + * + * It's possible to dynamically adjust level using variable g_debug_level, + * which is only declared if DEBUGLEVEL>=2, + * and is a global variable, not multi-thread protected (use with care) + */ + +#ifndef DEBUG_H_12987983217 +#define DEBUG_H_12987983217 + +#if defined (__cplusplus) +extern "C" { +#endif + + +/* static assert is triggered at compile time, leaving no runtime artefact. + * static assert only works with compile-time constants. + * Also, this variant can only be used inside a function. */ +#define DEBUG_STATIC_ASSERT(c) (void)sizeof(char[(c) ? 1 : -1]) + + +/* DEBUGLEVEL is expected to be defined externally, + * typically through compiler command line. + * Value must be a number. */ +#ifndef DEBUGLEVEL +# define DEBUGLEVEL 0 +#endif + + +/* recommended values for DEBUGLEVEL : + * 0 : release mode, no debug, all run-time checks disabled + * 1 : enables assert() only, no display + * 2 : reserved, for currently active debug path + * 3 : events once per object lifetime (CCtx, CDict, etc.) + * 4 : events once per frame + * 5 : events once per block + * 6 : events once per sequence (verbose) + * 7+: events at every position (*very* verbose) + * + * It's generally inconvenient to output traces > 5. + * In which case, it's possible to selectively trigger high verbosity levels + * by modifying g_debug_level. + */ + +#if (DEBUGLEVEL>=1) +# define ZSTD_DEPS_NEED_ASSERT +# include "zstd_deps.h" +#else +# ifndef assert /* assert may be already defined, due to prior #include */ +# define assert(condition) ((void)0) /* disable assert (default) */ +# endif +#endif + +#if (DEBUGLEVEL>=2) +# define ZSTD_DEPS_NEED_IO +# include "zstd_deps.h" +extern int g_debuglevel; /* the variable is only declared, + it actually lives in debug.c, + and is shared by the whole process. + It's not thread-safe. + It's useful when enabling very verbose levels + on selective conditions (such as position in src) */ + +# define RAWLOG(l, ...) \ + do { \ + if (l<=g_debuglevel) { \ + ZSTD_DEBUG_PRINT(__VA_ARGS__); \ + } \ + } while (0) + +#define STRINGIFY(x) #x +#define TOSTRING(x) STRINGIFY(x) +#define LINE_AS_STRING TOSTRING(__LINE__) + +# define DEBUGLOG(l, ...) \ + do { \ + if (l<=g_debuglevel) { \ + ZSTD_DEBUG_PRINT(__FILE__ ":" LINE_AS_STRING ": " __VA_ARGS__); \ + ZSTD_DEBUG_PRINT(" \n"); \ + } \ + } while (0) +#else +# define RAWLOG(l, ...) do { } while (0) /* disabled */ +# define DEBUGLOG(l, ...) do { } while (0) /* disabled */ +#endif + + +#if defined (__cplusplus) +} +#endif + +#endif /* DEBUG_H_12987983217 */ diff --git a/externals/zstd/lib/common/entropy_common.c b/externals/zstd/lib/common/entropy_common.c new file mode 100644 index 0000000..e2173af --- /dev/null +++ b/externals/zstd/lib/common/entropy_common.c @@ -0,0 +1,340 @@ +/* ****************************************************************** + * Common functions of New Generation Entropy library + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +/* ************************************* +* Dependencies +***************************************/ +#include "mem.h" +#include "error_private.h" /* ERR_*, ERROR */ +#define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */ +#include "fse.h" +#include "huf.h" +#include "bits.h" /* ZSDT_highbit32, ZSTD_countTrailingZeros32 */ + + +/*=== Version ===*/ +unsigned FSE_versionNumber(void) { return FSE_VERSION_NUMBER; } + + +/*=== Error Management ===*/ +unsigned FSE_isError(size_t code) { return ERR_isError(code); } +const char* FSE_getErrorName(size_t code) { return ERR_getErrorName(code); } + +unsigned HUF_isError(size_t code) { return ERR_isError(code); } +const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); } + + +/*-************************************************************** +* FSE NCount encoding-decoding +****************************************************************/ +FORCE_INLINE_TEMPLATE +size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + const BYTE* const istart = (const BYTE*) headerBuffer; + const BYTE* const iend = istart + hbSize; + const BYTE* ip = istart; + int nbBits; + int remaining; + int threshold; + U32 bitStream; + int bitCount; + unsigned charnum = 0; + unsigned const maxSV1 = *maxSVPtr + 1; + int previous0 = 0; + + if (hbSize < 8) { + /* This function only works when hbSize >= 8 */ + char buffer[8] = {0}; + ZSTD_memcpy(buffer, headerBuffer, hbSize); + { size_t const countSize = FSE_readNCount(normalizedCounter, maxSVPtr, tableLogPtr, + buffer, sizeof(buffer)); + if (FSE_isError(countSize)) return countSize; + if (countSize > hbSize) return ERROR(corruption_detected); + return countSize; + } } + assert(hbSize >= 8); + + /* init */ + ZSTD_memset(normalizedCounter, 0, (*maxSVPtr+1) * sizeof(normalizedCounter[0])); /* all symbols not present in NCount have a frequency of 0 */ + bitStream = MEM_readLE32(ip); + nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */ + if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge); + bitStream >>= 4; + bitCount = 4; + *tableLogPtr = nbBits; + remaining = (1<> 1; + while (repeats >= 12) { + charnum += 3 * 12; + if (LIKELY(ip <= iend-7)) { + ip += 3; + } else { + bitCount -= (int)(8 * (iend - 7 - ip)); + bitCount &= 31; + ip = iend - 4; + } + bitStream = MEM_readLE32(ip) >> bitCount; + repeats = ZSTD_countTrailingZeros32(~bitStream | 0x80000000) >> 1; + } + charnum += 3 * repeats; + bitStream >>= 2 * repeats; + bitCount += 2 * repeats; + + /* Add the final repeat which isn't 0b11. */ + assert((bitStream & 3) < 3); + charnum += bitStream & 3; + bitCount += 2; + + /* This is an error, but break and return an error + * at the end, because returning out of a loop makes + * it harder for the compiler to optimize. + */ + if (charnum >= maxSV1) break; + + /* We don't need to set the normalized count to 0 + * because we already memset the whole buffer to 0. + */ + + if (LIKELY(ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { + assert((bitCount >> 3) <= 3); /* For first condition to work */ + ip += bitCount>>3; + bitCount &= 7; + } else { + bitCount -= (int)(8 * (iend - 4 - ip)); + bitCount &= 31; + ip = iend - 4; + } + bitStream = MEM_readLE32(ip) >> bitCount; + } + { + int const max = (2*threshold-1) - remaining; + int count; + + if ((bitStream & (threshold-1)) < (U32)max) { + count = bitStream & (threshold-1); + bitCount += nbBits-1; + } else { + count = bitStream & (2*threshold-1); + if (count >= threshold) count -= max; + bitCount += nbBits; + } + + count--; /* extra accuracy */ + /* When it matters (small blocks), this is a + * predictable branch, because we don't use -1. + */ + if (count >= 0) { + remaining -= count; + } else { + assert(count == -1); + remaining += count; + } + normalizedCounter[charnum++] = (short)count; + previous0 = !count; + + assert(threshold > 1); + if (remaining < threshold) { + /* This branch can be folded into the + * threshold update condition because we + * know that threshold > 1. + */ + if (remaining <= 1) break; + nbBits = ZSTD_highbit32(remaining) + 1; + threshold = 1 << (nbBits - 1); + } + if (charnum >= maxSV1) break; + + if (LIKELY(ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { + ip += bitCount>>3; + bitCount &= 7; + } else { + bitCount -= (int)(8 * (iend - 4 - ip)); + bitCount &= 31; + ip = iend - 4; + } + bitStream = MEM_readLE32(ip) >> bitCount; + } } + if (remaining != 1) return ERROR(corruption_detected); + /* Only possible when there are too many zeros. */ + if (charnum > maxSV1) return ERROR(maxSymbolValue_tooSmall); + if (bitCount > 32) return ERROR(corruption_detected); + *maxSVPtr = charnum-1; + + ip += (bitCount+7)>>3; + return ip-istart; +} + +/* Avoids the FORCE_INLINE of the _body() function. */ +static size_t FSE_readNCount_body_default( + short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + return FSE_readNCount_body(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize); +} + +#if DYNAMIC_BMI2 +BMI2_TARGET_ATTRIBUTE static size_t FSE_readNCount_body_bmi2( + short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + return FSE_readNCount_body(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize); +} +#endif + +size_t FSE_readNCount_bmi2( + short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize, int bmi2) +{ +#if DYNAMIC_BMI2 + if (bmi2) { + return FSE_readNCount_body_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize); + } +#endif + (void)bmi2; + return FSE_readNCount_body_default(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize); +} + +size_t FSE_readNCount( + short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + return FSE_readNCount_bmi2(normalizedCounter, maxSVPtr, tableLogPtr, headerBuffer, hbSize, /* bmi2 */ 0); +} + + +/*! HUF_readStats() : + Read compact Huffman tree, saved by HUF_writeCTable(). + `huffWeight` is destination buffer. + `rankStats` is assumed to be a table of at least HUF_TABLELOG_MAX U32. + @return : size read from `src` , or an error Code . + Note : Needed by HUF_readCTable() and HUF_readDTableX?() . +*/ +size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize) +{ + U32 wksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; + return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* flags */ 0); +} + +FORCE_INLINE_TEMPLATE size_t +HUF_readStats_body(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize, + int bmi2) +{ + U32 weightTotal; + const BYTE* ip = (const BYTE*) src; + size_t iSize; + size_t oSize; + + if (!srcSize) return ERROR(srcSize_wrong); + iSize = ip[0]; + /* ZSTD_memset(huffWeight, 0, hwSize); *//* is not necessary, even though some analyzer complain ... */ + + if (iSize >= 128) { /* special header */ + oSize = iSize - 127; + iSize = ((oSize+1)/2); + if (iSize+1 > srcSize) return ERROR(srcSize_wrong); + if (oSize >= hwSize) return ERROR(corruption_detected); + ip += 1; + { U32 n; + for (n=0; n> 4; + huffWeight[n+1] = ip[n/2] & 15; + } } } + else { /* header compressed with FSE (normal case) */ + if (iSize+1 > srcSize) return ERROR(srcSize_wrong); + /* max (hwSize-1) values decoded, as last one is implied */ + oSize = FSE_decompress_wksp_bmi2(huffWeight, hwSize-1, ip+1, iSize, 6, workSpace, wkspSize, bmi2); + if (FSE_isError(oSize)) return oSize; + } + + /* collect weight stats */ + ZSTD_memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32)); + weightTotal = 0; + { U32 n; for (n=0; n HUF_TABLELOG_MAX) return ERROR(corruption_detected); + rankStats[huffWeight[n]]++; + weightTotal += (1 << huffWeight[n]) >> 1; + } } + if (weightTotal == 0) return ERROR(corruption_detected); + + /* get last non-null symbol weight (implied, total must be 2^n) */ + { U32 const tableLog = ZSTD_highbit32(weightTotal) + 1; + if (tableLog > HUF_TABLELOG_MAX) return ERROR(corruption_detected); + *tableLogPtr = tableLog; + /* determine last weight */ + { U32 const total = 1 << tableLog; + U32 const rest = total - weightTotal; + U32 const verif = 1 << ZSTD_highbit32(rest); + U32 const lastWeight = ZSTD_highbit32(rest) + 1; + if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */ + huffWeight[oSize] = (BYTE)lastWeight; + rankStats[lastWeight]++; + } } + + /* check tree construction validity */ + if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */ + + /* results */ + *nbSymbolsPtr = (U32)(oSize+1); + return iSize+1; +} + +/* Avoids the FORCE_INLINE of the _body() function. */ +static size_t HUF_readStats_body_default(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize) +{ + return HUF_readStats_body(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize, 0); +} + +#if DYNAMIC_BMI2 +static BMI2_TARGET_ATTRIBUTE size_t HUF_readStats_body_bmi2(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize) +{ + return HUF_readStats_body(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize, 1); +} +#endif + +size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize, + int flags) +{ +#if DYNAMIC_BMI2 + if (flags & HUF_flags_bmi2) { + return HUF_readStats_body_bmi2(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize); + } +#endif + (void)flags; + return HUF_readStats_body_default(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize); +} diff --git a/externals/zstd/lib/common/error_private.c b/externals/zstd/lib/common/error_private.c new file mode 100644 index 0000000..075fc5e --- /dev/null +++ b/externals/zstd/lib/common/error_private.c @@ -0,0 +1,63 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* The purpose of this file is to have a single list of error strings embedded in binary */ + +#include "error_private.h" + +const char* ERR_getErrorString(ERR_enum code) +{ +#ifdef ZSTD_STRIP_ERROR_STRINGS + (void)code; + return "Error strings stripped"; +#else + static const char* const notErrorCode = "Unspecified error code"; + switch( code ) + { + case PREFIX(no_error): return "No error detected"; + case PREFIX(GENERIC): return "Error (generic)"; + case PREFIX(prefix_unknown): return "Unknown frame descriptor"; + case PREFIX(version_unsupported): return "Version not supported"; + case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; + case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding"; + case PREFIX(corruption_detected): return "Data corruption detected"; + case PREFIX(checksum_wrong): return "Restored data doesn't match checksum"; + case PREFIX(literals_headerWrong): return "Header of Literals' block doesn't respect format specification"; + case PREFIX(parameter_unsupported): return "Unsupported parameter"; + case PREFIX(parameter_combination_unsupported): return "Unsupported combination of parameters"; + case PREFIX(parameter_outOfBound): return "Parameter is out of bound"; + case PREFIX(init_missing): return "Context should be init first"; + case PREFIX(memory_allocation): return "Allocation error : not enough memory"; + case PREFIX(workSpace_tooSmall): return "workSpace buffer is not large enough"; + case PREFIX(stage_wrong): return "Operation not authorized at current processing stage"; + case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; + case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; + case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; + case PREFIX(stabilityCondition_notRespected): return "pledged buffer stability condition is not respected"; + case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; + case PREFIX(dictionary_wrong): return "Dictionary mismatch"; + case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples"; + case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; + case PREFIX(srcSize_wrong): return "Src size is incorrect"; + case PREFIX(dstBuffer_null): return "Operation on NULL destination buffer"; + case PREFIX(noForwardProgress_destFull): return "Operation made no progress over multiple calls, due to output buffer being full"; + case PREFIX(noForwardProgress_inputEmpty): return "Operation made no progress over multiple calls, due to input being empty"; + /* following error codes are not stable and may be removed or changed in a future version */ + case PREFIX(frameIndex_tooLarge): return "Frame index is too large"; + case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking"; + case PREFIX(dstBuffer_wrong): return "Destination buffer is wrong"; + case PREFIX(srcBuffer_wrong): return "Source buffer is wrong"; + case PREFIX(sequenceProducer_failed): return "Block-level external sequence producer returned an error code"; + case PREFIX(externalSequences_invalid): return "External sequences are not valid"; + case PREFIX(maxCode): + default: return notErrorCode; + } +#endif +} diff --git a/externals/zstd/lib/common/error_private.h b/externals/zstd/lib/common/error_private.h new file mode 100644 index 0000000..0156010 --- /dev/null +++ b/externals/zstd/lib/common/error_private.h @@ -0,0 +1,168 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* Note : this module is expected to remain private, do not expose it */ + +#ifndef ERROR_H_MODULE +#define ERROR_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + + +/* **************************************** +* Dependencies +******************************************/ +#include "../zstd_errors.h" /* enum list */ +#include "compiler.h" +#include "debug.h" +#include "zstd_deps.h" /* size_t */ + + +/* **************************************** +* Compiler-specific +******************************************/ +#if defined(__GNUC__) +# define ERR_STATIC static __attribute__((unused)) +#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define ERR_STATIC static inline +#elif defined(_MSC_VER) +# define ERR_STATIC static __inline +#else +# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif + + +/*-**************************************** +* Customization (error_public.h) +******************************************/ +typedef ZSTD_ErrorCode ERR_enum; +#define PREFIX(name) ZSTD_error_##name + + +/*-**************************************** +* Error codes handling +******************************************/ +#undef ERROR /* already defined on Visual Studio */ +#define ERROR(name) ZSTD_ERROR(name) +#define ZSTD_ERROR(name) ((size_t)-PREFIX(name)) + +ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); } + +ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); } + +/* check and forward error code */ +#define CHECK_V_F(e, f) \ + size_t const e = f; \ + do { \ + if (ERR_isError(e)) \ + return e; \ + } while (0) +#define CHECK_F(f) do { CHECK_V_F(_var_err__, f); } while (0) + + +/*-**************************************** +* Error Strings +******************************************/ + +const char* ERR_getErrorString(ERR_enum code); /* error_private.c */ + +ERR_STATIC const char* ERR_getErrorName(size_t code) +{ + return ERR_getErrorString(ERR_getErrorCode(code)); +} + +/** + * Ignore: this is an internal helper. + * + * This is a helper function to help force C99-correctness during compilation. + * Under strict compilation modes, variadic macro arguments can't be empty. + * However, variadic function arguments can be. Using a function therefore lets + * us statically check that at least one (string) argument was passed, + * independent of the compilation flags. + */ +static INLINE_KEYWORD UNUSED_ATTR +void _force_has_format_string(const char *format, ...) { + (void)format; +} + +/** + * Ignore: this is an internal helper. + * + * We want to force this function invocation to be syntactically correct, but + * we don't want to force runtime evaluation of its arguments. + */ +#define _FORCE_HAS_FORMAT_STRING(...) \ + do { \ + if (0) { \ + _force_has_format_string(__VA_ARGS__); \ + } \ + } while (0) + +#define ERR_QUOTE(str) #str + +/** + * Return the specified error if the condition evaluates to true. + * + * In debug modes, prints additional information. + * In order to do that (particularly, printing the conditional that failed), + * this can't just wrap RETURN_ERROR(). + */ +#define RETURN_ERROR_IF(cond, err, ...) \ + do { \ + if (cond) { \ + RAWLOG(3, "%s:%d: ERROR!: check %s failed, returning %s", \ + __FILE__, __LINE__, ERR_QUOTE(cond), ERR_QUOTE(ERROR(err))); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return ERROR(err); \ + } \ + } while (0) + +/** + * Unconditionally return the specified error. + * + * In debug modes, prints additional information. + */ +#define RETURN_ERROR(err, ...) \ + do { \ + RAWLOG(3, "%s:%d: ERROR!: unconditional check failed, returning %s", \ + __FILE__, __LINE__, ERR_QUOTE(ERROR(err))); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return ERROR(err); \ + } while(0) + +/** + * If the provided expression evaluates to an error code, returns that error code. + * + * In debug modes, prints additional information. + */ +#define FORWARD_IF_ERROR(err, ...) \ + do { \ + size_t const err_code = (err); \ + if (ERR_isError(err_code)) { \ + RAWLOG(3, "%s:%d: ERROR!: forwarding error in %s: %s", \ + __FILE__, __LINE__, ERR_QUOTE(err), ERR_getErrorName(err_code)); \ + _FORCE_HAS_FORMAT_STRING(__VA_ARGS__); \ + RAWLOG(3, ": " __VA_ARGS__); \ + RAWLOG(3, "\n"); \ + return err_code; \ + } \ + } while(0) + +#if defined (__cplusplus) +} +#endif + +#endif /* ERROR_H_MODULE */ diff --git a/externals/zstd/lib/common/fse.h b/externals/zstd/lib/common/fse.h new file mode 100644 index 0000000..2ae128e --- /dev/null +++ b/externals/zstd/lib/common/fse.h @@ -0,0 +1,640 @@ +/* ****************************************************************** + * FSE : Finite State Entropy codec + * Public Prototypes declaration + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +#if defined (__cplusplus) +extern "C" { +#endif + +#ifndef FSE_H +#define FSE_H + + +/*-***************************************** +* Dependencies +******************************************/ +#include "zstd_deps.h" /* size_t, ptrdiff_t */ + + +/*-***************************************** +* FSE_PUBLIC_API : control library symbols visibility +******************************************/ +#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4) +# define FSE_PUBLIC_API __attribute__ ((visibility ("default"))) +#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */ +# define FSE_PUBLIC_API __declspec(dllexport) +#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1) +# define FSE_PUBLIC_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define FSE_PUBLIC_API +#endif + +/*------ Version ------*/ +#define FSE_VERSION_MAJOR 0 +#define FSE_VERSION_MINOR 9 +#define FSE_VERSION_RELEASE 0 + +#define FSE_LIB_VERSION FSE_VERSION_MAJOR.FSE_VERSION_MINOR.FSE_VERSION_RELEASE +#define FSE_QUOTE(str) #str +#define FSE_EXPAND_AND_QUOTE(str) FSE_QUOTE(str) +#define FSE_VERSION_STRING FSE_EXPAND_AND_QUOTE(FSE_LIB_VERSION) + +#define FSE_VERSION_NUMBER (FSE_VERSION_MAJOR *100*100 + FSE_VERSION_MINOR *100 + FSE_VERSION_RELEASE) +FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */ + + +/*-***************************************** +* Tool functions +******************************************/ +FSE_PUBLIC_API size_t FSE_compressBound(size_t size); /* maximum compressed size */ + +/* Error Management */ +FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return value is an error code */ +FSE_PUBLIC_API const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */ + + +/*-***************************************** +* FSE detailed API +******************************************/ +/*! +FSE_compress() does the following: +1. count symbol occurrence from source[] into table count[] (see hist.h) +2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog) +3. save normalized counters to memory buffer using writeNCount() +4. build encoding table 'CTable' from normalized counters +5. encode the data stream using encoding table 'CTable' + +FSE_decompress() does the following: +1. read normalized counters with readNCount() +2. build decoding table 'DTable' from normalized counters +3. decode the data stream using decoding table 'DTable' + +The following API allows targeting specific sub-functions for advanced tasks. +For example, it's possible to compress several blocks using the same 'CTable', +or to save and provide normalized distribution using external method. +*/ + +/* *** COMPRESSION *** */ + +/*! FSE_optimalTableLog(): + dynamically downsize 'tableLog' when conditions are met. + It saves CPU time, by using smaller tables, while preserving or even improving compression ratio. + @return : recommended tableLog (necessarily <= 'maxTableLog') */ +FSE_PUBLIC_API unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); + +/*! FSE_normalizeCount(): + normalize counts so that sum(count[]) == Power_of_2 (2^tableLog) + 'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1). + useLowProbCount is a boolean parameter which trades off compressed size for + faster header decoding. When it is set to 1, the compressed data will be slightly + smaller. And when it is set to 0, FSE_readNCount() and FSE_buildDTable() will be + faster. If you are compressing a small amount of data (< 2 KB) then useLowProbCount=0 + is a good default, since header deserialization makes a big speed difference. + Otherwise, useLowProbCount=1 is a good default, since the speed difference is small. + @return : tableLog, + or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, + const unsigned* count, size_t srcSize, unsigned maxSymbolValue, unsigned useLowProbCount); + +/*! FSE_NCountWriteBound(): + Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'. + Typically useful for allocation purpose. */ +FSE_PUBLIC_API size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog); + +/*! FSE_writeNCount(): + Compactly save 'normalizedCounter' into 'buffer'. + @return : size of the compressed table, + or an errorCode, which can be tested using FSE_isError(). */ +FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize, + const short* normalizedCounter, + unsigned maxSymbolValue, unsigned tableLog); + +/*! Constructor and Destructor of FSE_CTable. + Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */ +typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */ + +/*! FSE_buildCTable(): + Builds `ct`, which must be already allocated, using FSE_createCTable(). + @return : 0, or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); + +/*! FSE_compress_usingCTable(): + Compress `src` using `ct` into `dst` which must be already allocated. + @return : size of compressed data (<= `dstCapacity`), + or 0 if compressed data could not fit into `dst`, + or an errorCode, which can be tested using FSE_isError() */ +FSE_PUBLIC_API size_t FSE_compress_usingCTable (void* dst, size_t dstCapacity, const void* src, size_t srcSize, const FSE_CTable* ct); + +/*! +Tutorial : +---------- +The first step is to count all symbols. FSE_count() does this job very fast. +Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells. +'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0] +maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value) +FSE_count() will return the number of occurrence of the most frequent symbol. +This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility. +If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). + +The next step is to normalize the frequencies. +FSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'. +It also guarantees a minimum of 1 to any Symbol with frequency >= 1. +You can use 'tableLog'==0 to mean "use default tableLog value". +If you are unsure of which tableLog value to use, you can ask FSE_optimalTableLog(), +which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means "default"). + +The result of FSE_normalizeCount() will be saved into a table, +called 'normalizedCounter', which is a table of signed short. +'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells. +The return value is tableLog if everything proceeded as expected. +It is 0 if there is a single symbol within distribution. +If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()). + +'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeNCount(). +'buffer' must be already allocated. +For guaranteed success, buffer size must be at least FSE_headerBound(). +The result of the function is the number of bytes written into 'buffer'. +If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError(); ex : buffer size too small). + +'normalizedCounter' can then be used to create the compression table 'CTable'. +The space required by 'CTable' must be already allocated, using FSE_createCTable(). +You can then use FSE_buildCTable() to fill 'CTable'. +If there is an error, both functions will return an ErrorCode (which can be tested using FSE_isError()). + +'CTable' can then be used to compress 'src', with FSE_compress_usingCTable(). +Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize' +The function returns the size of compressed data (without header), necessarily <= `dstCapacity`. +If it returns '0', compressed data could not fit into 'dst'. +If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). +*/ + + +/* *** DECOMPRESSION *** */ + +/*! FSE_readNCount(): + Read compactly saved 'normalizedCounter' from 'rBuffer'. + @return : size read from 'rBuffer', + or an errorCode, which can be tested using FSE_isError(). + maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */ +FSE_PUBLIC_API size_t FSE_readNCount (short* normalizedCounter, + unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, + const void* rBuffer, size_t rBuffSize); + +/*! FSE_readNCount_bmi2(): + * Same as FSE_readNCount() but pass bmi2=1 when your CPU supports BMI2 and 0 otherwise. + */ +FSE_PUBLIC_API size_t FSE_readNCount_bmi2(short* normalizedCounter, + unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, + const void* rBuffer, size_t rBuffSize, int bmi2); + +typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */ + +/*! +Tutorial : +---------- +(Note : these functions only decompress FSE-compressed blocks. + If block is uncompressed, use memcpy() instead + If block is a single repeated byte, use memset() instead ) + +The first step is to obtain the normalized frequencies of symbols. +This can be performed by FSE_readNCount() if it was saved using FSE_writeNCount(). +'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short. +In practice, that means it's necessary to know 'maxSymbolValue' beforehand, +or size the table to handle worst case situations (typically 256). +FSE_readNCount() will provide 'tableLog' and 'maxSymbolValue'. +The result of FSE_readNCount() is the number of bytes read from 'rBuffer'. +Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that. +If there is an error, the function will return an error code, which can be tested using FSE_isError(). + +The next step is to build the decompression tables 'FSE_DTable' from 'normalizedCounter'. +This is performed by the function FSE_buildDTable(). +The space required by 'FSE_DTable' must be already allocated using FSE_createDTable(). +If there is an error, the function will return an error code, which can be tested using FSE_isError(). + +`FSE_DTable` can then be used to decompress `cSrc`, with FSE_decompress_usingDTable(). +`cSrcSize` must be strictly correct, otherwise decompression will fail. +FSE_decompress_usingDTable() result will tell how many bytes were regenerated (<=`dstCapacity`). +If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small) +*/ + +#endif /* FSE_H */ + + +#if defined(FSE_STATIC_LINKING_ONLY) && !defined(FSE_H_FSE_STATIC_LINKING_ONLY) +#define FSE_H_FSE_STATIC_LINKING_ONLY + +/* *** Dependency *** */ +#include "bitstream.h" + + +/* ***************************************** +* Static allocation +*******************************************/ +/* FSE buffer bounds */ +#define FSE_NCOUNTBOUND 512 +#define FSE_BLOCKBOUND(size) ((size) + ((size)>>7) + 4 /* fse states */ + sizeof(size_t) /* bitContainer */) +#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ + +/* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */ +#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<((maxTableLog)-1)) + (((maxSymbolValue)+1)*2)) +#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<<(maxTableLog))) + +/* or use the size to malloc() space directly. Pay attention to alignment restrictions though */ +#define FSE_CTABLE_SIZE(maxTableLog, maxSymbolValue) (FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(FSE_CTable)) +#define FSE_DTABLE_SIZE(maxTableLog) (FSE_DTABLE_SIZE_U32(maxTableLog) * sizeof(FSE_DTable)) + + +/* ***************************************** + * FSE advanced API + ***************************************** */ + +unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus); +/**< same as FSE_optimalTableLog(), which used `minus==2` */ + +size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue); +/**< build a fake FSE_CTable, designed to compress always the same symbolValue */ + +/* FSE_buildCTable_wksp() : + * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). + * `wkspSize` must be >= `FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog)` of `unsigned`. + * See FSE_buildCTable_wksp() for breakdown of workspace usage. + */ +#define FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog) (((maxSymbolValue + 2) + (1ull << (tableLog)))/2 + sizeof(U64)/sizeof(U32) /* additional 8 bytes for potential table overwrite */) +#define FSE_BUILD_CTABLE_WORKSPACE_SIZE(maxSymbolValue, tableLog) (sizeof(unsigned) * FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(maxSymbolValue, tableLog)) +size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); + +#define FSE_BUILD_DTABLE_WKSP_SIZE(maxTableLog, maxSymbolValue) (sizeof(short) * (maxSymbolValue + 1) + (1ULL << maxTableLog) + 8) +#define FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ((FSE_BUILD_DTABLE_WKSP_SIZE(maxTableLog, maxSymbolValue) + sizeof(unsigned) - 1) / sizeof(unsigned)) +FSE_PUBLIC_API size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); +/**< Same as FSE_buildDTable(), using an externally allocated `workspace` produced with `FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxSymbolValue)` */ + +#define FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) (FSE_DTABLE_SIZE_U32(maxTableLog) + 1 + FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) + (FSE_MAX_SYMBOL_VALUE + 1) / 2 + 1) +#define FSE_DECOMPRESS_WKSP_SIZE(maxTableLog, maxSymbolValue) (FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(unsigned)) +size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2); +/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DECOMPRESS_WKSP_SIZE_U32(maxLog, maxSymbolValue)`. + * Set bmi2 to 1 if your CPU supports BMI2 or 0 if it doesn't */ + +typedef enum { + FSE_repeat_none, /**< Cannot use the previous table */ + FSE_repeat_check, /**< Can use the previous table but it must be checked */ + FSE_repeat_valid /**< Can use the previous table and it is assumed to be valid */ + } FSE_repeat; + +/* ***************************************** +* FSE symbol compression API +*******************************************/ +/*! + This API consists of small unitary functions, which highly benefit from being inlined. + Hence their body are included in next section. +*/ +typedef struct { + ptrdiff_t value; + const void* stateTable; + const void* symbolTT; + unsigned stateLog; +} FSE_CState_t; + +static void FSE_initCState(FSE_CState_t* CStatePtr, const FSE_CTable* ct); + +static void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* CStatePtr, unsigned symbol); + +static void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* CStatePtr); + +/**< +These functions are inner components of FSE_compress_usingCTable(). +They allow the creation of custom streams, mixing multiple tables and bit sources. + +A key property to keep in mind is that encoding and decoding are done **in reverse direction**. +So the first symbol you will encode is the last you will decode, like a LIFO stack. + +You will need a few variables to track your CStream. They are : + +FSE_CTable ct; // Provided by FSE_buildCTable() +BIT_CStream_t bitStream; // bitStream tracking structure +FSE_CState_t state; // State tracking structure (can have several) + + +The first thing to do is to init bitStream and state. + size_t errorCode = BIT_initCStream(&bitStream, dstBuffer, maxDstSize); + FSE_initCState(&state, ct); + +Note that BIT_initCStream() can produce an error code, so its result should be tested, using FSE_isError(); +You can then encode your input data, byte after byte. +FSE_encodeSymbol() outputs a maximum of 'tableLog' bits at a time. +Remember decoding will be done in reverse direction. + FSE_encodeByte(&bitStream, &state, symbol); + +At any time, you can also add any bit sequence. +Note : maximum allowed nbBits is 25, for compatibility with 32-bits decoders + BIT_addBits(&bitStream, bitField, nbBits); + +The above methods don't commit data to memory, they just store it into local register, for speed. +Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). +Writing data to memory is a manual operation, performed by the flushBits function. + BIT_flushBits(&bitStream); + +Your last FSE encoding operation shall be to flush your last state value(s). + FSE_flushState(&bitStream, &state); + +Finally, you must close the bitStream. +The function returns the size of CStream in bytes. +If data couldn't fit into dstBuffer, it will return a 0 ( == not compressible) +If there is an error, it returns an errorCode (which can be tested using FSE_isError()). + size_t size = BIT_closeCStream(&bitStream); +*/ + + +/* ***************************************** +* FSE symbol decompression API +*******************************************/ +typedef struct { + size_t state; + const void* table; /* precise table may vary, depending on U16 */ +} FSE_DState_t; + + +static void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt); + +static unsigned char FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD); + +static unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr); + +/**< +Let's now decompose FSE_decompress_usingDTable() into its unitary components. +You will decode FSE-encoded symbols from the bitStream, +and also any other bitFields you put in, **in reverse order**. + +You will need a few variables to track your bitStream. They are : + +BIT_DStream_t DStream; // Stream context +FSE_DState_t DState; // State context. Multiple ones are possible +FSE_DTable* DTablePtr; // Decoding table, provided by FSE_buildDTable() + +The first thing to do is to init the bitStream. + errorCode = BIT_initDStream(&DStream, srcBuffer, srcSize); + +You should then retrieve your initial state(s) +(in reverse flushing order if you have several ones) : + errorCode = FSE_initDState(&DState, &DStream, DTablePtr); + +You can then decode your data, symbol after symbol. +For information the maximum number of bits read by FSE_decodeSymbol() is 'tableLog'. +Keep in mind that symbols are decoded in reverse order, like a LIFO stack (last in, first out). + unsigned char symbol = FSE_decodeSymbol(&DState, &DStream); + +You can retrieve any bitfield you eventually stored into the bitStream (in reverse order) +Note : maximum allowed nbBits is 25, for 32-bits compatibility + size_t bitField = BIT_readBits(&DStream, nbBits); + +All above operations only read from local register (which size depends on size_t). +Refueling the register from memory is manually performed by the reload method. + endSignal = FSE_reloadDStream(&DStream); + +BIT_reloadDStream() result tells if there is still some more data to read from DStream. +BIT_DStream_unfinished : there is still some data left into the DStream. +BIT_DStream_endOfBuffer : Dstream reached end of buffer. Its container may no longer be completely filled. +BIT_DStream_completed : Dstream reached its exact end, corresponding in general to decompression completed. +BIT_DStream_tooFar : Dstream went too far. Decompression result is corrupted. + +When reaching end of buffer (BIT_DStream_endOfBuffer), progress slowly, notably if you decode multiple symbols per loop, +to properly detect the exact end of stream. +After each decoded symbol, check if DStream is fully consumed using this simple test : + BIT_reloadDStream(&DStream) >= BIT_DStream_completed + +When it's done, verify decompression is fully completed, by checking both DStream and the relevant states. +Checking if DStream has reached its end is performed by : + BIT_endOfDStream(&DStream); +Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible. + FSE_endOfDState(&DState); +*/ + + +/* ***************************************** +* FSE unsafe API +*******************************************/ +static unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD); +/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */ + + +/* ***************************************** +* Implementation of inlined functions +*******************************************/ +typedef struct { + int deltaFindState; + U32 deltaNbBits; +} FSE_symbolCompressionTransform; /* total 8 bytes */ + +MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct) +{ + const void* ptr = ct; + const U16* u16ptr = (const U16*) ptr; + const U32 tableLog = MEM_read16(ptr); + statePtr->value = (ptrdiff_t)1<stateTable = u16ptr+2; + statePtr->symbolTT = ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1); + statePtr->stateLog = tableLog; +} + + +/*! FSE_initCState2() : +* Same as FSE_initCState(), but the first symbol to include (which will be the last to be read) +* uses the smallest state value possible, saving the cost of this symbol */ +MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U32 symbol) +{ + FSE_initCState(statePtr, ct); + { const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; + const U16* stateTable = (const U16*)(statePtr->stateTable); + U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1<<15)) >> 16); + statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits; + statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; + } +} + +MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, unsigned symbol) +{ + FSE_symbolCompressionTransform const symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; + const U16* const stateTable = (const U16*)(statePtr->stateTable); + U32 const nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16); + BIT_addBits(bitC, (size_t)statePtr->value, nbBitsOut); + statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; +} + +MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr) +{ + BIT_addBits(bitC, (size_t)statePtr->value, statePtr->stateLog); + BIT_flushBits(bitC); +} + + +/* FSE_getMaxNbBits() : + * Approximate maximum cost of a symbol, in bits. + * Fractional get rounded up (i.e. a symbol with a normalized frequency of 3 gives the same result as a frequency of 2) + * note 1 : assume symbolValue is valid (<= maxSymbolValue) + * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */ +MEM_STATIC U32 FSE_getMaxNbBits(const void* symbolTTPtr, U32 symbolValue) +{ + const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr; + return (symbolTT[symbolValue].deltaNbBits + ((1<<16)-1)) >> 16; +} + +/* FSE_bitCost() : + * Approximate symbol cost, as fractional value, using fixed-point format (accuracyLog fractional bits) + * note 1 : assume symbolValue is valid (<= maxSymbolValue) + * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */ +MEM_STATIC U32 FSE_bitCost(const void* symbolTTPtr, U32 tableLog, U32 symbolValue, U32 accuracyLog) +{ + const FSE_symbolCompressionTransform* symbolTT = (const FSE_symbolCompressionTransform*) symbolTTPtr; + U32 const minNbBits = symbolTT[symbolValue].deltaNbBits >> 16; + U32 const threshold = (minNbBits+1) << 16; + assert(tableLog < 16); + assert(accuracyLog < 31-tableLog); /* ensure enough room for renormalization double shift */ + { U32 const tableSize = 1 << tableLog; + U32 const deltaFromThreshold = threshold - (symbolTT[symbolValue].deltaNbBits + tableSize); + U32 const normalizedDeltaFromThreshold = (deltaFromThreshold << accuracyLog) >> tableLog; /* linear interpolation (very approximate) */ + U32 const bitMultiplier = 1 << accuracyLog; + assert(symbolTT[symbolValue].deltaNbBits + tableSize <= threshold); + assert(normalizedDeltaFromThreshold <= bitMultiplier); + return (minNbBits+1)*bitMultiplier - normalizedDeltaFromThreshold; + } +} + + +/* ====== Decompression ====== */ + +typedef struct { + U16 tableLog; + U16 fastMode; +} FSE_DTableHeader; /* sizeof U32 */ + +typedef struct +{ + unsigned short newState; + unsigned char symbol; + unsigned char nbBits; +} FSE_decode_t; /* size == U32 */ + +MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt) +{ + const void* ptr = dt; + const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr; + DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); + BIT_reloadDStream(bitD); + DStatePtr->table = dt + 1; +} + +MEM_STATIC BYTE FSE_peekSymbol(const FSE_DState_t* DStatePtr) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + return DInfo.symbol; +} + +MEM_STATIC void FSE_updateState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + size_t const lowBits = BIT_readBits(bitD, nbBits); + DStatePtr->state = DInfo.newState + lowBits; +} + +MEM_STATIC BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + BYTE const symbol = DInfo.symbol; + size_t const lowBits = BIT_readBits(bitD, nbBits); + + DStatePtr->state = DInfo.newState + lowBits; + return symbol; +} + +/*! FSE_decodeSymbolFast() : + unsafe, only works if no symbol has a probability > 50% */ +MEM_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + BYTE const symbol = DInfo.symbol; + size_t const lowBits = BIT_readBitsFast(bitD, nbBits); + + DStatePtr->state = DInfo.newState + lowBits; + return symbol; +} + +MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) +{ + return DStatePtr->state == 0; +} + + + +#ifndef FSE_COMMONDEFS_ONLY + +/* ************************************************************** +* Tuning parameters +****************************************************************/ +/*!MEMORY_USAGE : +* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) +* Increasing memory usage improves compression ratio +* Reduced memory usage can improve speed, due to cache effect +* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ +#ifndef FSE_MAX_MEMORY_USAGE +# define FSE_MAX_MEMORY_USAGE 14 +#endif +#ifndef FSE_DEFAULT_MEMORY_USAGE +# define FSE_DEFAULT_MEMORY_USAGE 13 +#endif +#if (FSE_DEFAULT_MEMORY_USAGE > FSE_MAX_MEMORY_USAGE) +# error "FSE_DEFAULT_MEMORY_USAGE must be <= FSE_MAX_MEMORY_USAGE" +#endif + +/*!FSE_MAX_SYMBOL_VALUE : +* Maximum symbol value authorized. +* Required for proper stack allocation */ +#ifndef FSE_MAX_SYMBOL_VALUE +# define FSE_MAX_SYMBOL_VALUE 255 +#endif + +/* ************************************************************** +* template functions type & suffix +****************************************************************/ +#define FSE_FUNCTION_TYPE BYTE +#define FSE_FUNCTION_EXTENSION +#define FSE_DECODE_TYPE FSE_decode_t + + +#endif /* !FSE_COMMONDEFS_ONLY */ + + +/* *************************************************************** +* Constants +*****************************************************************/ +#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2) +#define FSE_MAX_TABLESIZE (1U< FSE_TABLELOG_ABSOLUTE_MAX +# error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported" +#endif + +#define FSE_TABLESTEP(tableSize) (((tableSize)>>1) + ((tableSize)>>3) + 3) + + +#endif /* FSE_STATIC_LINKING_ONLY */ + + +#if defined (__cplusplus) +} +#endif diff --git a/externals/zstd/lib/common/fse_decompress.c b/externals/zstd/lib/common/fse_decompress.c new file mode 100644 index 0000000..0dcc464 --- /dev/null +++ b/externals/zstd/lib/common/fse_decompress.c @@ -0,0 +1,313 @@ +/* ****************************************************************** + * FSE : Finite State Entropy decoder + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + * - Public forum : https://groups.google.com/forum/#!forum/lz4c + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + + +/* ************************************************************** +* Includes +****************************************************************/ +#include "debug.h" /* assert */ +#include "bitstream.h" +#include "compiler.h" +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" +#include "error_private.h" +#include "zstd_deps.h" /* ZSTD_memcpy */ +#include "bits.h" /* ZSTD_highbit32 */ + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define FSE_isError ERR_isError +#define FSE_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) /* use only *after* variable declarations */ + + +/* ************************************************************** +* Templates +****************************************************************/ +/* + designed to be included + for type-specific functions (template emulation in C) + Objective is to write these functions only once, for improved maintenance +*/ + +/* safety checks */ +#ifndef FSE_FUNCTION_EXTENSION +# error "FSE_FUNCTION_EXTENSION must be defined" +#endif +#ifndef FSE_FUNCTION_TYPE +# error "FSE_FUNCTION_TYPE must be defined" +#endif + +/* Function names */ +#define FSE_CAT(X,Y) X##Y +#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) +#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) + +static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize) +{ + void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */ + FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr); + U16* symbolNext = (U16*)workSpace; + BYTE* spread = (BYTE*)(symbolNext + maxSymbolValue + 1); + + U32 const maxSV1 = maxSymbolValue + 1; + U32 const tableSize = 1 << tableLog; + U32 highThreshold = tableSize-1; + + /* Sanity Checks */ + if (FSE_BUILD_DTABLE_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(maxSymbolValue_tooLarge); + if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge); + if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); + + /* Init, lay down lowprob symbols */ + { FSE_DTableHeader DTableH; + DTableH.tableLog = (U16)tableLog; + DTableH.fastMode = 1; + { S16 const largeLimit= (S16)(1 << (tableLog-1)); + U32 s; + for (s=0; s= largeLimit) DTableH.fastMode=0; + symbolNext[s] = (U16)normalizedCounter[s]; + } } } + ZSTD_memcpy(dt, &DTableH, sizeof(DTableH)); + } + + /* Spread symbols */ + if (highThreshold == tableSize - 1) { + size_t const tableMask = tableSize-1; + size_t const step = FSE_TABLESTEP(tableSize); + /* First lay down the symbols in order. + * We use a uint64_t to lay down 8 bytes at a time. This reduces branch + * misses since small blocks generally have small table logs, so nearly + * all symbols have counts <= 8. We ensure we have 8 bytes at the end of + * our buffer to handle the over-write. + */ + { U64 const add = 0x0101010101010101ull; + size_t pos = 0; + U64 sv = 0; + U32 s; + for (s=0; s highThreshold) position = (position + step) & tableMask; /* lowprob area */ + } } + if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ + } + + /* Build Decoding table */ + { U32 u; + for (u=0; u sizeof(bitD.bitContainer)*8) /* This test must be static */ + BIT_reloadDStream(&bitD); + + op[1] = FSE_GETSYMBOL(&state2); + + if (FSE_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ + { if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { op+=2; break; } } + + op[2] = FSE_GETSYMBOL(&state1); + + if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ + BIT_reloadDStream(&bitD); + + op[3] = FSE_GETSYMBOL(&state2); + } + + /* tail */ + /* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */ + while (1) { + if (op>(omax-2)) return ERROR(dstSize_tooSmall); + *op++ = FSE_GETSYMBOL(&state1); + if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { + *op++ = FSE_GETSYMBOL(&state2); + break; + } + + if (op>(omax-2)) return ERROR(dstSize_tooSmall); + *op++ = FSE_GETSYMBOL(&state2); + if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { + *op++ = FSE_GETSYMBOL(&state1); + break; + } } + + assert(op >= ostart); + return (size_t)(op-ostart); +} + +typedef struct { + short ncount[FSE_MAX_SYMBOL_VALUE + 1]; +} FSE_DecompressWksp; + + +FORCE_INLINE_TEMPLATE size_t FSE_decompress_wksp_body( + void* dst, size_t dstCapacity, + const void* cSrc, size_t cSrcSize, + unsigned maxLog, void* workSpace, size_t wkspSize, + int bmi2) +{ + const BYTE* const istart = (const BYTE*)cSrc; + const BYTE* ip = istart; + unsigned tableLog; + unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE; + FSE_DecompressWksp* const wksp = (FSE_DecompressWksp*)workSpace; + size_t const dtablePos = sizeof(FSE_DecompressWksp) / sizeof(FSE_DTable); + FSE_DTable* const dtable = (FSE_DTable*)workSpace + dtablePos; + + FSE_STATIC_ASSERT((FSE_MAX_SYMBOL_VALUE + 1) % 2 == 0); + if (wkspSize < sizeof(*wksp)) return ERROR(GENERIC); + + /* correct offset to dtable depends on this property */ + FSE_STATIC_ASSERT(sizeof(FSE_DecompressWksp) % sizeof(FSE_DTable) == 0); + + /* normal FSE decoding mode */ + { size_t const NCountLength = + FSE_readNCount_bmi2(wksp->ncount, &maxSymbolValue, &tableLog, istart, cSrcSize, bmi2); + if (FSE_isError(NCountLength)) return NCountLength; + if (tableLog > maxLog) return ERROR(tableLog_tooLarge); + assert(NCountLength <= cSrcSize); + ip += NCountLength; + cSrcSize -= NCountLength; + } + + if (FSE_DECOMPRESS_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(tableLog_tooLarge); + assert(sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog) <= wkspSize); + workSpace = (BYTE*)workSpace + sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog); + wkspSize -= sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog); + + CHECK_F( FSE_buildDTable_internal(dtable, wksp->ncount, maxSymbolValue, tableLog, workSpace, wkspSize) ); + + { + const void* ptr = dtable; + const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr; + const U32 fastMode = DTableH->fastMode; + + /* select fast mode (static) */ + if (fastMode) return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, dtable, 1); + return FSE_decompress_usingDTable_generic(dst, dstCapacity, ip, cSrcSize, dtable, 0); + } +} + +/* Avoids the FORCE_INLINE of the _body() function. */ +static size_t FSE_decompress_wksp_body_default(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize) +{ + return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 0); +} + +#if DYNAMIC_BMI2 +BMI2_TARGET_ATTRIBUTE static size_t FSE_decompress_wksp_body_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize) +{ + return FSE_decompress_wksp_body(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, 1); +} +#endif + +size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2) +{ +#if DYNAMIC_BMI2 + if (bmi2) { + return FSE_decompress_wksp_body_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize); + } +#endif + (void)bmi2; + return FSE_decompress_wksp_body_default(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize); +} + +#endif /* FSE_COMMONDEFS_ONLY */ diff --git a/externals/zstd/lib/common/huf.h b/externals/zstd/lib/common/huf.h new file mode 100644 index 0000000..99bf85d --- /dev/null +++ b/externals/zstd/lib/common/huf.h @@ -0,0 +1,286 @@ +/* ****************************************************************** + * huff0 huffman codec, + * part of Finite State Entropy library + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +#if defined (__cplusplus) +extern "C" { +#endif + +#ifndef HUF_H_298734234 +#define HUF_H_298734234 + +/* *** Dependencies *** */ +#include "zstd_deps.h" /* size_t */ +#include "mem.h" /* U32 */ +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" + + +/* *** Tool functions *** */ +#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */ +size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */ + +/* Error Management */ +unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */ +const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */ + + +#define HUF_WORKSPACE_SIZE ((8 << 10) + 512 /* sorting scratch space */) +#define HUF_WORKSPACE_SIZE_U64 (HUF_WORKSPACE_SIZE / sizeof(U64)) + +/* *** Constants *** */ +#define HUF_TABLELOG_MAX 12 /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_TABLELOG_ABSOLUTEMAX */ +#define HUF_TABLELOG_DEFAULT 11 /* default tableLog value when none specified */ +#define HUF_SYMBOLVALUE_MAX 255 + +#define HUF_TABLELOG_ABSOLUTEMAX 12 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ +#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX) +# error "HUF_TABLELOG_MAX is too large !" +#endif + + +/* **************************************** +* Static allocation +******************************************/ +/* HUF buffer bounds */ +#define HUF_CTABLEBOUND 129 +#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true when incompressible is pre-filtered with fast heuristic */ +#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ + +/* static allocation of HUF's Compression Table */ +/* this is a private definition, just exposed for allocation and strict aliasing purpose. never EVER access its members directly */ +typedef size_t HUF_CElt; /* consider it an incomplete type */ +#define HUF_CTABLE_SIZE_ST(maxSymbolValue) ((maxSymbolValue)+2) /* Use tables of size_t, for proper alignment */ +#define HUF_CTABLE_SIZE(maxSymbolValue) (HUF_CTABLE_SIZE_ST(maxSymbolValue) * sizeof(size_t)) +#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \ + HUF_CElt name[HUF_CTABLE_SIZE_ST(maxSymbolValue)] /* no final ; */ + +/* static allocation of HUF's DTable */ +typedef U32 HUF_DTable; +#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog))) +#define HUF_CREATE_STATIC_DTABLEX1(DTable, maxTableLog) \ + HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1) * 0x01000001) } +#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \ + HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog) * 0x01000001) } + + +/* **************************************** +* Advanced decompression functions +******************************************/ + +/** + * Huffman flags bitset. + * For all flags, 0 is the default value. + */ +typedef enum { + /** + * If compiled with DYNAMIC_BMI2: Set flag only if the CPU supports BMI2 at runtime. + * Otherwise: Ignored. + */ + HUF_flags_bmi2 = (1 << 0), + /** + * If set: Test possible table depths to find the one that produces the smallest header + encoded size. + * If unset: Use heuristic to find the table depth. + */ + HUF_flags_optimalDepth = (1 << 1), + /** + * If set: If the previous table can encode the input, always reuse the previous table. + * If unset: If the previous table can encode the input, reuse the previous table if it results in a smaller output. + */ + HUF_flags_preferRepeat = (1 << 2), + /** + * If set: Sample the input and check if the sample is uncompressible, if it is then don't attempt to compress. + * If unset: Always histogram the entire input. + */ + HUF_flags_suspectUncompressible = (1 << 3), + /** + * If set: Don't use assembly implementations + * If unset: Allow using assembly implementations + */ + HUF_flags_disableAsm = (1 << 4), + /** + * If set: Don't use the fast decoding loop, always use the fallback decoding loop. + * If unset: Use the fast decoding loop when possible. + */ + HUF_flags_disableFast = (1 << 5) +} HUF_flags_e; + + +/* **************************************** + * HUF detailed API + * ****************************************/ +#define HUF_OPTIMAL_DEPTH_THRESHOLD ZSTD_btultra + +/*! HUF_compress() does the following: + * 1. count symbol occurrence from source[] into table count[] using FSE_count() (exposed within "fse.h") + * 2. (optional) refine tableLog using HUF_optimalTableLog() + * 3. build Huffman table from count using HUF_buildCTable() + * 4. save Huffman table to memory buffer using HUF_writeCTable() + * 5. encode the data stream using HUF_compress4X_usingCTable() + * + * The following API allows targeting specific sub-functions for advanced tasks. + * For example, it's possible to compress several blocks using the same 'CTable', + * or to save and regenerate 'CTable' using external methods. + */ +unsigned HUF_minTableLog(unsigned symbolCardinality); +unsigned HUF_cardinality(const unsigned* count, unsigned maxSymbolValue); +unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, void* workSpace, + size_t wkspSize, HUF_CElt* table, const unsigned* count, int flags); /* table is used as scratch space for building and testing tables, not a return value */ +size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog, void* workspace, size_t workspaceSize); +size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags); +size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue); +int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue); + +typedef enum { + HUF_repeat_none, /**< Cannot use the previous table */ + HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */ + HUF_repeat_valid /**< Can use the previous table and it is assumed to be valid */ + } HUF_repeat; + +/** HUF_compress4X_repeat() : + * Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. + * If it uses hufTable it does not modify hufTable or repeat. + * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. + * If preferRepeat then the old table will always be used if valid. + * If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */ +size_t HUF_compress4X_repeat(void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned tableLog, + void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */ + HUF_CElt* hufTable, HUF_repeat* repeat, int flags); + +/** HUF_buildCTable_wksp() : + * Same as HUF_buildCTable(), but using externally allocated scratch buffer. + * `workSpace` must be aligned on 4-bytes boundaries, and its size must be >= HUF_CTABLE_WORKSPACE_SIZE. + */ +#define HUF_CTABLE_WORKSPACE_SIZE_U32 ((4 * (HUF_SYMBOLVALUE_MAX + 1)) + 192) +#define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned)) +size_t HUF_buildCTable_wksp (HUF_CElt* tree, + const unsigned* count, U32 maxSymbolValue, U32 maxNbBits, + void* workSpace, size_t wkspSize); + +/*! HUF_readStats() : + * Read compact Huffman tree, saved by HUF_writeCTable(). + * `huffWeight` is destination buffer. + * @return : size read from `src` , or an error Code . + * Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */ +size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, + U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize); + +/*! HUF_readStats_wksp() : + * Same as HUF_readStats() but takes an external workspace which must be + * 4-byte aligned and its size must be >= HUF_READ_STATS_WORKSPACE_SIZE. + * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0. + */ +#define HUF_READ_STATS_WORKSPACE_SIZE_U32 FSE_DECOMPRESS_WKSP_SIZE_U32(6, HUF_TABLELOG_MAX-1) +#define HUF_READ_STATS_WORKSPACE_SIZE (HUF_READ_STATS_WORKSPACE_SIZE_U32 * sizeof(unsigned)) +size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, + U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize, + void* workspace, size_t wkspSize, + int flags); + +/** HUF_readCTable() : + * Loading a CTable saved with HUF_writeCTable() */ +size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned *hasZeroWeights); + +/** HUF_getNbBitsFromCTable() : + * Read nbBits from CTable symbolTable, for symbol `symbolValue` presumed <= HUF_SYMBOLVALUE_MAX + * Note 1 : If symbolValue > HUF_readCTableHeader(symbolTable).maxSymbolValue, returns 0 + * Note 2 : is not inlined, as HUF_CElt definition is private + */ +U32 HUF_getNbBitsFromCTable(const HUF_CElt* symbolTable, U32 symbolValue); + +typedef struct { + BYTE tableLog; + BYTE maxSymbolValue; + BYTE unused[sizeof(size_t) - 2]; +} HUF_CTableHeader; + +/** HUF_readCTableHeader() : + * @returns The header from the CTable specifying the tableLog and the maxSymbolValue. + */ +HUF_CTableHeader HUF_readCTableHeader(HUF_CElt const* ctable); + +/* + * HUF_decompress() does the following: + * 1. select the decompression algorithm (X1, X2) based on pre-computed heuristics + * 2. build Huffman table from save, using HUF_readDTableX?() + * 3. decode 1 or 4 segments in parallel using HUF_decompress?X?_usingDTable() + */ + +/** HUF_selectDecoder() : + * Tells which decoder is likely to decode faster, + * based on a set of pre-computed metrics. + * @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 . + * Assumption : 0 < dstSize <= 128 KB */ +U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize); + +/** + * The minimum workspace size for the `workSpace` used in + * HUF_readDTableX1_wksp() and HUF_readDTableX2_wksp(). + * + * The space used depends on HUF_TABLELOG_MAX, ranging from ~1500 bytes when + * HUF_TABLE_LOG_MAX=12 to ~1850 bytes when HUF_TABLE_LOG_MAX=15. + * Buffer overflow errors may potentially occur if code modifications result in + * a required workspace size greater than that specified in the following + * macro. + */ +#define HUF_DECOMPRESS_WORKSPACE_SIZE ((2 << 10) + (1 << 9)) +#define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32)) + + +/* ====================== */ +/* single stream variants */ +/* ====================== */ + +size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags); +/** HUF_compress1X_repeat() : + * Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. + * If it uses hufTable it does not modify hufTable or repeat. + * If it doesn't, it sets *repeat = HUF_repeat_none, and it sets hufTable to the table used. + * If preferRepeat then the old table will always be used if valid. + * If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */ +size_t HUF_compress1X_repeat(void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned tableLog, + void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */ + HUF_CElt* hufTable, HUF_repeat* repeat, int flags); + +size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); +#ifndef HUF_FORCE_DECOMPRESS_X1 +size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); /**< double-symbols decoder */ +#endif + +/* BMI2 variants. + * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0. + */ +size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags); +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); +#endif +size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags); +size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags); +#endif +#ifndef HUF_FORCE_DECOMPRESS_X1 +size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags); +#endif + +#endif /* HUF_H_298734234 */ + +#if defined (__cplusplus) +} +#endif diff --git a/externals/zstd/lib/common/mem.h b/externals/zstd/lib/common/mem.h new file mode 100644 index 0000000..096f4be --- /dev/null +++ b/externals/zstd/lib/common/mem.h @@ -0,0 +1,426 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef MEM_H_MODULE +#define MEM_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + +/*-**************************************** +* Dependencies +******************************************/ +#include /* size_t, ptrdiff_t */ +#include "compiler.h" /* __has_builtin */ +#include "debug.h" /* DEBUG_STATIC_ASSERT */ +#include "zstd_deps.h" /* ZSTD_memcpy */ + + +/*-**************************************** +* Compiler specifics +******************************************/ +#if defined(_MSC_VER) /* Visual Studio */ +# include /* _byteswap_ulong */ +# include /* _byteswap_* */ +#endif + +/*-************************************************************** +* Basic Types +*****************************************************************/ +#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# if defined(_AIX) +# include +# else +# include /* intptr_t */ +# endif + typedef uint8_t BYTE; + typedef uint8_t U8; + typedef int8_t S8; + typedef uint16_t U16; + typedef int16_t S16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; + typedef int64_t S64; +#else +# include +#if CHAR_BIT != 8 +# error "this implementation requires char to be exactly 8-bit type" +#endif + typedef unsigned char BYTE; + typedef unsigned char U8; + typedef signed char S8; +#if USHRT_MAX != 65535 +# error "this implementation requires short to be exactly 16-bit type" +#endif + typedef unsigned short U16; + typedef signed short S16; +#if UINT_MAX != 4294967295 +# error "this implementation requires int to be exactly 32-bit type" +#endif + typedef unsigned int U32; + typedef signed int S32; +/* note : there are no limits defined for long long type in C90. + * limits exist in C99, however, in such case, is preferred */ + typedef unsigned long long U64; + typedef signed long long S64; +#endif + + +/*-************************************************************** +* Memory I/O API +*****************************************************************/ +/*=== Static platform detection ===*/ +MEM_STATIC unsigned MEM_32bits(void); +MEM_STATIC unsigned MEM_64bits(void); +MEM_STATIC unsigned MEM_isLittleEndian(void); + +/*=== Native unaligned read/write ===*/ +MEM_STATIC U16 MEM_read16(const void* memPtr); +MEM_STATIC U32 MEM_read32(const void* memPtr); +MEM_STATIC U64 MEM_read64(const void* memPtr); +MEM_STATIC size_t MEM_readST(const void* memPtr); + +MEM_STATIC void MEM_write16(void* memPtr, U16 value); +MEM_STATIC void MEM_write32(void* memPtr, U32 value); +MEM_STATIC void MEM_write64(void* memPtr, U64 value); + +/*=== Little endian unaligned read/write ===*/ +MEM_STATIC U16 MEM_readLE16(const void* memPtr); +MEM_STATIC U32 MEM_readLE24(const void* memPtr); +MEM_STATIC U32 MEM_readLE32(const void* memPtr); +MEM_STATIC U64 MEM_readLE64(const void* memPtr); +MEM_STATIC size_t MEM_readLEST(const void* memPtr); + +MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val); +MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val); +MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32); +MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64); +MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val); + +/*=== Big endian unaligned read/write ===*/ +MEM_STATIC U32 MEM_readBE32(const void* memPtr); +MEM_STATIC U64 MEM_readBE64(const void* memPtr); +MEM_STATIC size_t MEM_readBEST(const void* memPtr); + +MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32); +MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64); +MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val); + +/*=== Byteswap ===*/ +MEM_STATIC U32 MEM_swap32(U32 in); +MEM_STATIC U64 MEM_swap64(U64 in); +MEM_STATIC size_t MEM_swapST(size_t in); + + +/*-************************************************************** +* Memory I/O Implementation +*****************************************************************/ +/* MEM_FORCE_MEMORY_ACCESS : For accessing unaligned memory: + * Method 0 : always use `memcpy()`. Safe and portable. + * Method 1 : Use compiler extension to set unaligned access. + * Method 2 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets depending on alignment. + * Default : method 1 if supported, else method 0 + */ +#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ +# ifdef __GNUC__ +# define MEM_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; } +MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; } + +MEM_STATIC unsigned MEM_isLittleEndian(void) +{ +#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) + return 1; +#elif defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) + return 0; +#elif defined(__clang__) && __LITTLE_ENDIAN__ + return 1; +#elif defined(__clang__) && __BIG_ENDIAN__ + return 0; +#elif defined(_MSC_VER) && (_M_AMD64 || _M_IX86) + return 1; +#elif defined(__DMC__) && defined(_M_IX86) + return 1; +#else + const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ + return one.c[0]; +#endif +} + +#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2) + +/* violates C standard, by lying on structure alignment. +Only use if no other choice to achieve best performance on target platform */ +MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; } +MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; } +MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; } +MEM_STATIC size_t MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; } + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } +MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } +MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; } + +#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1) + +typedef __attribute__((aligned(1))) U16 unalign16; +typedef __attribute__((aligned(1))) U32 unalign32; +typedef __attribute__((aligned(1))) U64 unalign64; +typedef __attribute__((aligned(1))) size_t unalignArch; + +MEM_STATIC U16 MEM_read16(const void* ptr) { return *(const unalign16*)ptr; } +MEM_STATIC U32 MEM_read32(const void* ptr) { return *(const unalign32*)ptr; } +MEM_STATIC U64 MEM_read64(const void* ptr) { return *(const unalign64*)ptr; } +MEM_STATIC size_t MEM_readST(const void* ptr) { return *(const unalignArch*)ptr; } + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(unalign16*)memPtr = value; } +MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(unalign32*)memPtr = value; } +MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(unalign64*)memPtr = value; } + +#else + +/* default method, safe and standard. + can sometimes prove slower */ + +MEM_STATIC U16 MEM_read16(const void* memPtr) +{ + U16 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC U32 MEM_read32(const void* memPtr) +{ + U32 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC U64 MEM_read64(const void* memPtr) +{ + U64 val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC size_t MEM_readST(const void* memPtr) +{ + size_t val; ZSTD_memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) +{ + ZSTD_memcpy(memPtr, &value, sizeof(value)); +} + +MEM_STATIC void MEM_write32(void* memPtr, U32 value) +{ + ZSTD_memcpy(memPtr, &value, sizeof(value)); +} + +MEM_STATIC void MEM_write64(void* memPtr, U64 value) +{ + ZSTD_memcpy(memPtr, &value, sizeof(value)); +} + +#endif /* MEM_FORCE_MEMORY_ACCESS */ + +MEM_STATIC U32 MEM_swap32_fallback(U32 in) +{ + return ((in << 24) & 0xff000000 ) | + ((in << 8) & 0x00ff0000 ) | + ((in >> 8) & 0x0000ff00 ) | + ((in >> 24) & 0x000000ff ); +} + +MEM_STATIC U32 MEM_swap32(U32 in) +{ +#if defined(_MSC_VER) /* Visual Studio */ + return _byteswap_ulong(in); +#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \ + || (defined(__clang__) && __has_builtin(__builtin_bswap32)) + return __builtin_bswap32(in); +#else + return MEM_swap32_fallback(in); +#endif +} + +MEM_STATIC U64 MEM_swap64_fallback(U64 in) +{ + return ((in << 56) & 0xff00000000000000ULL) | + ((in << 40) & 0x00ff000000000000ULL) | + ((in << 24) & 0x0000ff0000000000ULL) | + ((in << 8) & 0x000000ff00000000ULL) | + ((in >> 8) & 0x00000000ff000000ULL) | + ((in >> 24) & 0x0000000000ff0000ULL) | + ((in >> 40) & 0x000000000000ff00ULL) | + ((in >> 56) & 0x00000000000000ffULL); +} + +MEM_STATIC U64 MEM_swap64(U64 in) +{ +#if defined(_MSC_VER) /* Visual Studio */ + return _byteswap_uint64(in); +#elif (defined (__GNUC__) && (__GNUC__ * 100 + __GNUC_MINOR__ >= 403)) \ + || (defined(__clang__) && __has_builtin(__builtin_bswap64)) + return __builtin_bswap64(in); +#else + return MEM_swap64_fallback(in); +#endif +} + +MEM_STATIC size_t MEM_swapST(size_t in) +{ + if (MEM_32bits()) + return (size_t)MEM_swap32((U32)in); + else + return (size_t)MEM_swap64((U64)in); +} + +/*=== Little endian r/w ===*/ + +MEM_STATIC U16 MEM_readLE16(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read16(memPtr); + else { + const BYTE* p = (const BYTE*)memPtr; + return (U16)(p[0] + (p[1]<<8)); + } +} + +MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val) +{ + if (MEM_isLittleEndian()) { + MEM_write16(memPtr, val); + } else { + BYTE* p = (BYTE*)memPtr; + p[0] = (BYTE)val; + p[1] = (BYTE)(val>>8); + } +} + +MEM_STATIC U32 MEM_readLE24(const void* memPtr) +{ + return (U32)MEM_readLE16(memPtr) + ((U32)(((const BYTE*)memPtr)[2]) << 16); +} + +MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val) +{ + MEM_writeLE16(memPtr, (U16)val); + ((BYTE*)memPtr)[2] = (BYTE)(val>>16); +} + +MEM_STATIC U32 MEM_readLE32(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read32(memPtr); + else + return MEM_swap32(MEM_read32(memPtr)); +} + +MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32) +{ + if (MEM_isLittleEndian()) + MEM_write32(memPtr, val32); + else + MEM_write32(memPtr, MEM_swap32(val32)); +} + +MEM_STATIC U64 MEM_readLE64(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read64(memPtr); + else + return MEM_swap64(MEM_read64(memPtr)); +} + +MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64) +{ + if (MEM_isLittleEndian()) + MEM_write64(memPtr, val64); + else + MEM_write64(memPtr, MEM_swap64(val64)); +} + +MEM_STATIC size_t MEM_readLEST(const void* memPtr) +{ + if (MEM_32bits()) + return (size_t)MEM_readLE32(memPtr); + else + return (size_t)MEM_readLE64(memPtr); +} + +MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val) +{ + if (MEM_32bits()) + MEM_writeLE32(memPtr, (U32)val); + else + MEM_writeLE64(memPtr, (U64)val); +} + +/*=== Big endian r/w ===*/ + +MEM_STATIC U32 MEM_readBE32(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_swap32(MEM_read32(memPtr)); + else + return MEM_read32(memPtr); +} + +MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32) +{ + if (MEM_isLittleEndian()) + MEM_write32(memPtr, MEM_swap32(val32)); + else + MEM_write32(memPtr, val32); +} + +MEM_STATIC U64 MEM_readBE64(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_swap64(MEM_read64(memPtr)); + else + return MEM_read64(memPtr); +} + +MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64) +{ + if (MEM_isLittleEndian()) + MEM_write64(memPtr, MEM_swap64(val64)); + else + MEM_write64(memPtr, val64); +} + +MEM_STATIC size_t MEM_readBEST(const void* memPtr) +{ + if (MEM_32bits()) + return (size_t)MEM_readBE32(memPtr); + else + return (size_t)MEM_readBE64(memPtr); +} + +MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val) +{ + if (MEM_32bits()) + MEM_writeBE32(memPtr, (U32)val); + else + MEM_writeBE64(memPtr, (U64)val); +} + +/* code only tested on 32 and 64 bits systems */ +MEM_STATIC void MEM_check(void) { DEBUG_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); } + + +#if defined (__cplusplus) +} +#endif + +#endif /* MEM_H_MODULE */ diff --git a/externals/zstd/lib/common/pool.c b/externals/zstd/lib/common/pool.c new file mode 100644 index 0000000..3adcefc --- /dev/null +++ b/externals/zstd/lib/common/pool.c @@ -0,0 +1,371 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/* ====== Dependencies ======= */ +#include "../common/allocations.h" /* ZSTD_customCalloc, ZSTD_customFree */ +#include "zstd_deps.h" /* size_t */ +#include "debug.h" /* assert */ +#include "pool.h" + +/* ====== Compiler specifics ====== */ +#if defined(_MSC_VER) +# pragma warning(disable : 4204) /* disable: C4204: non-constant aggregate initializer */ +#endif + + +#ifdef ZSTD_MULTITHREAD + +#include "threading.h" /* pthread adaptation */ + +/* A job is a function and an opaque argument */ +typedef struct POOL_job_s { + POOL_function function; + void *opaque; +} POOL_job; + +struct POOL_ctx_s { + ZSTD_customMem customMem; + /* Keep track of the threads */ + ZSTD_pthread_t* threads; + size_t threadCapacity; + size_t threadLimit; + + /* The queue is a circular buffer */ + POOL_job *queue; + size_t queueHead; + size_t queueTail; + size_t queueSize; + + /* The number of threads working on jobs */ + size_t numThreadsBusy; + /* Indicates if the queue is empty */ + int queueEmpty; + + /* The mutex protects the queue */ + ZSTD_pthread_mutex_t queueMutex; + /* Condition variable for pushers to wait on when the queue is full */ + ZSTD_pthread_cond_t queuePushCond; + /* Condition variables for poppers to wait on when the queue is empty */ + ZSTD_pthread_cond_t queuePopCond; + /* Indicates if the queue is shutting down */ + int shutdown; +}; + +/* POOL_thread() : + * Work thread for the thread pool. + * Waits for jobs and executes them. + * @returns : NULL on failure else non-null. + */ +static void* POOL_thread(void* opaque) { + POOL_ctx* const ctx = (POOL_ctx*)opaque; + if (!ctx) { return NULL; } + for (;;) { + /* Lock the mutex and wait for a non-empty queue or until shutdown */ + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + + while ( ctx->queueEmpty + || (ctx->numThreadsBusy >= ctx->threadLimit) ) { + if (ctx->shutdown) { + /* even if !queueEmpty, (possible if numThreadsBusy >= threadLimit), + * a few threads will be shutdown while !queueEmpty, + * but enough threads will remain active to finish the queue */ + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + return opaque; + } + ZSTD_pthread_cond_wait(&ctx->queuePopCond, &ctx->queueMutex); + } + /* Pop a job off the queue */ + { POOL_job const job = ctx->queue[ctx->queueHead]; + ctx->queueHead = (ctx->queueHead + 1) % ctx->queueSize; + ctx->numThreadsBusy++; + ctx->queueEmpty = (ctx->queueHead == ctx->queueTail); + /* Unlock the mutex, signal a pusher, and run the job */ + ZSTD_pthread_cond_signal(&ctx->queuePushCond); + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + + job.function(job.opaque); + + /* If the intended queue size was 0, signal after finishing job */ + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + ctx->numThreadsBusy--; + ZSTD_pthread_cond_signal(&ctx->queuePushCond); + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + } + } /* for (;;) */ + assert(0); /* Unreachable */ +} + +/* ZSTD_createThreadPool() : public access point */ +POOL_ctx* ZSTD_createThreadPool(size_t numThreads) { + return POOL_create (numThreads, 0); +} + +POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) { + return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem); +} + +POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, + ZSTD_customMem customMem) +{ + POOL_ctx* ctx; + /* Check parameters */ + if (!numThreads) { return NULL; } + /* Allocate the context and zero initialize */ + ctx = (POOL_ctx*)ZSTD_customCalloc(sizeof(POOL_ctx), customMem); + if (!ctx) { return NULL; } + /* Initialize the job queue. + * It needs one extra space since one space is wasted to differentiate + * empty and full queues. + */ + ctx->queueSize = queueSize + 1; + ctx->queue = (POOL_job*)ZSTD_customCalloc(ctx->queueSize * sizeof(POOL_job), customMem); + ctx->queueHead = 0; + ctx->queueTail = 0; + ctx->numThreadsBusy = 0; + ctx->queueEmpty = 1; + { + int error = 0; + error |= ZSTD_pthread_mutex_init(&ctx->queueMutex, NULL); + error |= ZSTD_pthread_cond_init(&ctx->queuePushCond, NULL); + error |= ZSTD_pthread_cond_init(&ctx->queuePopCond, NULL); + if (error) { POOL_free(ctx); return NULL; } + } + ctx->shutdown = 0; + /* Allocate space for the thread handles */ + ctx->threads = (ZSTD_pthread_t*)ZSTD_customCalloc(numThreads * sizeof(ZSTD_pthread_t), customMem); + ctx->threadCapacity = 0; + ctx->customMem = customMem; + /* Check for errors */ + if (!ctx->threads || !ctx->queue) { POOL_free(ctx); return NULL; } + /* Initialize the threads */ + { size_t i; + for (i = 0; i < numThreads; ++i) { + if (ZSTD_pthread_create(&ctx->threads[i], NULL, &POOL_thread, ctx)) { + ctx->threadCapacity = i; + POOL_free(ctx); + return NULL; + } } + ctx->threadCapacity = numThreads; + ctx->threadLimit = numThreads; + } + return ctx; +} + +/*! POOL_join() : + Shutdown the queue, wake any sleeping threads, and join all of the threads. +*/ +static void POOL_join(POOL_ctx* ctx) { + /* Shut down the queue */ + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + ctx->shutdown = 1; + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + /* Wake up sleeping threads */ + ZSTD_pthread_cond_broadcast(&ctx->queuePushCond); + ZSTD_pthread_cond_broadcast(&ctx->queuePopCond); + /* Join all of the threads */ + { size_t i; + for (i = 0; i < ctx->threadCapacity; ++i) { + ZSTD_pthread_join(ctx->threads[i]); /* note : could fail */ + } } +} + +void POOL_free(POOL_ctx *ctx) { + if (!ctx) { return; } + POOL_join(ctx); + ZSTD_pthread_mutex_destroy(&ctx->queueMutex); + ZSTD_pthread_cond_destroy(&ctx->queuePushCond); + ZSTD_pthread_cond_destroy(&ctx->queuePopCond); + ZSTD_customFree(ctx->queue, ctx->customMem); + ZSTD_customFree(ctx->threads, ctx->customMem); + ZSTD_customFree(ctx, ctx->customMem); +} + +/*! POOL_joinJobs() : + * Waits for all queued jobs to finish executing. + */ +void POOL_joinJobs(POOL_ctx* ctx) { + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + while(!ctx->queueEmpty || ctx->numThreadsBusy > 0) { + ZSTD_pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex); + } + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); +} + +void ZSTD_freeThreadPool (ZSTD_threadPool* pool) { + POOL_free (pool); +} + +size_t POOL_sizeof(const POOL_ctx* ctx) { + if (ctx==NULL) return 0; /* supports sizeof NULL */ + return sizeof(*ctx) + + ctx->queueSize * sizeof(POOL_job) + + ctx->threadCapacity * sizeof(ZSTD_pthread_t); +} + + +/* @return : 0 on success, 1 on error */ +static int POOL_resize_internal(POOL_ctx* ctx, size_t numThreads) +{ + if (numThreads <= ctx->threadCapacity) { + if (!numThreads) return 1; + ctx->threadLimit = numThreads; + return 0; + } + /* numThreads > threadCapacity */ + { ZSTD_pthread_t* const threadPool = (ZSTD_pthread_t*)ZSTD_customCalloc(numThreads * sizeof(ZSTD_pthread_t), ctx->customMem); + if (!threadPool) return 1; + /* replace existing thread pool */ + ZSTD_memcpy(threadPool, ctx->threads, ctx->threadCapacity * sizeof(ZSTD_pthread_t)); + ZSTD_customFree(ctx->threads, ctx->customMem); + ctx->threads = threadPool; + /* Initialize additional threads */ + { size_t threadId; + for (threadId = ctx->threadCapacity; threadId < numThreads; ++threadId) { + if (ZSTD_pthread_create(&threadPool[threadId], NULL, &POOL_thread, ctx)) { + ctx->threadCapacity = threadId; + return 1; + } } + } } + /* successfully expanded */ + ctx->threadCapacity = numThreads; + ctx->threadLimit = numThreads; + return 0; +} + +/* @return : 0 on success, 1 on error */ +int POOL_resize(POOL_ctx* ctx, size_t numThreads) +{ + int result; + if (ctx==NULL) return 1; + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + result = POOL_resize_internal(ctx, numThreads); + ZSTD_pthread_cond_broadcast(&ctx->queuePopCond); + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + return result; +} + +/** + * Returns 1 if the queue is full and 0 otherwise. + * + * When queueSize is 1 (pool was created with an intended queueSize of 0), + * then a queue is empty if there is a thread free _and_ no job is waiting. + */ +static int isQueueFull(POOL_ctx const* ctx) { + if (ctx->queueSize > 1) { + return ctx->queueHead == ((ctx->queueTail + 1) % ctx->queueSize); + } else { + return (ctx->numThreadsBusy == ctx->threadLimit) || + !ctx->queueEmpty; + } +} + + +static void +POOL_add_internal(POOL_ctx* ctx, POOL_function function, void *opaque) +{ + POOL_job job; + job.function = function; + job.opaque = opaque; + assert(ctx != NULL); + if (ctx->shutdown) return; + + ctx->queueEmpty = 0; + ctx->queue[ctx->queueTail] = job; + ctx->queueTail = (ctx->queueTail + 1) % ctx->queueSize; + ZSTD_pthread_cond_signal(&ctx->queuePopCond); +} + +void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque) +{ + assert(ctx != NULL); + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + /* Wait until there is space in the queue for the new job */ + while (isQueueFull(ctx) && (!ctx->shutdown)) { + ZSTD_pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex); + } + POOL_add_internal(ctx, function, opaque); + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); +} + + +int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque) +{ + assert(ctx != NULL); + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + if (isQueueFull(ctx)) { + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + return 0; + } + POOL_add_internal(ctx, function, opaque); + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); + return 1; +} + + +#else /* ZSTD_MULTITHREAD not defined */ + +/* ========================== */ +/* No multi-threading support */ +/* ========================== */ + + +/* We don't need any data, but if it is empty, malloc() might return NULL. */ +struct POOL_ctx_s { + int dummy; +}; +static POOL_ctx g_poolCtx; + +POOL_ctx* POOL_create(size_t numThreads, size_t queueSize) { + return POOL_create_advanced(numThreads, queueSize, ZSTD_defaultCMem); +} + +POOL_ctx* +POOL_create_advanced(size_t numThreads, size_t queueSize, ZSTD_customMem customMem) +{ + (void)numThreads; + (void)queueSize; + (void)customMem; + return &g_poolCtx; +} + +void POOL_free(POOL_ctx* ctx) { + assert(!ctx || ctx == &g_poolCtx); + (void)ctx; +} + +void POOL_joinJobs(POOL_ctx* ctx){ + assert(!ctx || ctx == &g_poolCtx); + (void)ctx; +} + +int POOL_resize(POOL_ctx* ctx, size_t numThreads) { + (void)ctx; (void)numThreads; + return 0; +} + +void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque) { + (void)ctx; + function(opaque); +} + +int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque) { + (void)ctx; + function(opaque); + return 1; +} + +size_t POOL_sizeof(const POOL_ctx* ctx) { + if (ctx==NULL) return 0; /* supports sizeof NULL */ + assert(ctx == &g_poolCtx); + return sizeof(*ctx); +} + +#endif /* ZSTD_MULTITHREAD */ diff --git a/externals/zstd/lib/common/pool.h b/externals/zstd/lib/common/pool.h new file mode 100644 index 0000000..cca4de7 --- /dev/null +++ b/externals/zstd/lib/common/pool.h @@ -0,0 +1,90 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef POOL_H +#define POOL_H + +#if defined (__cplusplus) +extern "C" { +#endif + + +#include "zstd_deps.h" +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_customMem */ +#include "../zstd.h" + +typedef struct POOL_ctx_s POOL_ctx; + +/*! POOL_create() : + * Create a thread pool with at most `numThreads` threads. + * `numThreads` must be at least 1. + * The maximum number of queued jobs before blocking is `queueSize`. + * @return : POOL_ctx pointer on success, else NULL. +*/ +POOL_ctx* POOL_create(size_t numThreads, size_t queueSize); + +POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, + ZSTD_customMem customMem); + +/*! POOL_free() : + * Free a thread pool returned by POOL_create(). + */ +void POOL_free(POOL_ctx* ctx); + + +/*! POOL_joinJobs() : + * Waits for all queued jobs to finish executing. + */ +void POOL_joinJobs(POOL_ctx* ctx); + +/*! POOL_resize() : + * Expands or shrinks pool's number of threads. + * This is more efficient than releasing + creating a new context, + * since it tries to preserve and reuse existing threads. + * `numThreads` must be at least 1. + * @return : 0 when resize was successful, + * !0 (typically 1) if there is an error. + * note : only numThreads can be resized, queueSize remains unchanged. + */ +int POOL_resize(POOL_ctx* ctx, size_t numThreads); + +/*! POOL_sizeof() : + * @return threadpool memory usage + * note : compatible with NULL (returns 0 in this case) + */ +size_t POOL_sizeof(const POOL_ctx* ctx); + +/*! POOL_function : + * The function type that can be added to a thread pool. + */ +typedef void (*POOL_function)(void*); + +/*! POOL_add() : + * Add the job `function(opaque)` to the thread pool. `ctx` must be valid. + * Possibly blocks until there is room in the queue. + * Note : The function may be executed asynchronously, + * therefore, `opaque` must live until function has been completed. + */ +void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque); + + +/*! POOL_tryAdd() : + * Add the job `function(opaque)` to thread pool _if_ a queue slot is available. + * Returns immediately even if not (does not block). + * @return : 1 if successful, 0 if not. + */ +int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque); + + +#if defined (__cplusplus) +} +#endif + +#endif diff --git a/externals/zstd/lib/common/portability_macros.h b/externals/zstd/lib/common/portability_macros.h new file mode 100644 index 0000000..e50314a --- /dev/null +++ b/externals/zstd/lib/common/portability_macros.h @@ -0,0 +1,158 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_PORTABILITY_MACROS_H +#define ZSTD_PORTABILITY_MACROS_H + +/** + * This header file contains macro definitions to support portability. + * This header is shared between C and ASM code, so it MUST only + * contain macro definitions. It MUST not contain any C code. + * + * This header ONLY defines macros to detect platforms/feature support. + * + */ + + +/* compat. with non-clang compilers */ +#ifndef __has_attribute + #define __has_attribute(x) 0 +#endif + +/* compat. with non-clang compilers */ +#ifndef __has_builtin +# define __has_builtin(x) 0 +#endif + +/* compat. with non-clang compilers */ +#ifndef __has_feature +# define __has_feature(x) 0 +#endif + +/* detects whether we are being compiled under msan */ +#ifndef ZSTD_MEMORY_SANITIZER +# if __has_feature(memory_sanitizer) +# define ZSTD_MEMORY_SANITIZER 1 +# else +# define ZSTD_MEMORY_SANITIZER 0 +# endif +#endif + +/* detects whether we are being compiled under asan */ +#ifndef ZSTD_ADDRESS_SANITIZER +# if __has_feature(address_sanitizer) +# define ZSTD_ADDRESS_SANITIZER 1 +# elif defined(__SANITIZE_ADDRESS__) +# define ZSTD_ADDRESS_SANITIZER 1 +# else +# define ZSTD_ADDRESS_SANITIZER 0 +# endif +#endif + +/* detects whether we are being compiled under dfsan */ +#ifndef ZSTD_DATAFLOW_SANITIZER +# if __has_feature(dataflow_sanitizer) +# define ZSTD_DATAFLOW_SANITIZER 1 +# else +# define ZSTD_DATAFLOW_SANITIZER 0 +# endif +#endif + +/* Mark the internal assembly functions as hidden */ +#ifdef __ELF__ +# define ZSTD_HIDE_ASM_FUNCTION(func) .hidden func +#elif defined(__APPLE__) +# define ZSTD_HIDE_ASM_FUNCTION(func) .private_extern func +#else +# define ZSTD_HIDE_ASM_FUNCTION(func) +#endif + +/* Enable runtime BMI2 dispatch based on the CPU. + * Enabled for clang & gcc >=4.8 on x86 when BMI2 isn't enabled by default. + */ +#ifndef DYNAMIC_BMI2 + #if ((defined(__clang__) && __has_attribute(__target__)) \ + || (defined(__GNUC__) \ + && (__GNUC__ >= 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8)))) \ + && (defined(__x86_64__) || defined(_M_X64)) \ + && !defined(__BMI2__) + # define DYNAMIC_BMI2 1 + #else + # define DYNAMIC_BMI2 0 + #endif +#endif + +/** + * Only enable assembly for GNUC compatible compilers, + * because other platforms may not support GAS assembly syntax. + * + * Only enable assembly for Linux / MacOS, other platforms may + * work, but they haven't been tested. This could likely be + * extended to BSD systems. + * + * Disable assembly when MSAN is enabled, because MSAN requires + * 100% of code to be instrumented to work. + */ +#if defined(__GNUC__) +# if defined(__linux__) || defined(__linux) || defined(__APPLE__) +# if ZSTD_MEMORY_SANITIZER +# define ZSTD_ASM_SUPPORTED 0 +# elif ZSTD_DATAFLOW_SANITIZER +# define ZSTD_ASM_SUPPORTED 0 +# else +# define ZSTD_ASM_SUPPORTED 1 +# endif +# else +# define ZSTD_ASM_SUPPORTED 0 +# endif +#else +# define ZSTD_ASM_SUPPORTED 0 +#endif + +/** + * Determines whether we should enable assembly for x86-64 + * with BMI2. + * + * Enable if all of the following conditions hold: + * - ASM hasn't been explicitly disabled by defining ZSTD_DISABLE_ASM + * - Assembly is supported + * - We are compiling for x86-64 and either: + * - DYNAMIC_BMI2 is enabled + * - BMI2 is supported at compile time + */ +#if !defined(ZSTD_DISABLE_ASM) && \ + ZSTD_ASM_SUPPORTED && \ + defined(__x86_64__) && \ + (DYNAMIC_BMI2 || defined(__BMI2__)) +# define ZSTD_ENABLE_ASM_X86_64_BMI2 1 +#else +# define ZSTD_ENABLE_ASM_X86_64_BMI2 0 +#endif + +/* + * For x86 ELF targets, add .note.gnu.property section for Intel CET in + * assembly sources when CET is enabled. + * + * Additionally, any function that may be called indirectly must begin + * with ZSTD_CET_ENDBRANCH. + */ +#if defined(__ELF__) && (defined(__x86_64__) || defined(__i386__)) \ + && defined(__has_include) +# if __has_include() +# include +# define ZSTD_CET_ENDBRANCH _CET_ENDBR +# endif +#endif + +#ifndef ZSTD_CET_ENDBRANCH +# define ZSTD_CET_ENDBRANCH +#endif + +#endif /* ZSTD_PORTABILITY_MACROS_H */ diff --git a/externals/zstd/lib/common/threading.c b/externals/zstd/lib/common/threading.c new file mode 100644 index 0000000..25bb8b9 --- /dev/null +++ b/externals/zstd/lib/common/threading.c @@ -0,0 +1,182 @@ +/** + * Copyright (c) 2016 Tino Reichardt + * All rights reserved. + * + * You can contact the author at: + * - zstdmt source repository: https://github.com/mcmilk/zstdmt + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/** + * This file will hold wrapper for systems, which do not support pthreads + */ + +#include "threading.h" + +/* create fake symbol to avoid empty translation unit warning */ +int g_ZSTD_threading_useless_symbol; + +#if defined(ZSTD_MULTITHREAD) && defined(_WIN32) + +/** + * Windows minimalist Pthread Wrapper + */ + + +/* === Dependencies === */ +#include +#include + + +/* === Implementation === */ + +typedef struct { + void* (*start_routine)(void*); + void* arg; + int initialized; + ZSTD_pthread_cond_t initialized_cond; + ZSTD_pthread_mutex_t initialized_mutex; +} ZSTD_thread_params_t; + +static unsigned __stdcall worker(void *arg) +{ + void* (*start_routine)(void*); + void* thread_arg; + + /* Initialized thread_arg and start_routine and signal main thread that we don't need it + * to wait any longer. + */ + { + ZSTD_thread_params_t* thread_param = (ZSTD_thread_params_t*)arg; + thread_arg = thread_param->arg; + start_routine = thread_param->start_routine; + + /* Signal main thread that we are running and do not depend on its memory anymore */ + ZSTD_pthread_mutex_lock(&thread_param->initialized_mutex); + thread_param->initialized = 1; + ZSTD_pthread_cond_signal(&thread_param->initialized_cond); + ZSTD_pthread_mutex_unlock(&thread_param->initialized_mutex); + } + + start_routine(thread_arg); + + return 0; +} + +int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused, + void* (*start_routine) (void*), void* arg) +{ + ZSTD_thread_params_t thread_param; + (void)unused; + + if (thread==NULL) return -1; + *thread = NULL; + + thread_param.start_routine = start_routine; + thread_param.arg = arg; + thread_param.initialized = 0; + + /* Setup thread initialization synchronization */ + if(ZSTD_pthread_cond_init(&thread_param.initialized_cond, NULL)) { + /* Should never happen on Windows */ + return -1; + } + if(ZSTD_pthread_mutex_init(&thread_param.initialized_mutex, NULL)) { + /* Should never happen on Windows */ + ZSTD_pthread_cond_destroy(&thread_param.initialized_cond); + return -1; + } + + /* Spawn thread */ + *thread = (HANDLE)_beginthreadex(NULL, 0, worker, &thread_param, 0, NULL); + if (*thread==NULL) { + ZSTD_pthread_mutex_destroy(&thread_param.initialized_mutex); + ZSTD_pthread_cond_destroy(&thread_param.initialized_cond); + return errno; + } + + /* Wait for thread to be initialized */ + ZSTD_pthread_mutex_lock(&thread_param.initialized_mutex); + while(!thread_param.initialized) { + ZSTD_pthread_cond_wait(&thread_param.initialized_cond, &thread_param.initialized_mutex); + } + ZSTD_pthread_mutex_unlock(&thread_param.initialized_mutex); + ZSTD_pthread_mutex_destroy(&thread_param.initialized_mutex); + ZSTD_pthread_cond_destroy(&thread_param.initialized_cond); + + return 0; +} + +int ZSTD_pthread_join(ZSTD_pthread_t thread) +{ + DWORD result; + + if (!thread) return 0; + + result = WaitForSingleObject(thread, INFINITE); + CloseHandle(thread); + + switch (result) { + case WAIT_OBJECT_0: + return 0; + case WAIT_ABANDONED: + return EINVAL; + default: + return GetLastError(); + } +} + +#endif /* ZSTD_MULTITHREAD */ + +#if defined(ZSTD_MULTITHREAD) && DEBUGLEVEL >= 1 && !defined(_WIN32) + +#define ZSTD_DEPS_NEED_MALLOC +#include "zstd_deps.h" + +int ZSTD_pthread_mutex_init(ZSTD_pthread_mutex_t* mutex, pthread_mutexattr_t const* attr) +{ + assert(mutex != NULL); + *mutex = (pthread_mutex_t*)ZSTD_malloc(sizeof(pthread_mutex_t)); + if (!*mutex) + return 1; + return pthread_mutex_init(*mutex, attr); +} + +int ZSTD_pthread_mutex_destroy(ZSTD_pthread_mutex_t* mutex) +{ + assert(mutex != NULL); + if (!*mutex) + return 0; + { + int const ret = pthread_mutex_destroy(*mutex); + ZSTD_free(*mutex); + return ret; + } +} + +int ZSTD_pthread_cond_init(ZSTD_pthread_cond_t* cond, pthread_condattr_t const* attr) +{ + assert(cond != NULL); + *cond = (pthread_cond_t*)ZSTD_malloc(sizeof(pthread_cond_t)); + if (!*cond) + return 1; + return pthread_cond_init(*cond, attr); +} + +int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond) +{ + assert(cond != NULL); + if (!*cond) + return 0; + { + int const ret = pthread_cond_destroy(*cond); + ZSTD_free(*cond); + return ret; + } +} + +#endif diff --git a/externals/zstd/lib/common/threading.h b/externals/zstd/lib/common/threading.h new file mode 100644 index 0000000..fb5c1c8 --- /dev/null +++ b/externals/zstd/lib/common/threading.h @@ -0,0 +1,150 @@ +/** + * Copyright (c) 2016 Tino Reichardt + * All rights reserved. + * + * You can contact the author at: + * - zstdmt source repository: https://github.com/mcmilk/zstdmt + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef THREADING_H_938743 +#define THREADING_H_938743 + +#include "debug.h" + +#if defined (__cplusplus) +extern "C" { +#endif + +#if defined(ZSTD_MULTITHREAD) && defined(_WIN32) + +/** + * Windows minimalist Pthread Wrapper + */ +#ifdef WINVER +# undef WINVER +#endif +#define WINVER 0x0600 + +#ifdef _WIN32_WINNT +# undef _WIN32_WINNT +#endif +#define _WIN32_WINNT 0x0600 + +#ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +#endif + +#undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */ +#include +#undef ERROR +#define ERROR(name) ZSTD_ERROR(name) + + +/* mutex */ +#define ZSTD_pthread_mutex_t CRITICAL_SECTION +#define ZSTD_pthread_mutex_init(a, b) ((void)(b), InitializeCriticalSection((a)), 0) +#define ZSTD_pthread_mutex_destroy(a) DeleteCriticalSection((a)) +#define ZSTD_pthread_mutex_lock(a) EnterCriticalSection((a)) +#define ZSTD_pthread_mutex_unlock(a) LeaveCriticalSection((a)) + +/* condition variable */ +#define ZSTD_pthread_cond_t CONDITION_VARIABLE +#define ZSTD_pthread_cond_init(a, b) ((void)(b), InitializeConditionVariable((a)), 0) +#define ZSTD_pthread_cond_destroy(a) ((void)(a)) +#define ZSTD_pthread_cond_wait(a, b) SleepConditionVariableCS((a), (b), INFINITE) +#define ZSTD_pthread_cond_signal(a) WakeConditionVariable((a)) +#define ZSTD_pthread_cond_broadcast(a) WakeAllConditionVariable((a)) + +/* ZSTD_pthread_create() and ZSTD_pthread_join() */ +typedef HANDLE ZSTD_pthread_t; + +int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused, + void* (*start_routine) (void*), void* arg); + +int ZSTD_pthread_join(ZSTD_pthread_t thread); + +/** + * add here more wrappers as required + */ + + +#elif defined(ZSTD_MULTITHREAD) /* posix assumed ; need a better detection method */ +/* === POSIX Systems === */ +# include + +#if DEBUGLEVEL < 1 + +#define ZSTD_pthread_mutex_t pthread_mutex_t +#define ZSTD_pthread_mutex_init(a, b) pthread_mutex_init((a), (b)) +#define ZSTD_pthread_mutex_destroy(a) pthread_mutex_destroy((a)) +#define ZSTD_pthread_mutex_lock(a) pthread_mutex_lock((a)) +#define ZSTD_pthread_mutex_unlock(a) pthread_mutex_unlock((a)) + +#define ZSTD_pthread_cond_t pthread_cond_t +#define ZSTD_pthread_cond_init(a, b) pthread_cond_init((a), (b)) +#define ZSTD_pthread_cond_destroy(a) pthread_cond_destroy((a)) +#define ZSTD_pthread_cond_wait(a, b) pthread_cond_wait((a), (b)) +#define ZSTD_pthread_cond_signal(a) pthread_cond_signal((a)) +#define ZSTD_pthread_cond_broadcast(a) pthread_cond_broadcast((a)) + +#define ZSTD_pthread_t pthread_t +#define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d)) +#define ZSTD_pthread_join(a) pthread_join((a),NULL) + +#else /* DEBUGLEVEL >= 1 */ + +/* Debug implementation of threading. + * In this implementation we use pointers for mutexes and condition variables. + * This way, if we forget to init/destroy them the program will crash or ASAN + * will report leaks. + */ + +#define ZSTD_pthread_mutex_t pthread_mutex_t* +int ZSTD_pthread_mutex_init(ZSTD_pthread_mutex_t* mutex, pthread_mutexattr_t const* attr); +int ZSTD_pthread_mutex_destroy(ZSTD_pthread_mutex_t* mutex); +#define ZSTD_pthread_mutex_lock(a) pthread_mutex_lock(*(a)) +#define ZSTD_pthread_mutex_unlock(a) pthread_mutex_unlock(*(a)) + +#define ZSTD_pthread_cond_t pthread_cond_t* +int ZSTD_pthread_cond_init(ZSTD_pthread_cond_t* cond, pthread_condattr_t const* attr); +int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond); +#define ZSTD_pthread_cond_wait(a, b) pthread_cond_wait(*(a), *(b)) +#define ZSTD_pthread_cond_signal(a) pthread_cond_signal(*(a)) +#define ZSTD_pthread_cond_broadcast(a) pthread_cond_broadcast(*(a)) + +#define ZSTD_pthread_t pthread_t +#define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d)) +#define ZSTD_pthread_join(a) pthread_join((a),NULL) + +#endif + +#else /* ZSTD_MULTITHREAD not defined */ +/* No multithreading support */ + +typedef int ZSTD_pthread_mutex_t; +#define ZSTD_pthread_mutex_init(a, b) ((void)(a), (void)(b), 0) +#define ZSTD_pthread_mutex_destroy(a) ((void)(a)) +#define ZSTD_pthread_mutex_lock(a) ((void)(a)) +#define ZSTD_pthread_mutex_unlock(a) ((void)(a)) + +typedef int ZSTD_pthread_cond_t; +#define ZSTD_pthread_cond_init(a, b) ((void)(a), (void)(b), 0) +#define ZSTD_pthread_cond_destroy(a) ((void)(a)) +#define ZSTD_pthread_cond_wait(a, b) ((void)(a), (void)(b)) +#define ZSTD_pthread_cond_signal(a) ((void)(a)) +#define ZSTD_pthread_cond_broadcast(a) ((void)(a)) + +/* do not use ZSTD_pthread_t */ + +#endif /* ZSTD_MULTITHREAD */ + +#if defined (__cplusplus) +} +#endif + +#endif /* THREADING_H_938743 */ diff --git a/externals/zstd/lib/common/xxhash.c b/externals/zstd/lib/common/xxhash.c new file mode 100644 index 0000000..052cd52 --- /dev/null +++ b/externals/zstd/lib/common/xxhash.c @@ -0,0 +1,18 @@ +/* + * xxHash - Extremely Fast Hash algorithm + * Copyright (c) Yann Collet - Meta Platforms, Inc + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* + * xxhash.c instantiates functions defined in xxhash.h + */ + +#define XXH_STATIC_LINKING_ONLY /* access advanced declarations */ +#define XXH_IMPLEMENTATION /* access definitions */ + +#include "xxhash.h" diff --git a/externals/zstd/lib/common/xxhash.h b/externals/zstd/lib/common/xxhash.h new file mode 100644 index 0000000..e59e442 --- /dev/null +++ b/externals/zstd/lib/common/xxhash.h @@ -0,0 +1,7020 @@ +/* + * xxHash - Extremely Fast Hash algorithm + * Header File + * Copyright (c) Yann Collet - Meta Platforms, Inc + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* Local adaptations for Zstandard */ + +#ifndef XXH_NO_XXH3 +# define XXH_NO_XXH3 +#endif + +#ifndef XXH_NAMESPACE +# define XXH_NAMESPACE ZSTD_ +#endif + +/*! + * @mainpage xxHash + * + * xxHash is an extremely fast non-cryptographic hash algorithm, working at RAM speed + * limits. + * + * It is proposed in four flavors, in three families: + * 1. @ref XXH32_family + * - Classic 32-bit hash function. Simple, compact, and runs on almost all + * 32-bit and 64-bit systems. + * 2. @ref XXH64_family + * - Classic 64-bit adaptation of XXH32. Just as simple, and runs well on most + * 64-bit systems (but _not_ 32-bit systems). + * 3. @ref XXH3_family + * - Modern 64-bit and 128-bit hash function family which features improved + * strength and performance across the board, especially on smaller data. + * It benefits greatly from SIMD and 64-bit without requiring it. + * + * Benchmarks + * --- + * The reference system uses an Intel i7-9700K CPU, and runs Ubuntu x64 20.04. + * The open source benchmark program is compiled with clang v10.0 using -O3 flag. + * + * | Hash Name | ISA ext | Width | Large Data Speed | Small Data Velocity | + * | -------------------- | ------- | ----: | ---------------: | ------------------: | + * | XXH3_64bits() | @b AVX2 | 64 | 59.4 GB/s | 133.1 | + * | MeowHash | AES-NI | 128 | 58.2 GB/s | 52.5 | + * | XXH3_128bits() | @b AVX2 | 128 | 57.9 GB/s | 118.1 | + * | CLHash | PCLMUL | 64 | 37.1 GB/s | 58.1 | + * | XXH3_64bits() | @b SSE2 | 64 | 31.5 GB/s | 133.1 | + * | XXH3_128bits() | @b SSE2 | 128 | 29.6 GB/s | 118.1 | + * | RAM sequential read | | N/A | 28.0 GB/s | N/A | + * | ahash | AES-NI | 64 | 22.5 GB/s | 107.2 | + * | City64 | | 64 | 22.0 GB/s | 76.6 | + * | T1ha2 | | 64 | 22.0 GB/s | 99.0 | + * | City128 | | 128 | 21.7 GB/s | 57.7 | + * | FarmHash | AES-NI | 64 | 21.3 GB/s | 71.9 | + * | XXH64() | | 64 | 19.4 GB/s | 71.0 | + * | SpookyHash | | 64 | 19.3 GB/s | 53.2 | + * | Mum | | 64 | 18.0 GB/s | 67.0 | + * | CRC32C | SSE4.2 | 32 | 13.0 GB/s | 57.9 | + * | XXH32() | | 32 | 9.7 GB/s | 71.9 | + * | City32 | | 32 | 9.1 GB/s | 66.0 | + * | Blake3* | @b AVX2 | 256 | 4.4 GB/s | 8.1 | + * | Murmur3 | | 32 | 3.9 GB/s | 56.1 | + * | SipHash* | | 64 | 3.0 GB/s | 43.2 | + * | Blake3* | @b SSE2 | 256 | 2.4 GB/s | 8.1 | + * | HighwayHash | | 64 | 1.4 GB/s | 6.0 | + * | FNV64 | | 64 | 1.2 GB/s | 62.7 | + * | Blake2* | | 256 | 1.1 GB/s | 5.1 | + * | SHA1* | | 160 | 0.8 GB/s | 5.6 | + * | MD5* | | 128 | 0.6 GB/s | 7.8 | + * @note + * - Hashes which require a specific ISA extension are noted. SSE2 is also noted, + * even though it is mandatory on x64. + * - Hashes with an asterisk are cryptographic. Note that MD5 is non-cryptographic + * by modern standards. + * - Small data velocity is a rough average of algorithm's efficiency for small + * data. For more accurate information, see the wiki. + * - More benchmarks and strength tests are found on the wiki: + * https://github.com/Cyan4973/xxHash/wiki + * + * Usage + * ------ + * All xxHash variants use a similar API. Changing the algorithm is a trivial + * substitution. + * + * @pre + * For functions which take an input and length parameter, the following + * requirements are assumed: + * - The range from [`input`, `input + length`) is valid, readable memory. + * - The only exception is if the `length` is `0`, `input` may be `NULL`. + * - For C++, the objects must have the *TriviallyCopyable* property, as the + * functions access bytes directly as if it was an array of `unsigned char`. + * + * @anchor single_shot_example + * **Single Shot** + * + * These functions are stateless functions which hash a contiguous block of memory, + * immediately returning the result. They are the easiest and usually the fastest + * option. + * + * XXH32(), XXH64(), XXH3_64bits(), XXH3_128bits() + * + * @code{.c} + * #include + * #include "xxhash.h" + * + * // Example for a function which hashes a null terminated string with XXH32(). + * XXH32_hash_t hash_string(const char* string, XXH32_hash_t seed) + * { + * // NULL pointers are only valid if the length is zero + * size_t length = (string == NULL) ? 0 : strlen(string); + * return XXH32(string, length, seed); + * } + * @endcode + * + * + * @anchor streaming_example + * **Streaming** + * + * These groups of functions allow incremental hashing of unknown size, even + * more than what would fit in a size_t. + * + * XXH32_reset(), XXH64_reset(), XXH3_64bits_reset(), XXH3_128bits_reset() + * + * @code{.c} + * #include + * #include + * #include "xxhash.h" + * // Example for a function which hashes a FILE incrementally with XXH3_64bits(). + * XXH64_hash_t hashFile(FILE* f) + * { + * // Allocate a state struct. Do not just use malloc() or new. + * XXH3_state_t* state = XXH3_createState(); + * assert(state != NULL && "Out of memory!"); + * // Reset the state to start a new hashing session. + * XXH3_64bits_reset(state); + * char buffer[4096]; + * size_t count; + * // Read the file in chunks + * while ((count = fread(buffer, 1, sizeof(buffer), f)) != 0) { + * // Run update() as many times as necessary to process the data + * XXH3_64bits_update(state, buffer, count); + * } + * // Retrieve the finalized hash. This will not change the state. + * XXH64_hash_t result = XXH3_64bits_digest(state); + * // Free the state. Do not use free(). + * XXH3_freeState(state); + * return result; + * } + * @endcode + * + * Streaming functions generate the xxHash value from an incremental input. + * This method is slower than single-call functions, due to state management. + * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized. + * + * An XXH state must first be allocated using `XXH*_createState()`. + * + * Start a new hash by initializing the state with a seed using `XXH*_reset()`. + * + * Then, feed the hash state by calling `XXH*_update()` as many times as necessary. + * + * The function returns an error code, with 0 meaning OK, and any other value + * meaning there is an error. + * + * Finally, a hash value can be produced anytime, by using `XXH*_digest()`. + * This function returns the nn-bits hash as an int or long long. + * + * It's still possible to continue inserting input into the hash state after a + * digest, and generate new hash values later on by invoking `XXH*_digest()`. + * + * When done, release the state using `XXH*_freeState()`. + * + * + * @anchor canonical_representation_example + * **Canonical Representation** + * + * The default return values from XXH functions are unsigned 32, 64 and 128 bit + * integers. + * This the simplest and fastest format for further post-processing. + * + * However, this leaves open the question of what is the order on the byte level, + * since little and big endian conventions will store the same number differently. + * + * The canonical representation settles this issue by mandating big-endian + * convention, the same convention as human-readable numbers (large digits first). + * + * When writing hash values to storage, sending them over a network, or printing + * them, it's highly recommended to use the canonical representation to ensure + * portability across a wider range of systems, present and future. + * + * The following functions allow transformation of hash values to and from + * canonical format. + * + * XXH32_canonicalFromHash(), XXH32_hashFromCanonical(), + * XXH64_canonicalFromHash(), XXH64_hashFromCanonical(), + * XXH128_canonicalFromHash(), XXH128_hashFromCanonical(), + * + * @code{.c} + * #include + * #include "xxhash.h" + * + * // Example for a function which prints XXH32_hash_t in human readable format + * void printXxh32(XXH32_hash_t hash) + * { + * XXH32_canonical_t cano; + * XXH32_canonicalFromHash(&cano, hash); + * size_t i; + * for(i = 0; i < sizeof(cano.digest); ++i) { + * printf("%02x", cano.digest[i]); + * } + * printf("\n"); + * } + * + * // Example for a function which converts XXH32_canonical_t to XXH32_hash_t + * XXH32_hash_t convertCanonicalToXxh32(XXH32_canonical_t cano) + * { + * XXH32_hash_t hash = XXH32_hashFromCanonical(&cano); + * return hash; + * } + * @endcode + * + * + * @file xxhash.h + * xxHash prototypes and implementation + */ + +#if defined (__cplusplus) +extern "C" { +#endif + +/* **************************** + * INLINE mode + ******************************/ +/*! + * @defgroup public Public API + * Contains details on the public xxHash functions. + * @{ + */ +#ifdef XXH_DOXYGEN +/*! + * @brief Gives access to internal state declaration, required for static allocation. + * + * Incompatible with dynamic linking, due to risks of ABI changes. + * + * Usage: + * @code{.c} + * #define XXH_STATIC_LINKING_ONLY + * #include "xxhash.h" + * @endcode + */ +# define XXH_STATIC_LINKING_ONLY +/* Do not undef XXH_STATIC_LINKING_ONLY for Doxygen */ + +/*! + * @brief Gives access to internal definitions. + * + * Usage: + * @code{.c} + * #define XXH_STATIC_LINKING_ONLY + * #define XXH_IMPLEMENTATION + * #include "xxhash.h" + * @endcode + */ +# define XXH_IMPLEMENTATION +/* Do not undef XXH_IMPLEMENTATION for Doxygen */ + +/*! + * @brief Exposes the implementation and marks all functions as `inline`. + * + * Use these build macros to inline xxhash into the target unit. + * Inlining improves performance on small inputs, especially when the length is + * expressed as a compile-time constant: + * + * https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html + * + * It also keeps xxHash symbols private to the unit, so they are not exported. + * + * Usage: + * @code{.c} + * #define XXH_INLINE_ALL + * #include "xxhash.h" + * @endcode + * Do not compile and link xxhash.o as a separate object, as it is not useful. + */ +# define XXH_INLINE_ALL +# undef XXH_INLINE_ALL +/*! + * @brief Exposes the implementation without marking functions as inline. + */ +# define XXH_PRIVATE_API +# undef XXH_PRIVATE_API +/*! + * @brief Emulate a namespace by transparently prefixing all symbols. + * + * If you want to include _and expose_ xxHash functions from within your own + * library, but also want to avoid symbol collisions with other libraries which + * may also include xxHash, you can use @ref XXH_NAMESPACE to automatically prefix + * any public symbol from xxhash library with the value of @ref XXH_NAMESPACE + * (therefore, avoid empty or numeric values). + * + * Note that no change is required within the calling program as long as it + * includes `xxhash.h`: Regular symbol names will be automatically translated + * by this header. + */ +# define XXH_NAMESPACE /* YOUR NAME HERE */ +# undef XXH_NAMESPACE +#endif + +#if (defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)) \ + && !defined(XXH_INLINE_ALL_31684351384) + /* this section should be traversed only once */ +# define XXH_INLINE_ALL_31684351384 + /* give access to the advanced API, required to compile implementations */ +# undef XXH_STATIC_LINKING_ONLY /* avoid macro redef */ +# define XXH_STATIC_LINKING_ONLY + /* make all functions private */ +# undef XXH_PUBLIC_API +# if defined(__GNUC__) +# define XXH_PUBLIC_API static __inline __attribute__((unused)) +# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define XXH_PUBLIC_API static inline +# elif defined(_MSC_VER) +# define XXH_PUBLIC_API static __inline +# else + /* note: this version may generate warnings for unused static functions */ +# define XXH_PUBLIC_API static +# endif + + /* + * This part deals with the special case where a unit wants to inline xxHash, + * but "xxhash.h" has previously been included without XXH_INLINE_ALL, + * such as part of some previously included *.h header file. + * Without further action, the new include would just be ignored, + * and functions would effectively _not_ be inlined (silent failure). + * The following macros solve this situation by prefixing all inlined names, + * avoiding naming collision with previous inclusions. + */ + /* Before that, we unconditionally #undef all symbols, + * in case they were already defined with XXH_NAMESPACE. + * They will then be redefined for XXH_INLINE_ALL + */ +# undef XXH_versionNumber + /* XXH32 */ +# undef XXH32 +# undef XXH32_createState +# undef XXH32_freeState +# undef XXH32_reset +# undef XXH32_update +# undef XXH32_digest +# undef XXH32_copyState +# undef XXH32_canonicalFromHash +# undef XXH32_hashFromCanonical + /* XXH64 */ +# undef XXH64 +# undef XXH64_createState +# undef XXH64_freeState +# undef XXH64_reset +# undef XXH64_update +# undef XXH64_digest +# undef XXH64_copyState +# undef XXH64_canonicalFromHash +# undef XXH64_hashFromCanonical + /* XXH3_64bits */ +# undef XXH3_64bits +# undef XXH3_64bits_withSecret +# undef XXH3_64bits_withSeed +# undef XXH3_64bits_withSecretandSeed +# undef XXH3_createState +# undef XXH3_freeState +# undef XXH3_copyState +# undef XXH3_64bits_reset +# undef XXH3_64bits_reset_withSeed +# undef XXH3_64bits_reset_withSecret +# undef XXH3_64bits_update +# undef XXH3_64bits_digest +# undef XXH3_generateSecret + /* XXH3_128bits */ +# undef XXH128 +# undef XXH3_128bits +# undef XXH3_128bits_withSeed +# undef XXH3_128bits_withSecret +# undef XXH3_128bits_reset +# undef XXH3_128bits_reset_withSeed +# undef XXH3_128bits_reset_withSecret +# undef XXH3_128bits_reset_withSecretandSeed +# undef XXH3_128bits_update +# undef XXH3_128bits_digest +# undef XXH128_isEqual +# undef XXH128_cmp +# undef XXH128_canonicalFromHash +# undef XXH128_hashFromCanonical + /* Finally, free the namespace itself */ +# undef XXH_NAMESPACE + + /* employ the namespace for XXH_INLINE_ALL */ +# define XXH_NAMESPACE XXH_INLINE_ + /* + * Some identifiers (enums, type names) are not symbols, + * but they must nonetheless be renamed to avoid redeclaration. + * Alternative solution: do not redeclare them. + * However, this requires some #ifdefs, and has a more dispersed impact. + * Meanwhile, renaming can be achieved in a single place. + */ +# define XXH_IPREF(Id) XXH_NAMESPACE ## Id +# define XXH_OK XXH_IPREF(XXH_OK) +# define XXH_ERROR XXH_IPREF(XXH_ERROR) +# define XXH_errorcode XXH_IPREF(XXH_errorcode) +# define XXH32_canonical_t XXH_IPREF(XXH32_canonical_t) +# define XXH64_canonical_t XXH_IPREF(XXH64_canonical_t) +# define XXH128_canonical_t XXH_IPREF(XXH128_canonical_t) +# define XXH32_state_s XXH_IPREF(XXH32_state_s) +# define XXH32_state_t XXH_IPREF(XXH32_state_t) +# define XXH64_state_s XXH_IPREF(XXH64_state_s) +# define XXH64_state_t XXH_IPREF(XXH64_state_t) +# define XXH3_state_s XXH_IPREF(XXH3_state_s) +# define XXH3_state_t XXH_IPREF(XXH3_state_t) +# define XXH128_hash_t XXH_IPREF(XXH128_hash_t) + /* Ensure the header is parsed again, even if it was previously included */ +# undef XXHASH_H_5627135585666179 +# undef XXHASH_H_STATIC_13879238742 +#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */ + +/* **************************************************************** + * Stable API + *****************************************************************/ +#ifndef XXHASH_H_5627135585666179 +#define XXHASH_H_5627135585666179 1 + +/*! @brief Marks a global symbol. */ +#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) +# if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT)) +# ifdef XXH_EXPORT +# define XXH_PUBLIC_API __declspec(dllexport) +# elif XXH_IMPORT +# define XXH_PUBLIC_API __declspec(dllimport) +# endif +# else +# define XXH_PUBLIC_API /* do nothing */ +# endif +#endif + +#ifdef XXH_NAMESPACE +# define XXH_CAT(A,B) A##B +# define XXH_NAME2(A,B) XXH_CAT(A,B) +# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) +/* XXH32 */ +# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) +# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) +# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) +# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) +# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) +# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) +# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) +# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) +# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) +/* XXH64 */ +# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) +# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) +# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) +# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) +# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) +# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) +# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) +# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) +# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) +/* XXH3_64bits */ +# define XXH3_64bits XXH_NAME2(XXH_NAMESPACE, XXH3_64bits) +# define XXH3_64bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecret) +# define XXH3_64bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSeed) +# define XXH3_64bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecretandSeed) +# define XXH3_createState XXH_NAME2(XXH_NAMESPACE, XXH3_createState) +# define XXH3_freeState XXH_NAME2(XXH_NAMESPACE, XXH3_freeState) +# define XXH3_copyState XXH_NAME2(XXH_NAMESPACE, XXH3_copyState) +# define XXH3_64bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset) +# define XXH3_64bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSeed) +# define XXH3_64bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecret) +# define XXH3_64bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecretandSeed) +# define XXH3_64bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_update) +# define XXH3_64bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_digest) +# define XXH3_generateSecret XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret) +# define XXH3_generateSecret_fromSeed XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret_fromSeed) +/* XXH3_128bits */ +# define XXH128 XXH_NAME2(XXH_NAMESPACE, XXH128) +# define XXH3_128bits XXH_NAME2(XXH_NAMESPACE, XXH3_128bits) +# define XXH3_128bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSeed) +# define XXH3_128bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecret) +# define XXH3_128bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecretandSeed) +# define XXH3_128bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset) +# define XXH3_128bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSeed) +# define XXH3_128bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecret) +# define XXH3_128bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecretandSeed) +# define XXH3_128bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_update) +# define XXH3_128bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_digest) +# define XXH128_isEqual XXH_NAME2(XXH_NAMESPACE, XXH128_isEqual) +# define XXH128_cmp XXH_NAME2(XXH_NAMESPACE, XXH128_cmp) +# define XXH128_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH128_canonicalFromHash) +# define XXH128_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH128_hashFromCanonical) +#endif + + +/* ************************************* +* Compiler specifics +***************************************/ + +/* specific declaration modes for Windows */ +#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API) +# if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT)) +# ifdef XXH_EXPORT +# define XXH_PUBLIC_API __declspec(dllexport) +# elif XXH_IMPORT +# define XXH_PUBLIC_API __declspec(dllimport) +# endif +# else +# define XXH_PUBLIC_API /* do nothing */ +# endif +#endif + +#if defined (__GNUC__) +# define XXH_CONSTF __attribute__((const)) +# define XXH_PUREF __attribute__((pure)) +# define XXH_MALLOCF __attribute__((malloc)) +#else +# define XXH_CONSTF /* disable */ +# define XXH_PUREF +# define XXH_MALLOCF +#endif + +/* ************************************* +* Version +***************************************/ +#define XXH_VERSION_MAJOR 0 +#define XXH_VERSION_MINOR 8 +#define XXH_VERSION_RELEASE 2 +/*! @brief Version number, encoded as two digits each */ +#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) + +/*! + * @brief Obtains the xxHash version. + * + * This is mostly useful when xxHash is compiled as a shared library, + * since the returned value comes from the library, as opposed to header file. + * + * @return @ref XXH_VERSION_NUMBER of the invoked library. + */ +XXH_PUBLIC_API XXH_CONSTF unsigned XXH_versionNumber (void); + + +/* **************************** +* Common basic types +******************************/ +#include /* size_t */ +/*! + * @brief Exit code for the streaming API. + */ +typedef enum { + XXH_OK = 0, /*!< OK */ + XXH_ERROR /*!< Error */ +} XXH_errorcode; + + +/*-********************************************************************** +* 32-bit hash +************************************************************************/ +#if defined(XXH_DOXYGEN) /* Don't show include */ +/*! + * @brief An unsigned 32-bit integer. + * + * Not necessarily defined to `uint32_t` but functionally equivalent. + */ +typedef uint32_t XXH32_hash_t; + +#elif !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# ifdef _AIX +# include +# else +# include +# endif + typedef uint32_t XXH32_hash_t; + +#else +# include +# if UINT_MAX == 0xFFFFFFFFUL + typedef unsigned int XXH32_hash_t; +# elif ULONG_MAX == 0xFFFFFFFFUL + typedef unsigned long XXH32_hash_t; +# else +# error "unsupported platform: need a 32-bit type" +# endif +#endif + +/*! + * @} + * + * @defgroup XXH32_family XXH32 family + * @ingroup public + * Contains functions used in the classic 32-bit xxHash algorithm. + * + * @note + * XXH32 is useful for older platforms, with no or poor 64-bit performance. + * Note that the @ref XXH3_family provides competitive speed for both 32-bit + * and 64-bit systems, and offers true 64/128 bit hash results. + * + * @see @ref XXH64_family, @ref XXH3_family : Other xxHash families + * @see @ref XXH32_impl for implementation details + * @{ + */ + +/*! + * @brief Calculates the 32-bit hash of @p input using xxHash32. + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 32-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 32-bit xxHash32 value. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32 (const void* input, size_t length, XXH32_hash_t seed); + +#ifndef XXH_NO_STREAM +/*! + * @typedef struct XXH32_state_s XXH32_state_t + * @brief The opaque state struct for the XXH32 streaming API. + * + * @see XXH32_state_s for details. + */ +typedef struct XXH32_state_s XXH32_state_t; + +/*! + * @brief Allocates an @ref XXH32_state_t. + * + * @return An allocated pointer of @ref XXH32_state_t on success. + * @return `NULL` on failure. + * + * @note Must be freed with XXH32_freeState(). + */ +XXH_PUBLIC_API XXH_MALLOCF XXH32_state_t* XXH32_createState(void); +/*! + * @brief Frees an @ref XXH32_state_t. + * + * @param statePtr A pointer to an @ref XXH32_state_t allocated with @ref XXH32_createState(). + * + * @return @ref XXH_OK. + * + * @note @p statePtr must be allocated with XXH32_createState(). + * + */ +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); +/*! + * @brief Copies one @ref XXH32_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state); + +/*! + * @brief Resets an @ref XXH32_state_t to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param seed The 32-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note This function resets and seeds a state. Call it before @ref XXH32_update(). + */ +XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, XXH32_hash_t seed); + +/*! + * @brief Consumes a block of @p input to an @ref XXH32_state_t. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note Call this to incrementally consume blocks of data. + */ +XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); + +/*! + * @brief Returns the calculated hash value from an @ref XXH32_state_t. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated 32-bit xxHash32 value from that state. + * + * @note + * Calling XXH32_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + */ +XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ + +/******* Canonical representation *******/ + +/*! + * @brief Canonical (big endian) representation of @ref XXH32_hash_t. + */ +typedef struct { + unsigned char digest[4]; /*!< Hash bytes, big endian */ +} XXH32_canonical_t; + +/*! + * @brief Converts an @ref XXH32_hash_t to a big endian @ref XXH32_canonical_t. + * + * @param dst The @ref XXH32_canonical_t pointer to be stored to. + * @param hash The @ref XXH32_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + * + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); + +/*! + * @brief Converts an @ref XXH32_canonical_t to a native @ref XXH32_hash_t. + * + * @param src The @ref XXH32_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + * + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); + + +/*! @cond Doxygen ignores this part */ +#ifdef __has_attribute +# define XXH_HAS_ATTRIBUTE(x) __has_attribute(x) +#else +# define XXH_HAS_ATTRIBUTE(x) 0 +#endif +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +/* + * C23 __STDC_VERSION__ number hasn't been specified yet. For now + * leave as `201711L` (C17 + 1). + * TODO: Update to correct value when its been specified. + */ +#define XXH_C23_VN 201711L +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +/* C-language Attributes are added in C23. */ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= XXH_C23_VN) && defined(__has_c_attribute) +# define XXH_HAS_C_ATTRIBUTE(x) __has_c_attribute(x) +#else +# define XXH_HAS_C_ATTRIBUTE(x) 0 +#endif +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +#if defined(__cplusplus) && defined(__has_cpp_attribute) +# define XXH_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x) +#else +# define XXH_HAS_CPP_ATTRIBUTE(x) 0 +#endif +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +/* + * Define XXH_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute + * introduced in CPP17 and C23. + * CPP17 : https://en.cppreference.com/w/cpp/language/attributes/fallthrough + * C23 : https://en.cppreference.com/w/c/language/attributes/fallthrough + */ +#if XXH_HAS_C_ATTRIBUTE(fallthrough) || XXH_HAS_CPP_ATTRIBUTE(fallthrough) +# define XXH_FALLTHROUGH [[fallthrough]] +#elif XXH_HAS_ATTRIBUTE(__fallthrough__) +# define XXH_FALLTHROUGH __attribute__ ((__fallthrough__)) +#else +# define XXH_FALLTHROUGH /* fallthrough */ +#endif +/*! @endcond */ + +/*! @cond Doxygen ignores this part */ +/* + * Define XXH_NOESCAPE for annotated pointers in public API. + * https://clang.llvm.org/docs/AttributeReference.html#noescape + * As of writing this, only supported by clang. + */ +#if XXH_HAS_ATTRIBUTE(noescape) +# define XXH_NOESCAPE __attribute__((noescape)) +#else +# define XXH_NOESCAPE +#endif +/*! @endcond */ + + +/*! + * @} + * @ingroup public + * @{ + */ + +#ifndef XXH_NO_LONG_LONG +/*-********************************************************************** +* 64-bit hash +************************************************************************/ +#if defined(XXH_DOXYGEN) /* don't include */ +/*! + * @brief An unsigned 64-bit integer. + * + * Not necessarily defined to `uint64_t` but functionally equivalent. + */ +typedef uint64_t XXH64_hash_t; +#elif !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# ifdef _AIX +# include +# else +# include +# endif + typedef uint64_t XXH64_hash_t; +#else +# include +# if defined(__LP64__) && ULONG_MAX == 0xFFFFFFFFFFFFFFFFULL + /* LP64 ABI says uint64_t is unsigned long */ + typedef unsigned long XXH64_hash_t; +# else + /* the following type must have a width of 64-bit */ + typedef unsigned long long XXH64_hash_t; +# endif +#endif + +/*! + * @} + * + * @defgroup XXH64_family XXH64 family + * @ingroup public + * @{ + * Contains functions used in the classic 64-bit xxHash algorithm. + * + * @note + * XXH3 provides competitive speed for both 32-bit and 64-bit systems, + * and offers true 64/128 bit hash results. + * It provides better speed for systems with vector processing capabilities. + */ + +/*! + * @brief Calculates the 64-bit hash of @p input using xxHash64. + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 64-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit xxHash64 value. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed); + +/******* Streaming *******/ +#ifndef XXH_NO_STREAM +/*! + * @brief The opaque state struct for the XXH64 streaming API. + * + * @see XXH64_state_s for details. + */ +typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ + +/*! + * @brief Allocates an @ref XXH64_state_t. + * + * @return An allocated pointer of @ref XXH64_state_t on success. + * @return `NULL` on failure. + * + * @note Must be freed with XXH64_freeState(). + */ +XXH_PUBLIC_API XXH_MALLOCF XXH64_state_t* XXH64_createState(void); + +/*! + * @brief Frees an @ref XXH64_state_t. + * + * @param statePtr A pointer to an @ref XXH64_state_t allocated with @ref XXH64_createState(). + * + * @return @ref XXH_OK. + * + * @note @p statePtr must be allocated with XXH64_createState(). + */ +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); + +/*! + * @brief Copies one @ref XXH64_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dst_state, const XXH64_state_t* src_state); + +/*! + * @brief Resets an @ref XXH64_state_t to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note This function resets and seeds a state. Call it before @ref XXH64_update(). + */ +XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed); + +/*! + * @brief Consumes a block of @p input to an @ref XXH64_state_t. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note Call this to incrementally consume blocks of data. + */ +XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH_NOESCAPE XXH64_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); + +/*! + * @brief Returns the calculated hash value from an @ref XXH64_state_t. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated 64-bit xxHash64 value from that state. + * + * @note + * Calling XXH64_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_digest (XXH_NOESCAPE const XXH64_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ +/******* Canonical representation *******/ + +/*! + * @brief Canonical (big endian) representation of @ref XXH64_hash_t. + */ +typedef struct { unsigned char digest[sizeof(XXH64_hash_t)]; } XXH64_canonical_t; + +/*! + * @brief Converts an @ref XXH64_hash_t to a big endian @ref XXH64_canonical_t. + * + * @param dst The @ref XXH64_canonical_t pointer to be stored to. + * @param hash The @ref XXH64_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + * + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash); + +/*! + * @brief Converts an @ref XXH64_canonical_t to a native @ref XXH64_hash_t. + * + * @param src The @ref XXH64_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + * + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src); + +#ifndef XXH_NO_XXH3 + +/*! + * @} + * ************************************************************************ + * @defgroup XXH3_family XXH3 family + * @ingroup public + * @{ + * + * XXH3 is a more recent hash algorithm featuring: + * - Improved speed for both small and large inputs + * - True 64-bit and 128-bit outputs + * - SIMD acceleration + * - Improved 32-bit viability + * + * Speed analysis methodology is explained here: + * + * https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html + * + * Compared to XXH64, expect XXH3 to run approximately + * ~2x faster on large inputs and >3x faster on small ones, + * exact differences vary depending on platform. + * + * XXH3's speed benefits greatly from SIMD and 64-bit arithmetic, + * but does not require it. + * Most 32-bit and 64-bit targets that can run XXH32 smoothly can run XXH3 + * at competitive speeds, even without vector support. Further details are + * explained in the implementation. + * + * XXH3 has a fast scalar implementation, but it also includes accelerated SIMD + * implementations for many common platforms: + * - AVX512 + * - AVX2 + * - SSE2 + * - ARM NEON + * - WebAssembly SIMD128 + * - POWER8 VSX + * - s390x ZVector + * This can be controlled via the @ref XXH_VECTOR macro, but it automatically + * selects the best version according to predefined macros. For the x86 family, an + * automatic runtime dispatcher is included separately in @ref xxh_x86dispatch.c. + * + * XXH3 implementation is portable: + * it has a generic C90 formulation that can be compiled on any platform, + * all implementations generate exactly the same hash value on all platforms. + * Starting from v0.8.0, it's also labelled "stable", meaning that + * any future version will also generate the same hash value. + * + * XXH3 offers 2 variants, _64bits and _128bits. + * + * When only 64 bits are needed, prefer invoking the _64bits variant, as it + * reduces the amount of mixing, resulting in faster speed on small inputs. + * It's also generally simpler to manipulate a scalar return type than a struct. + * + * The API supports one-shot hashing, streaming mode, and custom secrets. + */ +/*-********************************************************************** +* XXH3 64-bit variant +************************************************************************/ + +/*! + * @brief Calculates 64-bit unseeded variant of XXH3 hash of @p input. + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit XXH3 hash value. + * + * @note + * This is equivalent to @ref XXH3_64bits_withSeed() with a seed of `0`, however + * it may have slightly better performance due to constant propagation of the + * defaults. + * + * @see + * XXH3_64bits_withSeed(), XXH3_64bits_withSecret(): other seeding variants + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length); + +/*! + * @brief Calculates 64-bit seeded variant of XXH3 hash of @p input. + * + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 64-bit XXH3 hash value. + * + * @note + * seed == 0 produces the same results as @ref XXH3_64bits(). + * + * This variant generates a custom secret on the fly based on default secret + * altered using the @p seed value. + * + * While this operation is decently fast, note that it's not completely free. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed); + +/*! + * The bare minimum size for a custom secret. + * + * @see + * XXH3_64bits_withSecret(), XXH3_64bits_reset_withSecret(), + * XXH3_128bits_withSecret(), XXH3_128bits_reset_withSecret(). + */ +#define XXH3_SECRET_SIZE_MIN 136 + +/*! + * @brief Calculates 64-bit variant of XXH3 with a custom "secret". + * + * @param data The block of data to be hashed, at least @p len bytes in size. + * @param len The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @return The calculated 64-bit XXH3 hash value. + * + * @pre + * The memory between @p data and @p data + @p len must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p data may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * It's possible to provide any blob of bytes as a "secret" to generate the hash. + * This makes it more difficult for an external actor to prepare an intentional collision. + * The main condition is that @p secretSize *must* be large enough (>= @ref XXH3_SECRET_SIZE_MIN). + * However, the quality of the secret impacts the dispersion of the hash algorithm. + * Therefore, the secret _must_ look like a bunch of random bytes. + * Avoid "trivial" or structured data such as repeated sequences or a text document. + * Whenever in doubt about the "randomness" of the blob of bytes, + * consider employing @ref XXH3_generateSecret() instead (see below). + * It will generate a proper high entropy secret derived from the blob of bytes. + * Another advantage of using XXH3_generateSecret() is that + * it guarantees that all bits within the initial blob of bytes + * will impact every bit of the output. + * This is not necessarily the case when using the blob of bytes directly + * because, when hashing _small_ inputs, only a portion of the secret is employed. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize); + + +/******* Streaming *******/ +#ifndef XXH_NO_STREAM +/* + * Streaming requires state maintenance. + * This operation costs memory and CPU. + * As a consequence, streaming is slower than one-shot hashing. + * For better performance, prefer one-shot functions whenever applicable. + */ + +/*! + * @brief The opaque state struct for the XXH3 streaming API. + * + * @see XXH3_state_s for details. + */ +typedef struct XXH3_state_s XXH3_state_t; +XXH_PUBLIC_API XXH_MALLOCF XXH3_state_t* XXH3_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr); + +/*! + * @brief Copies one @ref XXH3_state_t to another. + * + * @param dst_state The state to copy to. + * @param src_state The state to copy from. + * @pre + * @p dst_state and @p src_state must not be `NULL` and must not overlap. + */ +XXH_PUBLIC_API void XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOESCAPE const XXH3_state_t* src_state); + +/*! + * @brief Resets an @ref XXH3_state_t to begin a new hash. + * + * @param statePtr The state struct to reset. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret with default parameters. + * - Call this function before @ref XXH3_64bits_update(). + * - Digest will be equivalent to `XXH3_64bits()`. + * + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr); + +/*! + * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret from `seed`. + * - Call this function before @ref XXH3_64bits_update(). + * - Digest will be equivalent to `XXH3_64bits_withSeed()`. + * + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed); + +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * `secret` is referenced, it _must outlive_ the hash streaming session. + * + * Similar to one-shot API, `secretSize` must be >= @ref XXH3_SECRET_SIZE_MIN, + * and the quality of produced hash values depends on secret's entropy + * (secret's content should look like a bunch of random bytes). + * When in doubt about the randomness of a candidate `secret`, + * consider employing `XXH3_generateSecret()` instead (see below). + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize); + +/*! + * @brief Consumes a block of @p input to an @ref XXH3_state_t. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * @pre + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note Call this to incrementally consume blocks of data. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); + +/*! + * @brief Returns the calculated XXH3 64-bit hash value from an @ref XXH3_state_t. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated XXH3 64-bit hash value from that state. + * + * @note + * Calling XXH3_64bits_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ + +/* note : canonical representation of XXH3 is the same as XXH64 + * since they both produce XXH64_hash_t values */ + + +/*-********************************************************************** +* XXH3 128-bit variant +************************************************************************/ + +/*! + * @brief The return value from 128-bit hashes. + * + * Stored in little endian order, although the fields themselves are in native + * endianness. + */ +typedef struct { + XXH64_hash_t low64; /*!< `value & 0xFFFFFFFFFFFFFFFF` */ + XXH64_hash_t high64; /*!< `value >> 64` */ +} XXH128_hash_t; + +/*! + * @brief Calculates 128-bit unseeded variant of XXH3 of @p data. + * + * @param data The block of data to be hashed, at least @p length bytes in size. + * @param len The length of @p data, in bytes. + * + * @return The calculated 128-bit variant of XXH3 value. + * + * The 128-bit variant of XXH3 has more strength, but it has a bit of overhead + * for shorter inputs. + * + * This is equivalent to @ref XXH3_128bits_withSeed() with a seed of `0`, however + * it may have slightly better performance due to constant propagation of the + * defaults. + * + * @see XXH3_128bits_withSeed(), XXH3_128bits_withSecret(): other seeding variants + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* data, size_t len); +/*! @brief Calculates 128-bit seeded variant of XXH3 hash of @p data. + * + * @param data The block of data to be hashed, at least @p length bytes in size. + * @param len The length of @p data, in bytes. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @return The calculated 128-bit variant of XXH3 value. + * + * @note + * seed == 0 produces the same results as @ref XXH3_64bits(). + * + * This variant generates a custom secret on the fly based on default secret + * altered using the @p seed value. + * + * While this operation is decently fast, note that it's not completely free. + * + * @see XXH3_128bits(), XXH3_128bits_withSecret(): other seeding variants + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSeed(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed); +/*! + * @brief Calculates 128-bit variant of XXH3 with a custom "secret". + * + * @param data The block of data to be hashed, at least @p len bytes in size. + * @param len The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @return The calculated 128-bit variant of XXH3 value. + * + * It's possible to provide any blob of bytes as a "secret" to generate the hash. + * This makes it more difficult for an external actor to prepare an intentional collision. + * The main condition is that @p secretSize *must* be large enough (>= @ref XXH3_SECRET_SIZE_MIN). + * However, the quality of the secret impacts the dispersion of the hash algorithm. + * Therefore, the secret _must_ look like a bunch of random bytes. + * Avoid "trivial" or structured data such as repeated sequences or a text document. + * Whenever in doubt about the "randomness" of the blob of bytes, + * consider employing @ref XXH3_generateSecret() instead (see below). + * It will generate a proper high entropy secret derived from the blob of bytes. + * Another advantage of using XXH3_generateSecret() is that + * it guarantees that all bits within the initial blob of bytes + * will impact every bit of the output. + * This is not necessarily the case when using the blob of bytes directly + * because, when hashing _small_ inputs, only a portion of the secret is employed. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize); + +/******* Streaming *******/ +#ifndef XXH_NO_STREAM +/* + * Streaming requires state maintenance. + * This operation costs memory and CPU. + * As a consequence, streaming is slower than one-shot hashing. + * For better performance, prefer one-shot functions whenever applicable. + * + * XXH3_128bits uses the same XXH3_state_t as XXH3_64bits(). + * Use already declared XXH3_createState() and XXH3_freeState(). + * + * All reset and streaming functions have same meaning as their 64-bit counterpart. + */ + +/*! + * @brief Resets an @ref XXH3_state_t to begin a new hash. + * + * @param statePtr The state struct to reset. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret with default parameters. + * - Call it before @ref XXH3_128bits_update(). + * - Digest will be equivalent to `XXH3_128bits()`. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr); + +/*! + * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * - This function resets `statePtr` and generate a secret from `seed`. + * - Call it before @ref XXH3_128bits_update(). + * - Digest will be equivalent to `XXH3_128bits_withSeed()`. + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed); +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr The state struct to reset. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * `secret` is referenced, it _must outlive_ the hash streaming session. + * Similar to one-shot API, `secretSize` must be >= @ref XXH3_SECRET_SIZE_MIN, + * and the quality of produced hash values depends on secret's entropy + * (secret's content should look like a bunch of random bytes). + * When in doubt about the randomness of a candidate `secret`, + * consider employing `XXH3_generateSecret()` instead (see below). + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize); + +/*! + * @brief Consumes a block of @p input to an @ref XXH3_state_t. + * + * Call this to incrementally consume blocks of data. + * + * @param statePtr The state struct to update. + * @param input The block of data to be hashed, at least @p length bytes in size. + * @param length The length of @p input, in bytes. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @note + * The memory between @p input and @p input + @p length must be valid, + * readable, contiguous memory. However, if @p length is `0`, @p input may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + */ +XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length); + +/*! + * @brief Returns the calculated XXH3 128-bit hash value from an @ref XXH3_state_t. + * + * @param statePtr The state struct to calculate the hash from. + * + * @pre + * @p statePtr must not be `NULL`. + * + * @return The calculated XXH3 128-bit hash value from that state. + * + * @note + * Calling XXH3_128bits_digest() will not affect @p statePtr, so you can update, + * digest, and update again. + * + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr); +#endif /* !XXH_NO_STREAM */ + +/* Following helper functions make it possible to compare XXH128_hast_t values. + * Since XXH128_hash_t is a structure, this capability is not offered by the language. + * Note: For better performance, these functions can be inlined using XXH_INLINE_ALL */ + +/*! + * @brief Check equality of two XXH128_hash_t values + * + * @param h1 The 128-bit hash value. + * @param h2 Another 128-bit hash value. + * + * @return `1` if `h1` and `h2` are equal. + * @return `0` if they are not. + */ +XXH_PUBLIC_API XXH_PUREF int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2); + +/*! + * @brief Compares two @ref XXH128_hash_t + * + * This comparator is compatible with stdlib's `qsort()`/`bsearch()`. + * + * @param h128_1 Left-hand side value + * @param h128_2 Right-hand side value + * + * @return >0 if @p h128_1 > @p h128_2 + * @return =0 if @p h128_1 == @p h128_2 + * @return <0 if @p h128_1 < @p h128_2 + */ +XXH_PUBLIC_API XXH_PUREF int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2); + + +/******* Canonical representation *******/ +typedef struct { unsigned char digest[sizeof(XXH128_hash_t)]; } XXH128_canonical_t; + + +/*! + * @brief Converts an @ref XXH128_hash_t to a big endian @ref XXH128_canonical_t. + * + * @param dst The @ref XXH128_canonical_t pointer to be stored to. + * @param hash The @ref XXH128_hash_t to be converted. + * + * @pre + * @p dst must not be `NULL`. + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash); + +/*! + * @brief Converts an @ref XXH128_canonical_t to a native @ref XXH128_hash_t. + * + * @param src The @ref XXH128_canonical_t to convert. + * + * @pre + * @p src must not be `NULL`. + * + * @return The converted hash. + * @see @ref canonical_representation_example "Canonical Representation Example" + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src); + + +#endif /* !XXH_NO_XXH3 */ +#endif /* XXH_NO_LONG_LONG */ + +/*! + * @} + */ +#endif /* XXHASH_H_5627135585666179 */ + + + +#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) +#define XXHASH_H_STATIC_13879238742 +/* **************************************************************************** + * This section contains declarations which are not guaranteed to remain stable. + * They may change in future versions, becoming incompatible with a different + * version of the library. + * These declarations should only be used with static linking. + * Never use them in association with dynamic linking! + ***************************************************************************** */ + +/* + * These definitions are only present to allow static allocation + * of XXH states, on stack or in a struct, for example. + * Never **ever** access their members directly. + */ + +/*! + * @internal + * @brief Structure for XXH32 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * Typedef'd to @ref XXH32_state_t. + * Do not access the members of this struct directly. + * @see XXH64_state_s, XXH3_state_s + */ +struct XXH32_state_s { + XXH32_hash_t total_len_32; /*!< Total length hashed, modulo 2^32 */ + XXH32_hash_t large_len; /*!< Whether the hash is >= 16 (handles @ref total_len_32 overflow) */ + XXH32_hash_t v[4]; /*!< Accumulator lanes */ + XXH32_hash_t mem32[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[16]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem32 */ + XXH32_hash_t reserved; /*!< Reserved field. Do not read nor write to it. */ +}; /* typedef'd to XXH32_state_t */ + + +#ifndef XXH_NO_LONG_LONG /* defined when there is no 64-bit support */ + +/*! + * @internal + * @brief Structure for XXH64 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is + * an opaque type. This allows fields to safely be changed. + * + * Typedef'd to @ref XXH64_state_t. + * Do not access the members of this struct directly. + * @see XXH32_state_s, XXH3_state_s + */ +struct XXH64_state_s { + XXH64_hash_t total_len; /*!< Total length hashed. This is always 64-bit. */ + XXH64_hash_t v[4]; /*!< Accumulator lanes */ + XXH64_hash_t mem64[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[32]. */ + XXH32_hash_t memsize; /*!< Amount of data in @ref mem64 */ + XXH32_hash_t reserved32; /*!< Reserved field, needed for padding anyways*/ + XXH64_hash_t reserved64; /*!< Reserved field. Do not read or write to it. */ +}; /* typedef'd to XXH64_state_t */ + +#ifndef XXH_NO_XXH3 + +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 */ +# include +# define XXH_ALIGN(n) alignas(n) +#elif defined(__cplusplus) && (__cplusplus >= 201103L) /* >= C++11 */ +/* In C++ alignas() is a keyword */ +# define XXH_ALIGN(n) alignas(n) +#elif defined(__GNUC__) +# define XXH_ALIGN(n) __attribute__ ((aligned(n))) +#elif defined(_MSC_VER) +# define XXH_ALIGN(n) __declspec(align(n)) +#else +# define XXH_ALIGN(n) /* disabled */ +#endif + +/* Old GCC versions only accept the attribute after the type in structures. */ +#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) /* C11+ */ \ + && ! (defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 */ \ + && defined(__GNUC__) +# define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align) +#else +# define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type +#endif + +/*! + * @brief The size of the internal XXH3 buffer. + * + * This is the optimal update size for incremental hashing. + * + * @see XXH3_64b_update(), XXH3_128b_update(). + */ +#define XXH3_INTERNALBUFFER_SIZE 256 + +/*! + * @internal + * @brief Default size of the secret buffer (and @ref XXH3_kSecret). + * + * This is the size used in @ref XXH3_kSecret and the seeded functions. + * + * Not to be confused with @ref XXH3_SECRET_SIZE_MIN. + */ +#define XXH3_SECRET_DEFAULT_SIZE 192 + +/*! + * @internal + * @brief Structure for XXH3 streaming API. + * + * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY, + * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. + * Otherwise it is an opaque type. + * Never use this definition in combination with dynamic library. + * This allows fields to safely be changed in the future. + * + * @note ** This structure has a strict alignment requirement of 64 bytes!! ** + * Do not allocate this with `malloc()` or `new`, + * it will not be sufficiently aligned. + * Use @ref XXH3_createState() and @ref XXH3_freeState(), or stack allocation. + * + * Typedef'd to @ref XXH3_state_t. + * Do never access the members of this struct directly. + * + * @see XXH3_INITSTATE() for stack initialization. + * @see XXH3_createState(), XXH3_freeState(). + * @see XXH32_state_s, XXH64_state_s + */ +struct XXH3_state_s { + XXH_ALIGN_MEMBER(64, XXH64_hash_t acc[8]); + /*!< The 8 accumulators. See @ref XXH32_state_s::v and @ref XXH64_state_s::v */ + XXH_ALIGN_MEMBER(64, unsigned char customSecret[XXH3_SECRET_DEFAULT_SIZE]); + /*!< Used to store a custom secret generated from a seed. */ + XXH_ALIGN_MEMBER(64, unsigned char buffer[XXH3_INTERNALBUFFER_SIZE]); + /*!< The internal buffer. @see XXH32_state_s::mem32 */ + XXH32_hash_t bufferedSize; + /*!< The amount of memory in @ref buffer, @see XXH32_state_s::memsize */ + XXH32_hash_t useSeed; + /*!< Reserved field. Needed for padding on 64-bit. */ + size_t nbStripesSoFar; + /*!< Number or stripes processed. */ + XXH64_hash_t totalLen; + /*!< Total length hashed. 64-bit even on 32-bit targets. */ + size_t nbStripesPerBlock; + /*!< Number of stripes per block. */ + size_t secretLimit; + /*!< Size of @ref customSecret or @ref extSecret */ + XXH64_hash_t seed; + /*!< Seed for _withSeed variants. Must be zero otherwise, @see XXH3_INITSTATE() */ + XXH64_hash_t reserved64; + /*!< Reserved field. */ + const unsigned char* extSecret; + /*!< Reference to an external secret for the _withSecret variants, NULL + * for other variants. */ + /* note: there may be some padding at the end due to alignment on 64 bytes */ +}; /* typedef'd to XXH3_state_t */ + +#undef XXH_ALIGN_MEMBER + +/*! + * @brief Initializes a stack-allocated `XXH3_state_s`. + * + * When the @ref XXH3_state_t structure is merely emplaced on stack, + * it should be initialized with XXH3_INITSTATE() or a memset() + * in case its first reset uses XXH3_NNbits_reset_withSeed(). + * This init can be omitted if the first reset uses default or _withSecret mode. + * This operation isn't necessary when the state is created with XXH3_createState(). + * Note that this doesn't prepare the state for a streaming operation, + * it's still necessary to use XXH3_NNbits_reset*() afterwards. + */ +#define XXH3_INITSTATE(XXH3_state_ptr) \ + do { \ + XXH3_state_t* tmp_xxh3_state_ptr = (XXH3_state_ptr); \ + tmp_xxh3_state_ptr->seed = 0; \ + tmp_xxh3_state_ptr->extSecret = NULL; \ + } while(0) + + +/*! + * @brief Calculates the 128-bit hash of @p data using XXH3. + * + * @param data The block of data to be hashed, at least @p len bytes in size. + * @param len The length of @p data, in bytes. + * @param seed The 64-bit seed to alter the hash's output predictably. + * + * @pre + * The memory between @p data and @p data + @p len must be valid, + * readable, contiguous memory. However, if @p len is `0`, @p data may be + * `NULL`. In C++, this also must be *TriviallyCopyable*. + * + * @return The calculated 128-bit XXH3 value. + * + * @see @ref single_shot_example "Single Shot Example" for an example. + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed); + + +/* === Experimental API === */ +/* Symbols defined below must be considered tied to a specific library version. */ + +/*! + * @brief Derive a high-entropy secret from any user-defined content, named customSeed. + * + * @param secretBuffer A writable buffer for derived high-entropy secret data. + * @param secretSize Size of secretBuffer, in bytes. Must be >= XXH3_SECRET_DEFAULT_SIZE. + * @param customSeed A user-defined content. + * @param customSeedSize Size of customSeed, in bytes. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * The generated secret can be used in combination with `*_withSecret()` functions. + * The `_withSecret()` variants are useful to provide a higher level of protection + * than 64-bit seed, as it becomes much more difficult for an external actor to + * guess how to impact the calculation logic. + * + * The function accepts as input a custom seed of any length and any content, + * and derives from it a high-entropy secret of length @p secretSize into an + * already allocated buffer @p secretBuffer. + * + * The generated secret can then be used with any `*_withSecret()` variant. + * The functions @ref XXH3_128bits_withSecret(), @ref XXH3_64bits_withSecret(), + * @ref XXH3_128bits_reset_withSecret() and @ref XXH3_64bits_reset_withSecret() + * are part of this list. They all accept a `secret` parameter + * which must be large enough for implementation reasons (>= @ref XXH3_SECRET_SIZE_MIN) + * _and_ feature very high entropy (consist of random-looking bytes). + * These conditions can be a high bar to meet, so @ref XXH3_generateSecret() can + * be employed to ensure proper quality. + * + * @p customSeed can be anything. It can have any size, even small ones, + * and its content can be anything, even "poor entropy" sources such as a bunch + * of zeroes. The resulting `secret` will nonetheless provide all required qualities. + * + * @pre + * - @p secretSize must be >= @ref XXH3_SECRET_SIZE_MIN + * - When @p customSeedSize > 0, supplying NULL as customSeed is undefined behavior. + * + * Example code: + * @code{.c} + * #include + * #include + * #include + * #define XXH_STATIC_LINKING_ONLY // expose unstable API + * #include "xxhash.h" + * // Hashes argv[2] using the entropy from argv[1]. + * int main(int argc, char* argv[]) + * { + * char secret[XXH3_SECRET_SIZE_MIN]; + * if (argv != 3) { return 1; } + * XXH3_generateSecret(secret, sizeof(secret), argv[1], strlen(argv[1])); + * XXH64_hash_t h = XXH3_64bits_withSecret( + * argv[2], strlen(argv[2]), + * secret, sizeof(secret) + * ); + * printf("%016llx\n", (unsigned long long) h); + * } + * @endcode + */ +XXH_PUBLIC_API XXH_errorcode XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer, size_t secretSize, XXH_NOESCAPE const void* customSeed, size_t customSeedSize); + +/*! + * @brief Generate the same secret as the _withSeed() variants. + * + * @param secretBuffer A writable buffer of @ref XXH3_SECRET_SIZE_MIN bytes + * @param seed The 64-bit seed to alter the hash result predictably. + * + * The generated secret can be used in combination with + *`*_withSecret()` and `_withSecretandSeed()` variants. + * + * Example C++ `std::string` hash class: + * @code{.cpp} + * #include + * #define XXH_STATIC_LINKING_ONLY // expose unstable API + * #include "xxhash.h" + * // Slow, seeds each time + * class HashSlow { + * XXH64_hash_t seed; + * public: + * HashSlow(XXH64_hash_t s) : seed{s} {} + * size_t operator()(const std::string& x) const { + * return size_t{XXH3_64bits_withSeed(x.c_str(), x.length(), seed)}; + * } + * }; + * // Fast, caches the seeded secret for future uses. + * class HashFast { + * unsigned char secret[XXH3_SECRET_SIZE_MIN]; + * public: + * HashFast(XXH64_hash_t s) { + * XXH3_generateSecret_fromSeed(secret, seed); + * } + * size_t operator()(const std::string& x) const { + * return size_t{ + * XXH3_64bits_withSecret(x.c_str(), x.length(), secret, sizeof(secret)) + * }; + * } + * }; + * @endcode + */ +XXH_PUBLIC_API void XXH3_generateSecret_fromSeed(XXH_NOESCAPE void* secretBuffer, XXH64_hash_t seed); + +/*! + * @brief Calculates 64/128-bit seeded variant of XXH3 hash of @p data. + * + * @param data The block of data to be hashed, at least @p len bytes in size. + * @param len The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed The 64-bit seed to alter the hash result predictably. + * + * These variants generate hash values using either + * @p seed for "short" keys (< @ref XXH3_MIDSIZE_MAX = 240 bytes) + * or @p secret for "large" keys (>= @ref XXH3_MIDSIZE_MAX). + * + * This generally benefits speed, compared to `_withSeed()` or `_withSecret()`. + * `_withSeed()` has to generate the secret on the fly for "large" keys. + * It's fast, but can be perceptible for "not so large" keys (< 1 KB). + * `_withSecret()` has to generate the masks on the fly for "small" keys, + * which requires more instructions than _withSeed() variants. + * Therefore, _withSecretandSeed variant combines the best of both worlds. + * + * When @p secret has been generated by XXH3_generateSecret_fromSeed(), + * this variant produces *exactly* the same results as `_withSeed()` variant, + * hence offering only a pure speed benefit on "large" input, + * by skipping the need to regenerate the secret for every large input. + * + * Another usage scenario is to hash the secret to a 64-bit hash value, + * for example with XXH3_64bits(), which then becomes the seed, + * and then employ both the seed and the secret in _withSecretandSeed(). + * On top of speed, an added benefit is that each bit in the secret + * has a 50% chance to swap each bit in the output, via its impact to the seed. + * + * This is not guaranteed when using the secret directly in "small data" scenarios, + * because only portions of the secret are employed for small data. + */ +XXH_PUBLIC_API XXH_PUREF XXH64_hash_t +XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* data, size_t len, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed); +/*! + * @brief Calculates 128-bit seeded variant of XXH3 hash of @p data. + * + * @param input The block of data to be hashed, at least @p len bytes in size. + * @param length The length of @p data, in bytes. + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed64 The 64-bit seed to alter the hash result predictably. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @see XXH3_64bits_withSecretandSeed() + */ +XXH_PUBLIC_API XXH_PUREF XXH128_hash_t +XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed64); +#ifndef XXH_NO_STREAM +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState(). + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed64 The 64-bit seed to alter the hash result predictably. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @see XXH3_64bits_withSecretandSeed() + */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed64); +/*! + * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash. + * + * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState(). + * @param secret The secret data. + * @param secretSize The length of @p secret, in bytes. + * @param seed64 The 64-bit seed to alter the hash result predictably. + * + * @return @ref XXH_OK on success. + * @return @ref XXH_ERROR on failure. + * + * @see XXH3_64bits_withSecretandSeed() + */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, + XXH_NOESCAPE const void* secret, size_t secretSize, + XXH64_hash_t seed64); +#endif /* !XXH_NO_STREAM */ + +#endif /* !XXH_NO_XXH3 */ +#endif /* XXH_NO_LONG_LONG */ +#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) +# define XXH_IMPLEMENTATION +#endif + +#endif /* defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) */ + + +/* ======================================================================== */ +/* ======================================================================== */ +/* ======================================================================== */ + + +/*-********************************************************************** + * xxHash implementation + *-********************************************************************** + * xxHash's implementation used to be hosted inside xxhash.c. + * + * However, inlining requires implementation to be visible to the compiler, + * hence be included alongside the header. + * Previously, implementation was hosted inside xxhash.c, + * which was then #included when inlining was activated. + * This construction created issues with a few build and install systems, + * as it required xxhash.c to be stored in /include directory. + * + * xxHash implementation is now directly integrated within xxhash.h. + * As a consequence, xxhash.c is no longer needed in /include. + * + * xxhash.c is still available and is still useful. + * In a "normal" setup, when xxhash is not inlined, + * xxhash.h only exposes the prototypes and public symbols, + * while xxhash.c can be built into an object file xxhash.o + * which can then be linked into the final binary. + ************************************************************************/ + +#if ( defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) \ + || defined(XXH_IMPLEMENTATION) ) && !defined(XXH_IMPLEM_13a8737387) +# define XXH_IMPLEM_13a8737387 + +/* ************************************* +* Tuning parameters +***************************************/ + +/*! + * @defgroup tuning Tuning parameters + * @{ + * + * Various macros to control xxHash's behavior. + */ +#ifdef XXH_DOXYGEN +/*! + * @brief Define this to disable 64-bit code. + * + * Useful if only using the @ref XXH32_family and you have a strict C90 compiler. + */ +# define XXH_NO_LONG_LONG +# undef XXH_NO_LONG_LONG /* don't actually */ +/*! + * @brief Controls how unaligned memory is accessed. + * + * By default, access to unaligned memory is controlled by `memcpy()`, which is + * safe and portable. + * + * Unfortunately, on some target/compiler combinations, the generated assembly + * is sub-optimal. + * + * The below switch allow selection of a different access method + * in the search for improved performance. + * + * @par Possible options: + * + * - `XXH_FORCE_MEMORY_ACCESS=0` (default): `memcpy` + * @par + * Use `memcpy()`. Safe and portable. Note that most modern compilers will + * eliminate the function call and treat it as an unaligned access. + * + * - `XXH_FORCE_MEMORY_ACCESS=1`: `__attribute__((aligned(1)))` + * @par + * Depends on compiler extensions and is therefore not portable. + * This method is safe _if_ your compiler supports it, + * and *generally* as fast or faster than `memcpy`. + * + * - `XXH_FORCE_MEMORY_ACCESS=2`: Direct cast + * @par + * Casts directly and dereferences. This method doesn't depend on the + * compiler, but it violates the C standard as it directly dereferences an + * unaligned pointer. It can generate buggy code on targets which do not + * support unaligned memory accesses, but in some circumstances, it's the + * only known way to get the most performance. + * + * - `XXH_FORCE_MEMORY_ACCESS=3`: Byteshift + * @par + * Also portable. This can generate the best code on old compilers which don't + * inline small `memcpy()` calls, and it might also be faster on big-endian + * systems which lack a native byteswap instruction. However, some compilers + * will emit literal byteshifts even if the target supports unaligned access. + * + * + * @warning + * Methods 1 and 2 rely on implementation-defined behavior. Use these with + * care, as what works on one compiler/platform/optimization level may cause + * another to read garbage data or even crash. + * + * See https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html for details. + * + * Prefer these methods in priority order (0 > 3 > 1 > 2) + */ +# define XXH_FORCE_MEMORY_ACCESS 0 + +/*! + * @def XXH_SIZE_OPT + * @brief Controls how much xxHash optimizes for size. + * + * xxHash, when compiled, tends to result in a rather large binary size. This + * is mostly due to heavy usage to forced inlining and constant folding of the + * @ref XXH3_family to increase performance. + * + * However, some developers prefer size over speed. This option can + * significantly reduce the size of the generated code. When using the `-Os` + * or `-Oz` options on GCC or Clang, this is defined to 1 by default, + * otherwise it is defined to 0. + * + * Most of these size optimizations can be controlled manually. + * + * This is a number from 0-2. + * - `XXH_SIZE_OPT` == 0: Default. xxHash makes no size optimizations. Speed + * comes first. + * - `XXH_SIZE_OPT` == 1: Default for `-Os` and `-Oz`. xxHash is more + * conservative and disables hacks that increase code size. It implies the + * options @ref XXH_NO_INLINE_HINTS == 1, @ref XXH_FORCE_ALIGN_CHECK == 0, + * and @ref XXH3_NEON_LANES == 8 if they are not already defined. + * - `XXH_SIZE_OPT` == 2: xxHash tries to make itself as small as possible. + * Performance may cry. For example, the single shot functions just use the + * streaming API. + */ +# define XXH_SIZE_OPT 0 + +/*! + * @def XXH_FORCE_ALIGN_CHECK + * @brief If defined to non-zero, adds a special path for aligned inputs (XXH32() + * and XXH64() only). + * + * This is an important performance trick for architectures without decent + * unaligned memory access performance. + * + * It checks for input alignment, and when conditions are met, uses a "fast + * path" employing direct 32-bit/64-bit reads, resulting in _dramatically + * faster_ read speed. + * + * The check costs one initial branch per hash, which is generally negligible, + * but not zero. + * + * Moreover, it's not useful to generate an additional code path if memory + * access uses the same instruction for both aligned and unaligned + * addresses (e.g. x86 and aarch64). + * + * In these cases, the alignment check can be removed by setting this macro to 0. + * Then the code will always use unaligned memory access. + * Align check is automatically disabled on x86, x64, ARM64, and some ARM chips + * which are platforms known to offer good unaligned memory accesses performance. + * + * It is also disabled by default when @ref XXH_SIZE_OPT >= 1. + * + * This option does not affect XXH3 (only XXH32 and XXH64). + */ +# define XXH_FORCE_ALIGN_CHECK 0 + +/*! + * @def XXH_NO_INLINE_HINTS + * @brief When non-zero, sets all functions to `static`. + * + * By default, xxHash tries to force the compiler to inline almost all internal + * functions. + * + * This can usually improve performance due to reduced jumping and improved + * constant folding, but significantly increases the size of the binary which + * might not be favorable. + * + * Additionally, sometimes the forced inlining can be detrimental to performance, + * depending on the architecture. + * + * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the + * compiler full control on whether to inline or not. + * + * When not optimizing (-O0), using `-fno-inline` with GCC or Clang, or if + * @ref XXH_SIZE_OPT >= 1, this will automatically be defined. + */ +# define XXH_NO_INLINE_HINTS 0 + +/*! + * @def XXH3_INLINE_SECRET + * @brief Determines whether to inline the XXH3 withSecret code. + * + * When the secret size is known, the compiler can improve the performance + * of XXH3_64bits_withSecret() and XXH3_128bits_withSecret(). + * + * However, if the secret size is not known, it doesn't have any benefit. This + * happens when xxHash is compiled into a global symbol. Therefore, if + * @ref XXH_INLINE_ALL is *not* defined, this will be defined to 0. + * + * Additionally, this defaults to 0 on GCC 12+, which has an issue with function pointers + * that are *sometimes* force inline on -Og, and it is impossible to automatically + * detect this optimization level. + */ +# define XXH3_INLINE_SECRET 0 + +/*! + * @def XXH32_ENDJMP + * @brief Whether to use a jump for `XXH32_finalize`. + * + * For performance, `XXH32_finalize` uses multiple branches in the finalizer. + * This is generally preferable for performance, + * but depending on exact architecture, a jmp may be preferable. + * + * This setting is only possibly making a difference for very small inputs. + */ +# define XXH32_ENDJMP 0 + +/*! + * @internal + * @brief Redefines old internal names. + * + * For compatibility with code that uses xxHash's internals before the names + * were changed to improve namespacing. There is no other reason to use this. + */ +# define XXH_OLD_NAMES +# undef XXH_OLD_NAMES /* don't actually use, it is ugly. */ + +/*! + * @def XXH_NO_STREAM + * @brief Disables the streaming API. + * + * When xxHash is not inlined and the streaming functions are not used, disabling + * the streaming functions can improve code size significantly, especially with + * the @ref XXH3_family which tends to make constant folded copies of itself. + */ +# define XXH_NO_STREAM +# undef XXH_NO_STREAM /* don't actually */ +#endif /* XXH_DOXYGEN */ +/*! + * @} + */ + +#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ + /* prefer __packed__ structures (method 1) for GCC + * < ARMv7 with unaligned access (e.g. Raspbian armhf) still uses byte shifting, so we use memcpy + * which for some reason does unaligned loads. */ +# if defined(__GNUC__) && !(defined(__ARM_ARCH) && __ARM_ARCH < 7 && defined(__ARM_FEATURE_UNALIGNED)) +# define XXH_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +#ifndef XXH_SIZE_OPT + /* default to 1 for -Os or -Oz */ +# if (defined(__GNUC__) || defined(__clang__)) && defined(__OPTIMIZE_SIZE__) +# define XXH_SIZE_OPT 1 +# else +# define XXH_SIZE_OPT 0 +# endif +#endif + +#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ + /* don't check on sizeopt, x86, aarch64, or arm when unaligned access is available */ +# if XXH_SIZE_OPT >= 1 || \ + defined(__i386) || defined(__x86_64__) || defined(__aarch64__) || defined(__ARM_FEATURE_UNALIGNED) \ + || defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) || defined(_M_ARM) /* visual */ +# define XXH_FORCE_ALIGN_CHECK 0 +# else +# define XXH_FORCE_ALIGN_CHECK 1 +# endif +#endif + +#ifndef XXH_NO_INLINE_HINTS +# if XXH_SIZE_OPT >= 1 || defined(__NO_INLINE__) /* -O0, -fno-inline */ +# define XXH_NO_INLINE_HINTS 1 +# else +# define XXH_NO_INLINE_HINTS 0 +# endif +#endif + +#ifndef XXH3_INLINE_SECRET +# if (defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 12) \ + || !defined(XXH_INLINE_ALL) +# define XXH3_INLINE_SECRET 0 +# else +# define XXH3_INLINE_SECRET 1 +# endif +#endif + +#ifndef XXH32_ENDJMP +/* generally preferable for performance */ +# define XXH32_ENDJMP 0 +#endif + +/*! + * @defgroup impl Implementation + * @{ + */ + + +/* ************************************* +* Includes & Memory related functions +***************************************/ +#if defined(XXH_NO_STREAM) +/* nothing */ +#elif defined(XXH_NO_STDLIB) + +/* When requesting to disable any mention of stdlib, + * the library loses the ability to invoked malloc / free. + * In practice, it means that functions like `XXH*_createState()` + * will always fail, and return NULL. + * This flag is useful in situations where + * xxhash.h is integrated into some kernel, embedded or limited environment + * without access to dynamic allocation. + */ + +static XXH_CONSTF void* XXH_malloc(size_t s) { (void)s; return NULL; } +static void XXH_free(void* p) { (void)p; } + +#else + +/* + * Modify the local functions below should you wish to use + * different memory routines for malloc() and free() + */ +#include + +/*! + * @internal + * @brief Modify this function to use a different routine than malloc(). + */ +static XXH_MALLOCF void* XXH_malloc(size_t s) { return malloc(s); } + +/*! + * @internal + * @brief Modify this function to use a different routine than free(). + */ +static void XXH_free(void* p) { free(p); } + +#endif /* XXH_NO_STDLIB */ + +#include + +/*! + * @internal + * @brief Modify this function to use a different routine than memcpy(). + */ +static void* XXH_memcpy(void* dest, const void* src, size_t size) +{ + return memcpy(dest,src,size); +} + +#include /* ULLONG_MAX */ + + +/* ************************************* +* Compiler Specific Options +***************************************/ +#ifdef _MSC_VER /* Visual Studio warning fix */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + +#if XXH_NO_INLINE_HINTS /* disable inlining hints */ +# if defined(__GNUC__) || defined(__clang__) +# define XXH_FORCE_INLINE static __attribute__((unused)) +# else +# define XXH_FORCE_INLINE static +# endif +# define XXH_NO_INLINE static +/* enable inlining hints */ +#elif defined(__GNUC__) || defined(__clang__) +# define XXH_FORCE_INLINE static __inline__ __attribute__((always_inline, unused)) +# define XXH_NO_INLINE static __attribute__((noinline)) +#elif defined(_MSC_VER) /* Visual Studio */ +# define XXH_FORCE_INLINE static __forceinline +# define XXH_NO_INLINE static __declspec(noinline) +#elif defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* C99 */ +# define XXH_FORCE_INLINE static inline +# define XXH_NO_INLINE static +#else +# define XXH_FORCE_INLINE static +# define XXH_NO_INLINE static +#endif + +#if XXH3_INLINE_SECRET +# define XXH3_WITH_SECRET_INLINE XXH_FORCE_INLINE +#else +# define XXH3_WITH_SECRET_INLINE XXH_NO_INLINE +#endif + + +/* ************************************* +* Debug +***************************************/ +/*! + * @ingroup tuning + * @def XXH_DEBUGLEVEL + * @brief Sets the debugging level. + * + * XXH_DEBUGLEVEL is expected to be defined externally, typically via the + * compiler's command line options. The value must be a number. + */ +#ifndef XXH_DEBUGLEVEL +# ifdef DEBUGLEVEL /* backwards compat */ +# define XXH_DEBUGLEVEL DEBUGLEVEL +# else +# define XXH_DEBUGLEVEL 0 +# endif +#endif + +#if (XXH_DEBUGLEVEL>=1) +# include /* note: can still be disabled with NDEBUG */ +# define XXH_ASSERT(c) assert(c) +#else +# if defined(__INTEL_COMPILER) +# define XXH_ASSERT(c) XXH_ASSUME((unsigned char) (c)) +# else +# define XXH_ASSERT(c) XXH_ASSUME(c) +# endif +#endif + +/* note: use after variable declarations */ +#ifndef XXH_STATIC_ASSERT +# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */ +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { _Static_assert((c),m); } while(0) +# elif defined(__cplusplus) && (__cplusplus >= 201103L) /* C++11 */ +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0) +# else +# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { struct xxh_sa { char x[(c) ? 1 : -1]; }; } while(0) +# endif +# define XXH_STATIC_ASSERT(c) XXH_STATIC_ASSERT_WITH_MESSAGE((c),#c) +#endif + +/*! + * @internal + * @def XXH_COMPILER_GUARD(var) + * @brief Used to prevent unwanted optimizations for @p var. + * + * It uses an empty GCC inline assembly statement with a register constraint + * which forces @p var into a general purpose register (eg eax, ebx, ecx + * on x86) and marks it as modified. + * + * This is used in a few places to avoid unwanted autovectorization (e.g. + * XXH32_round()). All vectorization we want is explicit via intrinsics, + * and _usually_ isn't wanted elsewhere. + * + * We also use it to prevent unwanted constant folding for AArch64 in + * XXH3_initCustomSecret_scalar(). + */ +#if defined(__GNUC__) || defined(__clang__) +# define XXH_COMPILER_GUARD(var) __asm__("" : "+r" (var)) +#else +# define XXH_COMPILER_GUARD(var) ((void)0) +#endif + +/* Specifically for NEON vectors which use the "w" constraint, on + * Clang. */ +#if defined(__clang__) && defined(__ARM_ARCH) && !defined(__wasm__) +# define XXH_COMPILER_GUARD_CLANG_NEON(var) __asm__("" : "+w" (var)) +#else +# define XXH_COMPILER_GUARD_CLANG_NEON(var) ((void)0) +#endif + +/* ************************************* +* Basic Types +***************************************/ +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# ifdef _AIX +# include +# else +# include +# endif + typedef uint8_t xxh_u8; +#else + typedef unsigned char xxh_u8; +#endif +typedef XXH32_hash_t xxh_u32; + +#ifdef XXH_OLD_NAMES +# warning "XXH_OLD_NAMES is planned to be removed starting v0.9. If the program depends on it, consider moving away from it by employing newer type names directly" +# define BYTE xxh_u8 +# define U8 xxh_u8 +# define U32 xxh_u32 +#endif + +/* *** Memory access *** */ + +/*! + * @internal + * @fn xxh_u32 XXH_read32(const void* ptr) + * @brief Reads an unaligned 32-bit integer from @p ptr in native endianness. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit native endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readLE32(const void* ptr) + * @brief Reads an unaligned 32-bit little endian integer from @p ptr. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit little endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readBE32(const void* ptr) + * @brief Reads an unaligned 32-bit big endian integer from @p ptr. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * + * @param ptr The pointer to read from. + * @return The 32-bit big endian integer from the bytes at @p ptr. + */ + +/*! + * @internal + * @fn xxh_u32 XXH_readLE32_align(const void* ptr, XXH_alignment align) + * @brief Like @ref XXH_readLE32(), but has an option for aligned reads. + * + * Affected by @ref XXH_FORCE_MEMORY_ACCESS. + * Note that when @ref XXH_FORCE_ALIGN_CHECK == 0, the @p align parameter is + * always @ref XXH_alignment::XXH_unaligned. + * + * @param ptr The pointer to read from. + * @param align Whether @p ptr is aligned. + * @pre + * If @p align == @ref XXH_alignment::XXH_aligned, @p ptr must be 4 byte + * aligned. + * @return The 32-bit little endian integer from the bytes at @p ptr. + */ + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE32 and XXH_readBE32. + */ +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* + * Force direct memory access. Only works on CPU which support unaligned memory + * access in hardware. + */ +static xxh_u32 XXH_read32(const void* memPtr) { return *(const xxh_u32*) memPtr; } + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* + * __attribute__((aligned(1))) is supported by gcc and clang. Originally the + * documentation claimed that it only increased the alignment, but actually it + * can decrease it on gcc, clang, and icc: + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502, + * https://gcc.godbolt.org/z/xYez1j67Y. + */ +#ifdef XXH_OLD_NAMES +typedef union { xxh_u32 u32; } __attribute__((packed)) unalign; +#endif +static xxh_u32 XXH_read32(const void* ptr) +{ + typedef __attribute__((aligned(1))) xxh_u32 xxh_unalign32; + return *((const xxh_unalign32*)ptr); +} + +#else + +/* + * Portable and safe solution. Generally efficient. + * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html + */ +static xxh_u32 XXH_read32(const void* memPtr) +{ + xxh_u32 val; + XXH_memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + + +/* *** Endianness *** */ + +/*! + * @ingroup tuning + * @def XXH_CPU_LITTLE_ENDIAN + * @brief Whether the target is little endian. + * + * Defined to 1 if the target is little endian, or 0 if it is big endian. + * It can be defined externally, for example on the compiler command line. + * + * If it is not defined, + * a runtime check (which is usually constant folded) is used instead. + * + * @note + * This is not necessarily defined to an integer constant. + * + * @see XXH_isLittleEndian() for the runtime check. + */ +#ifndef XXH_CPU_LITTLE_ENDIAN +/* + * Try to detect endianness automatically, to avoid the nonstandard behavior + * in `XXH_isLittleEndian()` + */ +# if defined(_WIN32) /* Windows is always little endian */ \ + || defined(__LITTLE_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) +# define XXH_CPU_LITTLE_ENDIAN 1 +# elif defined(__BIG_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# define XXH_CPU_LITTLE_ENDIAN 0 +# else +/*! + * @internal + * @brief Runtime check for @ref XXH_CPU_LITTLE_ENDIAN. + * + * Most compilers will constant fold this. + */ +static int XXH_isLittleEndian(void) +{ + /* + * Portable and well-defined behavior. + * Don't use static: it is detrimental to performance. + */ + const union { xxh_u32 u; xxh_u8 c[4]; } one = { 1 }; + return one.c[0]; +} +# define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian() +# endif +#endif + + + + +/* **************************************** +* Compiler-specific Functions and Macros +******************************************/ +#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +#ifdef __has_builtin +# define XXH_HAS_BUILTIN(x) __has_builtin(x) +#else +# define XXH_HAS_BUILTIN(x) 0 +#endif + + + +/* + * C23 and future versions have standard "unreachable()". + * Once it has been implemented reliably we can add it as an + * additional case: + * + * ``` + * #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= XXH_C23_VN) + * # include + * # ifdef unreachable + * # define XXH_UNREACHABLE() unreachable() + * # endif + * #endif + * ``` + * + * Note C++23 also has std::unreachable() which can be detected + * as follows: + * ``` + * #if defined(__cpp_lib_unreachable) && (__cpp_lib_unreachable >= 202202L) + * # include + * # define XXH_UNREACHABLE() std::unreachable() + * #endif + * ``` + * NB: `__cpp_lib_unreachable` is defined in the `` header. + * We don't use that as including `` in `extern "C"` blocks + * doesn't work on GCC12 + */ + +#if XXH_HAS_BUILTIN(__builtin_unreachable) +# define XXH_UNREACHABLE() __builtin_unreachable() + +#elif defined(_MSC_VER) +# define XXH_UNREACHABLE() __assume(0) + +#else +# define XXH_UNREACHABLE() +#endif + +#if XXH_HAS_BUILTIN(__builtin_assume) +# define XXH_ASSUME(c) __builtin_assume(c) +#else +# define XXH_ASSUME(c) if (!(c)) { XXH_UNREACHABLE(); } +#endif + +/*! + * @internal + * @def XXH_rotl32(x,r) + * @brief 32-bit rotate left. + * + * @param x The 32-bit integer to be rotated. + * @param r The number of bits to rotate. + * @pre + * @p r > 0 && @p r < 32 + * @note + * @p x and @p r may be evaluated multiple times. + * @return The rotated result. + */ +#if !defined(NO_CLANG_BUILTIN) && XXH_HAS_BUILTIN(__builtin_rotateleft32) \ + && XXH_HAS_BUILTIN(__builtin_rotateleft64) +# define XXH_rotl32 __builtin_rotateleft32 +# define XXH_rotl64 __builtin_rotateleft64 +/* Note: although _rotl exists for minGW (GCC under windows), performance seems poor */ +#elif defined(_MSC_VER) +# define XXH_rotl32(x,r) _rotl(x,r) +# define XXH_rotl64(x,r) _rotl64(x,r) +#else +# define XXH_rotl32(x,r) (((x) << (r)) | ((x) >> (32 - (r)))) +# define XXH_rotl64(x,r) (((x) << (r)) | ((x) >> (64 - (r)))) +#endif + +/*! + * @internal + * @fn xxh_u32 XXH_swap32(xxh_u32 x) + * @brief A 32-bit byteswap. + * + * @param x The 32-bit integer to byteswap. + * @return @p x, byteswapped. + */ +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap32 _byteswap_ulong +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap32 __builtin_bswap32 +#else +static xxh_u32 XXH_swap32 (xxh_u32 x) +{ + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff ); +} +#endif + + +/* *************************** +* Memory reads +*****************************/ + +/*! + * @internal + * @brief Enum to indicate whether a pointer is aligned. + */ +typedef enum { + XXH_aligned, /*!< Aligned */ + XXH_unaligned /*!< Possibly unaligned */ +} XXH_alignment; + +/* + * XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. + * + * This is ideal for older compilers which don't inline memcpy. + */ +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) + +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] + | ((xxh_u32)bytePtr[1] << 8) + | ((xxh_u32)bytePtr[2] << 16) + | ((xxh_u32)bytePtr[3] << 24); +} + +XXH_FORCE_INLINE xxh_u32 XXH_readBE32(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[3] + | ((xxh_u32)bytePtr[2] << 8) + | ((xxh_u32)bytePtr[1] << 16) + | ((xxh_u32)bytePtr[0] << 24); +} + +#else +XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); +} + +static xxh_u32 XXH_readBE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); +} +#endif + +XXH_FORCE_INLINE xxh_u32 +XXH_readLE32_align(const void* ptr, XXH_alignment align) +{ + if (align==XXH_unaligned) { + return XXH_readLE32(ptr); + } else { + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u32*)ptr : XXH_swap32(*(const xxh_u32*)ptr); + } +} + + +/* ************************************* +* Misc +***************************************/ +/*! @ingroup public */ +XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } + + +/* ******************************************************************* +* 32-bit hash functions +*********************************************************************/ +/*! + * @} + * @defgroup XXH32_impl XXH32 implementation + * @ingroup impl + * + * Details on the XXH32 implementation. + * @{ + */ + /* #define instead of static const, to be used as initializers */ +#define XXH_PRIME32_1 0x9E3779B1U /*!< 0b10011110001101110111100110110001 */ +#define XXH_PRIME32_2 0x85EBCA77U /*!< 0b10000101111010111100101001110111 */ +#define XXH_PRIME32_3 0xC2B2AE3DU /*!< 0b11000010101100101010111000111101 */ +#define XXH_PRIME32_4 0x27D4EB2FU /*!< 0b00100111110101001110101100101111 */ +#define XXH_PRIME32_5 0x165667B1U /*!< 0b00010110010101100110011110110001 */ + +#ifdef XXH_OLD_NAMES +# define PRIME32_1 XXH_PRIME32_1 +# define PRIME32_2 XXH_PRIME32_2 +# define PRIME32_3 XXH_PRIME32_3 +# define PRIME32_4 XXH_PRIME32_4 +# define PRIME32_5 XXH_PRIME32_5 +#endif + +/*! + * @internal + * @brief Normal stripe processing routine. + * + * This shuffles the bits so that any bit from @p input impacts several bits in + * @p acc. + * + * @param acc The accumulator lane. + * @param input The stripe of input to mix. + * @return The mixed accumulator lane. + */ +static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input) +{ + acc += input * XXH_PRIME32_2; + acc = XXH_rotl32(acc, 13); + acc *= XXH_PRIME32_1; +#if (defined(__SSE4_1__) || defined(__aarch64__) || defined(__wasm_simd128__)) && !defined(XXH_ENABLE_AUTOVECTORIZE) + /* + * UGLY HACK: + * A compiler fence is the only thing that prevents GCC and Clang from + * autovectorizing the XXH32 loop (pragmas and attributes don't work for some + * reason) without globally disabling SSE4.1. + * + * The reason we want to avoid vectorization is because despite working on + * 4 integers at a time, there are multiple factors slowing XXH32 down on + * SSE4: + * - There's a ridiculous amount of lag from pmulld (10 cycles of latency on + * newer chips!) making it slightly slower to multiply four integers at + * once compared to four integers independently. Even when pmulld was + * fastest, Sandy/Ivy Bridge, it is still not worth it to go into SSE + * just to multiply unless doing a long operation. + * + * - Four instructions are required to rotate, + * movqda tmp, v // not required with VEX encoding + * pslld tmp, 13 // tmp <<= 13 + * psrld v, 19 // x >>= 19 + * por v, tmp // x |= tmp + * compared to one for scalar: + * roll v, 13 // reliably fast across the board + * shldl v, v, 13 // Sandy Bridge and later prefer this for some reason + * + * - Instruction level parallelism is actually more beneficial here because + * the SIMD actually serializes this operation: While v1 is rotating, v2 + * can load data, while v3 can multiply. SSE forces them to operate + * together. + * + * This is also enabled on AArch64, as Clang is *very aggressive* in vectorizing + * the loop. NEON is only faster on the A53, and with the newer cores, it is less + * than half the speed. + * + * Additionally, this is used on WASM SIMD128 because it JITs to the same + * SIMD instructions and has the same issue. + */ + XXH_COMPILER_GUARD(acc); +#endif + return acc; +} + +/*! + * @internal + * @brief Mixes all bits to finalize the hash. + * + * The final mix ensures that all input bits have a chance to impact any bit in + * the output digest, resulting in an unbiased distribution. + * + * @param hash The hash to avalanche. + * @return The avalanched hash. + */ +static xxh_u32 XXH32_avalanche(xxh_u32 hash) +{ + hash ^= hash >> 15; + hash *= XXH_PRIME32_2; + hash ^= hash >> 13; + hash *= XXH_PRIME32_3; + hash ^= hash >> 16; + return hash; +} + +#define XXH_get32bits(p) XXH_readLE32_align(p, align) + +/*! + * @internal + * @brief Processes the last 0-15 bytes of @p ptr. + * + * There may be up to 15 bytes remaining to consume from the input. + * This final stage will digest them to ensure that all input bytes are present + * in the final mix. + * + * @param hash The hash to finalize. + * @param ptr The pointer to the remaining input. + * @param len The remaining length, modulo 16. + * @param align Whether @p ptr is aligned. + * @return The finalized hash. + * @see XXH64_finalize(). + */ +static XXH_PUREF xxh_u32 +XXH32_finalize(xxh_u32 hash, const xxh_u8* ptr, size_t len, XXH_alignment align) +{ +#define XXH_PROCESS1 do { \ + hash += (*ptr++) * XXH_PRIME32_5; \ + hash = XXH_rotl32(hash, 11) * XXH_PRIME32_1; \ +} while (0) + +#define XXH_PROCESS4 do { \ + hash += XXH_get32bits(ptr) * XXH_PRIME32_3; \ + ptr += 4; \ + hash = XXH_rotl32(hash, 17) * XXH_PRIME32_4; \ +} while (0) + + if (ptr==NULL) XXH_ASSERT(len == 0); + + /* Compact rerolled version; generally faster */ + if (!XXH32_ENDJMP) { + len &= 15; + while (len >= 4) { + XXH_PROCESS4; + len -= 4; + } + while (len > 0) { + XXH_PROCESS1; + --len; + } + return XXH32_avalanche(hash); + } else { + switch(len&15) /* or switch(bEnd - p) */ { + case 12: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 8: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 4: XXH_PROCESS4; + return XXH32_avalanche(hash); + + case 13: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 9: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 5: XXH_PROCESS4; + XXH_PROCESS1; + return XXH32_avalanche(hash); + + case 14: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 10: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 6: XXH_PROCESS4; + XXH_PROCESS1; + XXH_PROCESS1; + return XXH32_avalanche(hash); + + case 15: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 11: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 7: XXH_PROCESS4; + XXH_FALLTHROUGH; /* fallthrough */ + case 3: XXH_PROCESS1; + XXH_FALLTHROUGH; /* fallthrough */ + case 2: XXH_PROCESS1; + XXH_FALLTHROUGH; /* fallthrough */ + case 1: XXH_PROCESS1; + XXH_FALLTHROUGH; /* fallthrough */ + case 0: return XXH32_avalanche(hash); + } + XXH_ASSERT(0); + return hash; /* reaching this point is deemed impossible */ + } +} + +#ifdef XXH_OLD_NAMES +# define PROCESS1 XXH_PROCESS1 +# define PROCESS4 XXH_PROCESS4 +#else +# undef XXH_PROCESS1 +# undef XXH_PROCESS4 +#endif + +/*! + * @internal + * @brief The implementation for @ref XXH32(). + * + * @param input , len , seed Directly passed from @ref XXH32(). + * @param align Whether @p input is aligned. + * @return The calculated hash. + */ +XXH_FORCE_INLINE XXH_PUREF xxh_u32 +XXH32_endian_align(const xxh_u8* input, size_t len, xxh_u32 seed, XXH_alignment align) +{ + xxh_u32 h32; + + if (input==NULL) XXH_ASSERT(len == 0); + + if (len>=16) { + const xxh_u8* const bEnd = input + len; + const xxh_u8* const limit = bEnd - 15; + xxh_u32 v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2; + xxh_u32 v2 = seed + XXH_PRIME32_2; + xxh_u32 v3 = seed + 0; + xxh_u32 v4 = seed - XXH_PRIME32_1; + + do { + v1 = XXH32_round(v1, XXH_get32bits(input)); input += 4; + v2 = XXH32_round(v2, XXH_get32bits(input)); input += 4; + v3 = XXH32_round(v3, XXH_get32bits(input)); input += 4; + v4 = XXH32_round(v4, XXH_get32bits(input)); input += 4; + } while (input < limit); + + h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); + } else { + h32 = seed + XXH_PRIME32_5; + } + + h32 += (xxh_u32)len; + + return XXH32_finalize(h32, input, len&15, align); +} + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t len, XXH32_hash_t seed) +{ +#if !defined(XXH_NO_STREAM) && XXH_SIZE_OPT >= 2 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH32_state_t state; + XXH32_reset(&state, seed); + XXH32_update(&state, (const xxh_u8*)input, len); + return XXH32_digest(&state); +#else + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ + return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); + } } + + return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); +#endif +} + + + +/******* Hash streaming *******/ +#ifndef XXH_NO_STREAM +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) +{ + return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); +} +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState) +{ + XXH_memcpy(dstState, srcState, sizeof(*dstState)); +} + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, XXH32_hash_t seed) +{ + XXH_ASSERT(statePtr != NULL); + memset(statePtr, 0, sizeof(*statePtr)); + statePtr->v[0] = seed + XXH_PRIME32_1 + XXH_PRIME32_2; + statePtr->v[1] = seed + XXH_PRIME32_2; + statePtr->v[2] = seed + 0; + statePtr->v[3] = seed - XXH_PRIME32_1; + return XXH_OK; +} + + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH_errorcode +XXH32_update(XXH32_state_t* state, const void* input, size_t len) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + { const xxh_u8* p = (const xxh_u8*)input; + const xxh_u8* const bEnd = p + len; + + state->total_len_32 += (XXH32_hash_t)len; + state->large_len |= (XXH32_hash_t)((len>=16) | (state->total_len_32>=16)); + + if (state->memsize + len < 16) { /* fill in tmp buffer */ + XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, len); + state->memsize += (XXH32_hash_t)len; + return XXH_OK; + } + + if (state->memsize) { /* some data left from previous update */ + XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, 16-state->memsize); + { const xxh_u32* p32 = state->mem32; + state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p32)); p32++; + state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p32)); p32++; + state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p32)); p32++; + state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p32)); + } + p += 16-state->memsize; + state->memsize = 0; + } + + if (p <= bEnd-16) { + const xxh_u8* const limit = bEnd - 16; + + do { + state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p)); p+=4; + state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p)); p+=4; + state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p)); p+=4; + state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p)); p+=4; + } while (p<=limit); + + } + + if (p < bEnd) { + XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + } + + return XXH_OK; +} + + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t* state) +{ + xxh_u32 h32; + + if (state->large_len) { + h32 = XXH_rotl32(state->v[0], 1) + + XXH_rotl32(state->v[1], 7) + + XXH_rotl32(state->v[2], 12) + + XXH_rotl32(state->v[3], 18); + } else { + h32 = state->v[2] /* == seed */ + XXH_PRIME32_5; + } + + h32 += state->total_len_32; + + return XXH32_finalize(h32, (const xxh_u8*)state->mem32, state->memsize, XXH_aligned); +} +#endif /* !XXH_NO_STREAM */ + +/******* Canonical representation *******/ + +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); + XXH_memcpy(dst, &hash, sizeof(*dst)); +} +/*! @ingroup XXH32_family */ +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) +{ + return XXH_readBE32(src); +} + + +#ifndef XXH_NO_LONG_LONG + +/* ******************************************************************* +* 64-bit hash functions +*********************************************************************/ +/*! + * @} + * @ingroup impl + * @{ + */ +/******* Memory access *******/ + +typedef XXH64_hash_t xxh_u64; + +#ifdef XXH_OLD_NAMES +# define U64 xxh_u64 +#endif + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) +/* + * Manual byteshift. Best for old compilers which don't inline memcpy. + * We actually directly use XXH_readLE64 and XXH_readBE64. + */ +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ +static xxh_u64 XXH_read64(const void* memPtr) +{ + return *(const xxh_u64*) memPtr; +} + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* + * __attribute__((aligned(1))) is supported by gcc and clang. Originally the + * documentation claimed that it only increased the alignment, but actually it + * can decrease it on gcc, clang, and icc: + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502, + * https://gcc.godbolt.org/z/xYez1j67Y. + */ +#ifdef XXH_OLD_NAMES +typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) unalign64; +#endif +static xxh_u64 XXH_read64(const void* ptr) +{ + typedef __attribute__((aligned(1))) xxh_u64 xxh_unalign64; + return *((const xxh_unalign64*)ptr); +} + +#else + +/* + * Portable and safe solution. Generally efficient. + * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html + */ +static xxh_u64 XXH_read64(const void* memPtr) +{ + xxh_u64 val; + XXH_memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap64 _byteswap_uint64 +#elif XXH_GCC_VERSION >= 403 +# define XXH_swap64 __builtin_bswap64 +#else +static xxh_u64 XXH_swap64(xxh_u64 x) +{ + return ((x << 56) & 0xff00000000000000ULL) | + ((x << 40) & 0x00ff000000000000ULL) | + ((x << 24) & 0x0000ff0000000000ULL) | + ((x << 8) & 0x000000ff00000000ULL) | + ((x >> 8) & 0x00000000ff000000ULL) | + ((x >> 24) & 0x0000000000ff0000ULL) | + ((x >> 40) & 0x000000000000ff00ULL) | + ((x >> 56) & 0x00000000000000ffULL); +} +#endif + + +/* XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. */ +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3)) + +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[0] + | ((xxh_u64)bytePtr[1] << 8) + | ((xxh_u64)bytePtr[2] << 16) + | ((xxh_u64)bytePtr[3] << 24) + | ((xxh_u64)bytePtr[4] << 32) + | ((xxh_u64)bytePtr[5] << 40) + | ((xxh_u64)bytePtr[6] << 48) + | ((xxh_u64)bytePtr[7] << 56); +} + +XXH_FORCE_INLINE xxh_u64 XXH_readBE64(const void* memPtr) +{ + const xxh_u8* bytePtr = (const xxh_u8 *)memPtr; + return bytePtr[7] + | ((xxh_u64)bytePtr[6] << 8) + | ((xxh_u64)bytePtr[5] << 16) + | ((xxh_u64)bytePtr[4] << 24) + | ((xxh_u64)bytePtr[3] << 32) + | ((xxh_u64)bytePtr[2] << 40) + | ((xxh_u64)bytePtr[1] << 48) + | ((xxh_u64)bytePtr[0] << 56); +} + +#else +XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); +} + +static xxh_u64 XXH_readBE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); +} +#endif + +XXH_FORCE_INLINE xxh_u64 +XXH_readLE64_align(const void* ptr, XXH_alignment align) +{ + if (align==XXH_unaligned) + return XXH_readLE64(ptr); + else + return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u64*)ptr : XXH_swap64(*(const xxh_u64*)ptr); +} + + +/******* xxh64 *******/ +/*! + * @} + * @defgroup XXH64_impl XXH64 implementation + * @ingroup impl + * + * Details on the XXH64 implementation. + * @{ + */ +/* #define rather that static const, to be used as initializers */ +#define XXH_PRIME64_1 0x9E3779B185EBCA87ULL /*!< 0b1001111000110111011110011011000110000101111010111100101010000111 */ +#define XXH_PRIME64_2 0xC2B2AE3D27D4EB4FULL /*!< 0b1100001010110010101011100011110100100111110101001110101101001111 */ +#define XXH_PRIME64_3 0x165667B19E3779F9ULL /*!< 0b0001011001010110011001111011000110011110001101110111100111111001 */ +#define XXH_PRIME64_4 0x85EBCA77C2B2AE63ULL /*!< 0b1000010111101011110010100111011111000010101100101010111001100011 */ +#define XXH_PRIME64_5 0x27D4EB2F165667C5ULL /*!< 0b0010011111010100111010110010111100010110010101100110011111000101 */ + +#ifdef XXH_OLD_NAMES +# define PRIME64_1 XXH_PRIME64_1 +# define PRIME64_2 XXH_PRIME64_2 +# define PRIME64_3 XXH_PRIME64_3 +# define PRIME64_4 XXH_PRIME64_4 +# define PRIME64_5 XXH_PRIME64_5 +#endif + +/*! @copydoc XXH32_round */ +static xxh_u64 XXH64_round(xxh_u64 acc, xxh_u64 input) +{ + acc += input * XXH_PRIME64_2; + acc = XXH_rotl64(acc, 31); + acc *= XXH_PRIME64_1; +#if (defined(__AVX512F__)) && !defined(XXH_ENABLE_AUTOVECTORIZE) + /* + * DISABLE AUTOVECTORIZATION: + * A compiler fence is used to prevent GCC and Clang from + * autovectorizing the XXH64 loop (pragmas and attributes don't work for some + * reason) without globally disabling AVX512. + * + * Autovectorization of XXH64 tends to be detrimental, + * though the exact outcome may change depending on exact cpu and compiler version. + * For information, it has been reported as detrimental for Skylake-X, + * but possibly beneficial for Zen4. + * + * The default is to disable auto-vectorization, + * but you can select to enable it instead using `XXH_ENABLE_AUTOVECTORIZE` build variable. + */ + XXH_COMPILER_GUARD(acc); +#endif + return acc; +} + +static xxh_u64 XXH64_mergeRound(xxh_u64 acc, xxh_u64 val) +{ + val = XXH64_round(0, val); + acc ^= val; + acc = acc * XXH_PRIME64_1 + XXH_PRIME64_4; + return acc; +} + +/*! @copydoc XXH32_avalanche */ +static xxh_u64 XXH64_avalanche(xxh_u64 hash) +{ + hash ^= hash >> 33; + hash *= XXH_PRIME64_2; + hash ^= hash >> 29; + hash *= XXH_PRIME64_3; + hash ^= hash >> 32; + return hash; +} + + +#define XXH_get64bits(p) XXH_readLE64_align(p, align) + +/*! + * @internal + * @brief Processes the last 0-31 bytes of @p ptr. + * + * There may be up to 31 bytes remaining to consume from the input. + * This final stage will digest them to ensure that all input bytes are present + * in the final mix. + * + * @param hash The hash to finalize. + * @param ptr The pointer to the remaining input. + * @param len The remaining length, modulo 32. + * @param align Whether @p ptr is aligned. + * @return The finalized hash + * @see XXH32_finalize(). + */ +static XXH_PUREF xxh_u64 +XXH64_finalize(xxh_u64 hash, const xxh_u8* ptr, size_t len, XXH_alignment align) +{ + if (ptr==NULL) XXH_ASSERT(len == 0); + len &= 31; + while (len >= 8) { + xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr)); + ptr += 8; + hash ^= k1; + hash = XXH_rotl64(hash,27) * XXH_PRIME64_1 + XXH_PRIME64_4; + len -= 8; + } + if (len >= 4) { + hash ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1; + ptr += 4; + hash = XXH_rotl64(hash, 23) * XXH_PRIME64_2 + XXH_PRIME64_3; + len -= 4; + } + while (len > 0) { + hash ^= (*ptr++) * XXH_PRIME64_5; + hash = XXH_rotl64(hash, 11) * XXH_PRIME64_1; + --len; + } + return XXH64_avalanche(hash); +} + +#ifdef XXH_OLD_NAMES +# define PROCESS1_64 XXH_PROCESS1_64 +# define PROCESS4_64 XXH_PROCESS4_64 +# define PROCESS8_64 XXH_PROCESS8_64 +#else +# undef XXH_PROCESS1_64 +# undef XXH_PROCESS4_64 +# undef XXH_PROCESS8_64 +#endif + +/*! + * @internal + * @brief The implementation for @ref XXH64(). + * + * @param input , len , seed Directly passed from @ref XXH64(). + * @param align Whether @p input is aligned. + * @return The calculated hash. + */ +XXH_FORCE_INLINE XXH_PUREF xxh_u64 +XXH64_endian_align(const xxh_u8* input, size_t len, xxh_u64 seed, XXH_alignment align) +{ + xxh_u64 h64; + if (input==NULL) XXH_ASSERT(len == 0); + + if (len>=32) { + const xxh_u8* const bEnd = input + len; + const xxh_u8* const limit = bEnd - 31; + xxh_u64 v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2; + xxh_u64 v2 = seed + XXH_PRIME64_2; + xxh_u64 v3 = seed + 0; + xxh_u64 v4 = seed - XXH_PRIME64_1; + + do { + v1 = XXH64_round(v1, XXH_get64bits(input)); input+=8; + v2 = XXH64_round(v2, XXH_get64bits(input)); input+=8; + v3 = XXH64_round(v3, XXH_get64bits(input)); input+=8; + v4 = XXH64_round(v4, XXH_get64bits(input)); input+=8; + } while (input= 2 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH64_state_t state; + XXH64_reset(&state, seed); + XXH64_update(&state, (const xxh_u8*)input, len); + return XXH64_digest(&state); +#else + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */ + return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_aligned); + } } + + return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned); + +#endif +} + +/******* Hash Streaming *******/ +#ifndef XXH_NO_STREAM +/*! @ingroup XXH64_family*/ +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) +{ + return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); +} +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dstState, const XXH64_state_t* srcState) +{ + XXH_memcpy(dstState, srcState, sizeof(*dstState)); +} + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed) +{ + XXH_ASSERT(statePtr != NULL); + memset(statePtr, 0, sizeof(*statePtr)); + statePtr->v[0] = seed + XXH_PRIME64_1 + XXH_PRIME64_2; + statePtr->v[1] = seed + XXH_PRIME64_2; + statePtr->v[2] = seed + 0; + statePtr->v[3] = seed - XXH_PRIME64_1; + return XXH_OK; +} + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH_errorcode +XXH64_update (XXH_NOESCAPE XXH64_state_t* state, XXH_NOESCAPE const void* input, size_t len) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + { const xxh_u8* p = (const xxh_u8*)input; + const xxh_u8* const bEnd = p + len; + + state->total_len += len; + + if (state->memsize + len < 32) { /* fill in tmp buffer */ + XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, len); + state->memsize += (xxh_u32)len; + return XXH_OK; + } + + if (state->memsize) { /* tmp buffer is full */ + XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, 32-state->memsize); + state->v[0] = XXH64_round(state->v[0], XXH_readLE64(state->mem64+0)); + state->v[1] = XXH64_round(state->v[1], XXH_readLE64(state->mem64+1)); + state->v[2] = XXH64_round(state->v[2], XXH_readLE64(state->mem64+2)); + state->v[3] = XXH64_round(state->v[3], XXH_readLE64(state->mem64+3)); + p += 32 - state->memsize; + state->memsize = 0; + } + + if (p+32 <= bEnd) { + const xxh_u8* const limit = bEnd - 32; + + do { + state->v[0] = XXH64_round(state->v[0], XXH_readLE64(p)); p+=8; + state->v[1] = XXH64_round(state->v[1], XXH_readLE64(p)); p+=8; + state->v[2] = XXH64_round(state->v[2], XXH_readLE64(p)); p+=8; + state->v[3] = XXH64_round(state->v[3], XXH_readLE64(p)); p+=8; + } while (p<=limit); + + } + + if (p < bEnd) { + XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + } + + return XXH_OK; +} + + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64_digest(XXH_NOESCAPE const XXH64_state_t* state) +{ + xxh_u64 h64; + + if (state->total_len >= 32) { + h64 = XXH_rotl64(state->v[0], 1) + XXH_rotl64(state->v[1], 7) + XXH_rotl64(state->v[2], 12) + XXH_rotl64(state->v[3], 18); + h64 = XXH64_mergeRound(h64, state->v[0]); + h64 = XXH64_mergeRound(h64, state->v[1]); + h64 = XXH64_mergeRound(h64, state->v[2]); + h64 = XXH64_mergeRound(h64, state->v[3]); + } else { + h64 = state->v[2] /*seed*/ + XXH_PRIME64_5; + } + + h64 += (xxh_u64) state->total_len; + + return XXH64_finalize(h64, (const xxh_u8*)state->mem64, (size_t)state->total_len, XXH_aligned); +} +#endif /* !XXH_NO_STREAM */ + +/******* Canonical representation *******/ + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); + XXH_memcpy(dst, &hash, sizeof(*dst)); +} + +/*! @ingroup XXH64_family */ +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src) +{ + return XXH_readBE64(src); +} + +#ifndef XXH_NO_XXH3 + +/* ********************************************************************* +* XXH3 +* New generation hash designed for speed on small keys and vectorization +************************************************************************ */ +/*! + * @} + * @defgroup XXH3_impl XXH3 implementation + * @ingroup impl + * @{ + */ + +/* === Compiler specifics === */ + +#if ((defined(sun) || defined(__sun)) && __cplusplus) /* Solaris includes __STDC_VERSION__ with C++. Tested with GCC 5.5 */ +# define XXH_RESTRICT /* disable */ +#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* >= C99 */ +# define XXH_RESTRICT restrict +#elif (defined (__GNUC__) && ((__GNUC__ > 3) || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1))) \ + || (defined (__clang__)) \ + || (defined (_MSC_VER) && (_MSC_VER >= 1400)) \ + || (defined (__INTEL_COMPILER) && (__INTEL_COMPILER >= 1300)) +/* + * There are a LOT more compilers that recognize __restrict but this + * covers the major ones. + */ +# define XXH_RESTRICT __restrict +#else +# define XXH_RESTRICT /* disable */ +#endif + +#if (defined(__GNUC__) && (__GNUC__ >= 3)) \ + || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) \ + || defined(__clang__) +# define XXH_likely(x) __builtin_expect(x, 1) +# define XXH_unlikely(x) __builtin_expect(x, 0) +#else +# define XXH_likely(x) (x) +# define XXH_unlikely(x) (x) +#endif + +#ifndef XXH_HAS_INCLUDE +# ifdef __has_include +/* + * Not defined as XXH_HAS_INCLUDE(x) (function-like) because + * this causes segfaults in Apple Clang 4.2 (on Mac OS X 10.7 Lion) + */ +# define XXH_HAS_INCLUDE __has_include +# else +# define XXH_HAS_INCLUDE(x) 0 +# endif +#endif + +#if defined(__GNUC__) || defined(__clang__) +# if defined(__ARM_FEATURE_SVE) +# include +# endif +# if defined(__ARM_NEON__) || defined(__ARM_NEON) \ + || (defined(_M_ARM) && _M_ARM >= 7) \ + || defined(_M_ARM64) || defined(_M_ARM64EC) \ + || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE()) /* WASM SIMD128 via SIMDe */ +# define inline __inline__ /* circumvent a clang bug */ +# include +# undef inline +# elif defined(__AVX2__) +# include +# elif defined(__SSE2__) +# include +# endif +#endif + +#if defined(_MSC_VER) +# include +#endif + +/* + * One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while + * remaining a true 64-bit/128-bit hash function. + * + * This is done by prioritizing a subset of 64-bit operations that can be + * emulated without too many steps on the average 32-bit machine. + * + * For example, these two lines seem similar, and run equally fast on 64-bit: + * + * xxh_u64 x; + * x ^= (x >> 47); // good + * x ^= (x >> 13); // bad + * + * However, to a 32-bit machine, there is a major difference. + * + * x ^= (x >> 47) looks like this: + * + * x.lo ^= (x.hi >> (47 - 32)); + * + * while x ^= (x >> 13) looks like this: + * + * // note: funnel shifts are not usually cheap. + * x.lo ^= (x.lo >> 13) | (x.hi << (32 - 13)); + * x.hi ^= (x.hi >> 13); + * + * The first one is significantly faster than the second, simply because the + * shift is larger than 32. This means: + * - All the bits we need are in the upper 32 bits, so we can ignore the lower + * 32 bits in the shift. + * - The shift result will always fit in the lower 32 bits, and therefore, + * we can ignore the upper 32 bits in the xor. + * + * Thanks to this optimization, XXH3 only requires these features to be efficient: + * + * - Usable unaligned access + * - A 32-bit or 64-bit ALU + * - If 32-bit, a decent ADC instruction + * - A 32 or 64-bit multiply with a 64-bit result + * - For the 128-bit variant, a decent byteswap helps short inputs. + * + * The first two are already required by XXH32, and almost all 32-bit and 64-bit + * platforms which can run XXH32 can run XXH3 efficiently. + * + * Thumb-1, the classic 16-bit only subset of ARM's instruction set, is one + * notable exception. + * + * First of all, Thumb-1 lacks support for the UMULL instruction which + * performs the important long multiply. This means numerous __aeabi_lmul + * calls. + * + * Second of all, the 8 functional registers are just not enough. + * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic need + * Lo registers, and this shuffling results in thousands more MOVs than A32. + * + * A32 and T32 don't have this limitation. They can access all 14 registers, + * do a 32->64 multiply with UMULL, and the flexible operand allowing free + * shifts is helpful, too. + * + * Therefore, we do a quick sanity check. + * + * If compiling Thumb-1 for a target which supports ARM instructions, we will + * emit a warning, as it is not a "sane" platform to compile for. + * + * Usually, if this happens, it is because of an accident and you probably need + * to specify -march, as you likely meant to compile for a newer architecture. + * + * Credit: large sections of the vectorial and asm source code paths + * have been contributed by @easyaspi314 + */ +#if defined(__thumb__) && !defined(__thumb2__) && defined(__ARM_ARCH_ISA_ARM) +# warning "XXH3 is highly inefficient without ARM or Thumb-2." +#endif + +/* ========================================== + * Vectorization detection + * ========================================== */ + +#ifdef XXH_DOXYGEN +/*! + * @ingroup tuning + * @brief Overrides the vectorization implementation chosen for XXH3. + * + * Can be defined to 0 to disable SIMD or any of the values mentioned in + * @ref XXH_VECTOR_TYPE. + * + * If this is not defined, it uses predefined macros to determine the best + * implementation. + */ +# define XXH_VECTOR XXH_SCALAR +/*! + * @ingroup tuning + * @brief Possible values for @ref XXH_VECTOR. + * + * Note that these are actually implemented as macros. + * + * If this is not defined, it is detected automatically. + * internal macro XXH_X86DISPATCH overrides this. + */ +enum XXH_VECTOR_TYPE /* fake enum */ { + XXH_SCALAR = 0, /*!< Portable scalar version */ + XXH_SSE2 = 1, /*!< + * SSE2 for Pentium 4, Opteron, all x86_64. + * + * @note SSE2 is also guaranteed on Windows 10, macOS, and + * Android x86. + */ + XXH_AVX2 = 2, /*!< AVX2 for Haswell and Bulldozer */ + XXH_AVX512 = 3, /*!< AVX512 for Skylake and Icelake */ + XXH_NEON = 4, /*!< + * NEON for most ARMv7-A, all AArch64, and WASM SIMD128 + * via the SIMDeverywhere polyfill provided with the + * Emscripten SDK. + */ + XXH_VSX = 5, /*!< VSX and ZVector for POWER8/z13 (64-bit) */ + XXH_SVE = 6, /*!< SVE for some ARMv8-A and ARMv9-A */ +}; +/*! + * @ingroup tuning + * @brief Selects the minimum alignment for XXH3's accumulators. + * + * When using SIMD, this should match the alignment required for said vector + * type, so, for example, 32 for AVX2. + * + * Default: Auto detected. + */ +# define XXH_ACC_ALIGN 8 +#endif + +/* Actual definition */ +#ifndef XXH_DOXYGEN +# define XXH_SCALAR 0 +# define XXH_SSE2 1 +# define XXH_AVX2 2 +# define XXH_AVX512 3 +# define XXH_NEON 4 +# define XXH_VSX 5 +# define XXH_SVE 6 +#endif + +#ifndef XXH_VECTOR /* can be defined on command line */ +# if defined(__ARM_FEATURE_SVE) +# define XXH_VECTOR XXH_SVE +# elif ( \ + defined(__ARM_NEON__) || defined(__ARM_NEON) /* gcc */ \ + || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_ARM64EC) /* msvc */ \ + || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE()) /* wasm simd128 via SIMDe */ \ + ) && ( \ + defined(_WIN32) || defined(__LITTLE_ENDIAN__) /* little endian only */ \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) \ + ) +# define XXH_VECTOR XXH_NEON +# elif defined(__AVX512F__) +# define XXH_VECTOR XXH_AVX512 +# elif defined(__AVX2__) +# define XXH_VECTOR XXH_AVX2 +# elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP == 2)) +# define XXH_VECTOR XXH_SSE2 +# elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) \ + || (defined(__s390x__) && defined(__VEC__)) \ + && defined(__GNUC__) /* TODO: IBM XL */ +# define XXH_VECTOR XXH_VSX +# else +# define XXH_VECTOR XXH_SCALAR +# endif +#endif + +/* __ARM_FEATURE_SVE is only supported by GCC & Clang. */ +#if (XXH_VECTOR == XXH_SVE) && !defined(__ARM_FEATURE_SVE) +# ifdef _MSC_VER +# pragma warning(once : 4606) +# else +# warning "__ARM_FEATURE_SVE isn't supported. Use SCALAR instead." +# endif +# undef XXH_VECTOR +# define XXH_VECTOR XXH_SCALAR +#endif + +/* + * Controls the alignment of the accumulator, + * for compatibility with aligned vector loads, which are usually faster. + */ +#ifndef XXH_ACC_ALIGN +# if defined(XXH_X86DISPATCH) +# define XXH_ACC_ALIGN 64 /* for compatibility with avx512 */ +# elif XXH_VECTOR == XXH_SCALAR /* scalar */ +# define XXH_ACC_ALIGN 8 +# elif XXH_VECTOR == XXH_SSE2 /* sse2 */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_AVX2 /* avx2 */ +# define XXH_ACC_ALIGN 32 +# elif XXH_VECTOR == XXH_NEON /* neon */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_VSX /* vsx */ +# define XXH_ACC_ALIGN 16 +# elif XXH_VECTOR == XXH_AVX512 /* avx512 */ +# define XXH_ACC_ALIGN 64 +# elif XXH_VECTOR == XXH_SVE /* sve */ +# define XXH_ACC_ALIGN 64 +# endif +#endif + +#if defined(XXH_X86DISPATCH) || XXH_VECTOR == XXH_SSE2 \ + || XXH_VECTOR == XXH_AVX2 || XXH_VECTOR == XXH_AVX512 +# define XXH_SEC_ALIGN XXH_ACC_ALIGN +#elif XXH_VECTOR == XXH_SVE +# define XXH_SEC_ALIGN XXH_ACC_ALIGN +#else +# define XXH_SEC_ALIGN 8 +#endif + +#if defined(__GNUC__) || defined(__clang__) +# define XXH_ALIASING __attribute__((may_alias)) +#else +# define XXH_ALIASING /* nothing */ +#endif + +/* + * UGLY HACK: + * GCC usually generates the best code with -O3 for xxHash. + * + * However, when targeting AVX2, it is overzealous in its unrolling resulting + * in code roughly 3/4 the speed of Clang. + * + * There are other issues, such as GCC splitting _mm256_loadu_si256 into + * _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization which + * only applies to Sandy and Ivy Bridge... which don't even support AVX2. + * + * That is why when compiling the AVX2 version, it is recommended to use either + * -O2 -mavx2 -march=haswell + * or + * -O2 -mavx2 -mno-avx256-split-unaligned-load + * for decent performance, or to use Clang instead. + * + * Fortunately, we can control the first one with a pragma that forces GCC into + * -O2, but the other one we can't control without "failed to inline always + * inline function due to target mismatch" warnings. + */ +#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \ + && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__OPTIMIZE__) && XXH_SIZE_OPT <= 0 /* respect -O0 and -Os */ +# pragma GCC push_options +# pragma GCC optimize("-O2") +#endif + +#if XXH_VECTOR == XXH_NEON + +/* + * UGLY HACK: While AArch64 GCC on Linux does not seem to care, on macOS, GCC -O3 + * optimizes out the entire hashLong loop because of the aliasing violation. + * + * However, GCC is also inefficient at load-store optimization with vld1q/vst1q, + * so the only option is to mark it as aliasing. + */ +typedef uint64x2_t xxh_aliasing_uint64x2_t XXH_ALIASING; + +/*! + * @internal + * @brief `vld1q_u64` but faster and alignment-safe. + * + * On AArch64, unaligned access is always safe, but on ARMv7-a, it is only + * *conditionally* safe (`vld1` has an alignment bit like `movdq[ua]` in x86). + * + * GCC for AArch64 sees `vld1q_u8` as an intrinsic instead of a load, so it + * prohibits load-store optimizations. Therefore, a direct dereference is used. + * + * Otherwise, `vld1q_u8` is used with `vreinterpretq_u8_u64` to do a safe + * unaligned load. + */ +#if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__) +XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr) /* silence -Wcast-align */ +{ + return *(xxh_aliasing_uint64x2_t const *)ptr; +} +#else +XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr) +{ + return vreinterpretq_u64_u8(vld1q_u8((uint8_t const*)ptr)); +} +#endif + +/*! + * @internal + * @brief `vmlal_u32` on low and high halves of a vector. + * + * This is a workaround for AArch64 GCC < 11 which implemented arm_neon.h with + * inline assembly and were therefore incapable of merging the `vget_{low, high}_u32` + * with `vmlal_u32`. + */ +#if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__) && __GNUC__ < 11 +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + /* Inline assembly is the only way */ + __asm__("umlal %0.2d, %1.2s, %2.2s" : "+w" (acc) : "w" (lhs), "w" (rhs)); + return acc; +} +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + /* This intrinsic works as expected */ + return vmlal_high_u32(acc, lhs, rhs); +} +#else +/* Portable intrinsic versions */ +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + return vmlal_u32(acc, vget_low_u32(lhs), vget_low_u32(rhs)); +} +/*! @copydoc XXH_vmlal_low_u32 + * Assume the compiler converts this to vmlal_high_u32 on aarch64 */ +XXH_FORCE_INLINE uint64x2_t +XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs) +{ + return vmlal_u32(acc, vget_high_u32(lhs), vget_high_u32(rhs)); +} +#endif + +/*! + * @ingroup tuning + * @brief Controls the NEON to scalar ratio for XXH3 + * + * This can be set to 2, 4, 6, or 8. + * + * ARM Cortex CPUs are _very_ sensitive to how their pipelines are used. + * + * For example, the Cortex-A73 can dispatch 3 micro-ops per cycle, but only 2 of those + * can be NEON. If you are only using NEON instructions, you are only using 2/3 of the CPU + * bandwidth. + * + * This is even more noticeable on the more advanced cores like the Cortex-A76 which + * can dispatch 8 micro-ops per cycle, but still only 2 NEON micro-ops at once. + * + * Therefore, to make the most out of the pipeline, it is beneficial to run 6 NEON lanes + * and 2 scalar lanes, which is chosen by default. + * + * This does not apply to Apple processors or 32-bit processors, which run better with + * full NEON. These will default to 8. Additionally, size-optimized builds run 8 lanes. + * + * This change benefits CPUs with large micro-op buffers without negatively affecting + * most other CPUs: + * + * | Chipset | Dispatch type | NEON only | 6:2 hybrid | Diff. | + * |:----------------------|:--------------------|----------:|-----------:|------:| + * | Snapdragon 730 (A76) | 2 NEON/8 micro-ops | 8.8 GB/s | 10.1 GB/s | ~16% | + * | Snapdragon 835 (A73) | 2 NEON/3 micro-ops | 5.1 GB/s | 5.3 GB/s | ~5% | + * | Marvell PXA1928 (A53) | In-order dual-issue | 1.9 GB/s | 1.9 GB/s | 0% | + * | Apple M1 | 4 NEON/8 micro-ops | 37.3 GB/s | 36.1 GB/s | ~-3% | + * + * It also seems to fix some bad codegen on GCC, making it almost as fast as clang. + * + * When using WASM SIMD128, if this is 2 or 6, SIMDe will scalarize 2 of the lanes meaning + * it effectively becomes worse 4. + * + * @see XXH3_accumulate_512_neon() + */ +# ifndef XXH3_NEON_LANES +# if (defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64) || defined(_M_ARM64EC)) \ + && !defined(__APPLE__) && XXH_SIZE_OPT <= 0 +# define XXH3_NEON_LANES 6 +# else +# define XXH3_NEON_LANES XXH_ACC_NB +# endif +# endif +#endif /* XXH_VECTOR == XXH_NEON */ + +/* + * VSX and Z Vector helpers. + * + * This is very messy, and any pull requests to clean this up are welcome. + * + * There are a lot of problems with supporting VSX and s390x, due to + * inconsistent intrinsics, spotty coverage, and multiple endiannesses. + */ +#if XXH_VECTOR == XXH_VSX +/* Annoyingly, these headers _may_ define three macros: `bool`, `vector`, + * and `pixel`. This is a problem for obvious reasons. + * + * These keywords are unnecessary; the spec literally says they are + * equivalent to `__bool`, `__vector`, and `__pixel` and may be undef'd + * after including the header. + * + * We use pragma push_macro/pop_macro to keep the namespace clean. */ +# pragma push_macro("bool") +# pragma push_macro("vector") +# pragma push_macro("pixel") +/* silence potential macro redefined warnings */ +# undef bool +# undef vector +# undef pixel + +# if defined(__s390x__) +# include +# else +# include +# endif + +/* Restore the original macro values, if applicable. */ +# pragma pop_macro("pixel") +# pragma pop_macro("vector") +# pragma pop_macro("bool") + +typedef __vector unsigned long long xxh_u64x2; +typedef __vector unsigned char xxh_u8x16; +typedef __vector unsigned xxh_u32x4; + +/* + * UGLY HACK: Similar to aarch64 macOS GCC, s390x GCC has the same aliasing issue. + */ +typedef xxh_u64x2 xxh_aliasing_u64x2 XXH_ALIASING; + +# ifndef XXH_VSX_BE +# if defined(__BIG_ENDIAN__) \ + || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +# define XXH_VSX_BE 1 +# elif defined(__VEC_ELEMENT_REG_ORDER__) && __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__ +# warning "-maltivec=be is not recommended. Please use native endianness." +# define XXH_VSX_BE 1 +# else +# define XXH_VSX_BE 0 +# endif +# endif /* !defined(XXH_VSX_BE) */ + +# if XXH_VSX_BE +# if defined(__POWER9_VECTOR__) || (defined(__clang__) && defined(__s390x__)) +# define XXH_vec_revb vec_revb +# else +/*! + * A polyfill for POWER9's vec_revb(). + */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val) +{ + xxh_u8x16 const vByteSwap = { 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00, + 0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08 }; + return vec_perm(val, val, vByteSwap); +} +# endif +# endif /* XXH_VSX_BE */ + +/*! + * Performs an unaligned vector load and byte swaps it on big endian. + */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr) +{ + xxh_u64x2 ret; + XXH_memcpy(&ret, ptr, sizeof(xxh_u64x2)); +# if XXH_VSX_BE + ret = XXH_vec_revb(ret); +# endif + return ret; +} + +/* + * vec_mulo and vec_mule are very problematic intrinsics on PowerPC + * + * These intrinsics weren't added until GCC 8, despite existing for a while, + * and they are endian dependent. Also, their meaning swap depending on version. + * */ +# if defined(__s390x__) + /* s390x is always big endian, no issue on this platform */ +# define XXH_vec_mulo vec_mulo +# define XXH_vec_mule vec_mule +# elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw) && !defined(__ibmxl__) +/* Clang has a better way to control this, we can just use the builtin which doesn't swap. */ + /* The IBM XL Compiler (which defined __clang__) only implements the vec_* operations */ +# define XXH_vec_mulo __builtin_altivec_vmulouw +# define XXH_vec_mule __builtin_altivec_vmuleuw +# else +/* gcc needs inline assembly */ +/* Adapted from https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */ +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mulo(xxh_u32x4 a, xxh_u32x4 b) +{ + xxh_u64x2 result; + __asm__("vmulouw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); + return result; +} +XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b) +{ + xxh_u64x2 result; + __asm__("vmuleuw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b)); + return result; +} +# endif /* XXH_vec_mulo, XXH_vec_mule */ +#endif /* XXH_VECTOR == XXH_VSX */ + +#if XXH_VECTOR == XXH_SVE +#define ACCRND(acc, offset) \ +do { \ + svuint64_t input_vec = svld1_u64(mask, xinput + offset); \ + svuint64_t secret_vec = svld1_u64(mask, xsecret + offset); \ + svuint64_t mixed = sveor_u64_x(mask, secret_vec, input_vec); \ + svuint64_t swapped = svtbl_u64(input_vec, kSwap); \ + svuint64_t mixed_lo = svextw_u64_x(mask, mixed); \ + svuint64_t mixed_hi = svlsr_n_u64_x(mask, mixed, 32); \ + svuint64_t mul = svmad_u64_x(mask, mixed_lo, mixed_hi, swapped); \ + acc = svadd_u64_x(mask, acc, mul); \ +} while (0) +#endif /* XXH_VECTOR == XXH_SVE */ + +/* prefetch + * can be disabled, by declaring XXH_NO_PREFETCH build macro */ +#if defined(XXH_NO_PREFETCH) +# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ +#else +# if XXH_SIZE_OPT >= 1 +# define XXH_PREFETCH(ptr) (void)(ptr) +# elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) /* _mm_prefetch() not defined outside of x86/x64 */ +# include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ +# define XXH_PREFETCH(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0) +# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) ) +# define XXH_PREFETCH(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */) +# else +# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */ +# endif +#endif /* XXH_NO_PREFETCH */ + + +/* ========================================== + * XXH3 default settings + * ========================================== */ + +#define XXH_SECRET_DEFAULT_SIZE 192 /* minimum XXH3_SECRET_SIZE_MIN */ + +#if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN) +# error "default keyset is not large enough" +#endif + +/*! Pseudorandom secret taken directly from FARSH. */ +XXH_ALIGN(64) static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = { + 0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c, + 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f, + 0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21, + 0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c, + 0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3, + 0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8, + 0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d, + 0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64, + 0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb, + 0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e, + 0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce, + 0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e, +}; + +static const xxh_u64 PRIME_MX1 = 0x165667919E3779F9ULL; /*!< 0b0001011001010110011001111001000110011110001101110111100111111001 */ +static const xxh_u64 PRIME_MX2 = 0x9FB21C651E98DF25ULL; /*!< 0b1001111110110010000111000110010100011110100110001101111100100101 */ + +#ifdef XXH_OLD_NAMES +# define kSecret XXH3_kSecret +#endif + +#ifdef XXH_DOXYGEN +/*! + * @brief Calculates a 32-bit to 64-bit long multiply. + * + * Implemented as a macro. + * + * Wraps `__emulu` on MSVC x86 because it tends to call `__allmul` when it doesn't + * need to (but it shouldn't need to anyways, it is about 7 instructions to do + * a 64x64 multiply...). Since we know that this will _always_ emit `MULL`, we + * use that instead of the normal method. + * + * If you are compiling for platforms like Thumb-1 and don't have a better option, + * you may also want to write your own long multiply routine here. + * + * @param x, y Numbers to be multiplied + * @return 64-bit product of the low 32 bits of @p x and @p y. + */ +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64(xxh_u64 x, xxh_u64 y) +{ + return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF); +} +#elif defined(_MSC_VER) && defined(_M_IX86) +# define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y)) +#else +/* + * Downcast + upcast is usually better than masking on older compilers like + * GCC 4.2 (especially 32-bit ones), all without affecting newer compilers. + * + * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both operands + * and perform a full 64x64 multiply -- entirely redundant on 32-bit. + */ +# define XXH_mult32to64(x, y) ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y)) +#endif + +/*! + * @brief Calculates a 64->128-bit long multiply. + * + * Uses `__uint128_t` and `_umul128` if available, otherwise uses a scalar + * version. + * + * @param lhs , rhs The 64-bit integers to be multiplied + * @return The 128-bit result represented in an @ref XXH128_hash_t. + */ +static XXH128_hash_t +XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs) +{ + /* + * GCC/Clang __uint128_t method. + * + * On most 64-bit targets, GCC and Clang define a __uint128_t type. + * This is usually the best way as it usually uses a native long 64-bit + * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64. + * + * Usually. + * + * Despite being a 32-bit platform, Clang (and emscripten) define this type + * despite not having the arithmetic for it. This results in a laggy + * compiler builtin call which calculates a full 128-bit multiply. + * In that case it is best to use the portable one. + * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677 + */ +#if (defined(__GNUC__) || defined(__clang__)) && !defined(__wasm__) \ + && defined(__SIZEOF_INT128__) \ + || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128) + + __uint128_t const product = (__uint128_t)lhs * (__uint128_t)rhs; + XXH128_hash_t r128; + r128.low64 = (xxh_u64)(product); + r128.high64 = (xxh_u64)(product >> 64); + return r128; + + /* + * MSVC for x64's _umul128 method. + * + * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 *HighProduct); + * + * This compiles to single operand MUL on x64. + */ +#elif (defined(_M_X64) || defined(_M_IA64)) && !defined(_M_ARM64EC) + +#ifndef _MSC_VER +# pragma intrinsic(_umul128) +#endif + xxh_u64 product_high; + xxh_u64 const product_low = _umul128(lhs, rhs, &product_high); + XXH128_hash_t r128; + r128.low64 = product_low; + r128.high64 = product_high; + return r128; + + /* + * MSVC for ARM64's __umulh method. + * + * This compiles to the same MUL + UMULH as GCC/Clang's __uint128_t method. + */ +#elif defined(_M_ARM64) || defined(_M_ARM64EC) + +#ifndef _MSC_VER +# pragma intrinsic(__umulh) +#endif + XXH128_hash_t r128; + r128.low64 = lhs * rhs; + r128.high64 = __umulh(lhs, rhs); + return r128; + +#else + /* + * Portable scalar method. Optimized for 32-bit and 64-bit ALUs. + * + * This is a fast and simple grade school multiply, which is shown below + * with base 10 arithmetic instead of base 0x100000000. + * + * 9 3 // D2 lhs = 93 + * x 7 5 // D2 rhs = 75 + * ---------- + * 1 5 // D2 lo_lo = (93 % 10) * (75 % 10) = 15 + * 4 5 | // D2 hi_lo = (93 / 10) * (75 % 10) = 45 + * 2 1 | // D2 lo_hi = (93 % 10) * (75 / 10) = 21 + * + 6 3 | | // D2 hi_hi = (93 / 10) * (75 / 10) = 63 + * --------- + * 2 7 | // D2 cross = (15 / 10) + (45 % 10) + 21 = 27 + * + 6 7 | | // D2 upper = (27 / 10) + (45 / 10) + 63 = 67 + * --------- + * 6 9 7 5 // D4 res = (27 * 10) + (15 % 10) + (67 * 100) = 6975 + * + * The reasons for adding the products like this are: + * 1. It avoids manual carry tracking. Just like how + * (9 * 9) + 9 + 9 = 99, the same applies with this for UINT64_MAX. + * This avoids a lot of complexity. + * + * 2. It hints for, and on Clang, compiles to, the powerful UMAAL + * instruction available in ARM's Digital Signal Processing extension + * in 32-bit ARMv6 and later, which is shown below: + * + * void UMAAL(xxh_u32 *RdLo, xxh_u32 *RdHi, xxh_u32 Rn, xxh_u32 Rm) + * { + * xxh_u64 product = (xxh_u64)*RdLo * (xxh_u64)*RdHi + Rn + Rm; + * *RdLo = (xxh_u32)(product & 0xFFFFFFFF); + * *RdHi = (xxh_u32)(product >> 32); + * } + * + * This instruction was designed for efficient long multiplication, and + * allows this to be calculated in only 4 instructions at speeds + * comparable to some 64-bit ALUs. + * + * 3. It isn't terrible on other platforms. Usually this will be a couple + * of 32-bit ADD/ADCs. + */ + + /* First calculate all of the cross products. */ + xxh_u64 const lo_lo = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs & 0xFFFFFFFF); + xxh_u64 const hi_lo = XXH_mult32to64(lhs >> 32, rhs & 0xFFFFFFFF); + xxh_u64 const lo_hi = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs >> 32); + xxh_u64 const hi_hi = XXH_mult32to64(lhs >> 32, rhs >> 32); + + /* Now add the products together. These will never overflow. */ + xxh_u64 const cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi; + xxh_u64 const upper = (hi_lo >> 32) + (cross >> 32) + hi_hi; + xxh_u64 const lower = (cross << 32) | (lo_lo & 0xFFFFFFFF); + + XXH128_hash_t r128; + r128.low64 = lower; + r128.high64 = upper; + return r128; +#endif +} + +/*! + * @brief Calculates a 64-bit to 128-bit multiply, then XOR folds it. + * + * The reason for the separate function is to prevent passing too many structs + * around by value. This will hopefully inline the multiply, but we don't force it. + * + * @param lhs , rhs The 64-bit integers to multiply + * @return The low 64 bits of the product XOR'd by the high 64 bits. + * @see XXH_mult64to128() + */ +static xxh_u64 +XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs) +{ + XXH128_hash_t product = XXH_mult64to128(lhs, rhs); + return product.low64 ^ product.high64; +} + +/*! Seems to produce slightly better code on GCC for some reason. */ +XXH_FORCE_INLINE XXH_CONSTF xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift) +{ + XXH_ASSERT(0 <= shift && shift < 64); + return v64 ^ (v64 >> shift); +} + +/* + * This is a fast avalanche stage, + * suitable when input bits are already partially mixed + */ +static XXH64_hash_t XXH3_avalanche(xxh_u64 h64) +{ + h64 = XXH_xorshift64(h64, 37); + h64 *= PRIME_MX1; + h64 = XXH_xorshift64(h64, 32); + return h64; +} + +/* + * This is a stronger avalanche, + * inspired by Pelle Evensen's rrmxmx + * preferable when input has not been previously mixed + */ +static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len) +{ + /* this mix is inspired by Pelle Evensen's rrmxmx */ + h64 ^= XXH_rotl64(h64, 49) ^ XXH_rotl64(h64, 24); + h64 *= PRIME_MX2; + h64 ^= (h64 >> 35) + len ; + h64 *= PRIME_MX2; + return XXH_xorshift64(h64, 28); +} + + +/* ========================================== + * Short keys + * ========================================== + * One of the shortcomings of XXH32 and XXH64 was that their performance was + * sub-optimal on short lengths. It used an iterative algorithm which strongly + * favored lengths that were a multiple of 4 or 8. + * + * Instead of iterating over individual inputs, we use a set of single shot + * functions which piece together a range of lengths and operate in constant time. + * + * Additionally, the number of multiplies has been significantly reduced. This + * reduces latency, especially when emulating 64-bit multiplies on 32-bit. + * + * Depending on the platform, this may or may not be faster than XXH32, but it + * is almost guaranteed to be faster than XXH64. + */ + +/* + * At very short lengths, there isn't enough input to fully hide secrets, or use + * the entire secret. + * + * There is also only a limited amount of mixing we can do before significantly + * impacting performance. + * + * Therefore, we use different sections of the secret and always mix two secret + * samples with an XOR. This should have no effect on performance on the + * seedless or withSeed variants because everything _should_ be constant folded + * by modern compilers. + * + * The XOR mixing hides individual parts of the secret and increases entropy. + * + * This adds an extra layer of strength for custom secrets. + */ +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_1to3_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combined = { input[0], 0x01, input[0], input[0] } + * len = 2: combined = { input[1], 0x02, input[0], input[1] } + * len = 3: combined = { input[2], 0x03, input[0], input[1] } + */ + { xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combined = ((xxh_u32)c1 << 16) | ((xxh_u32)c2 << 24) + | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u64 const bitflip = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; + xxh_u64 const keyed = (xxh_u64)combined ^ bitflip; + return XXH64_avalanche(keyed); + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_4to8_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len <= 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { xxh_u32 const input1 = XXH_readLE32(input); + xxh_u32 const input2 = XXH_readLE32(input + len - 4); + xxh_u64 const bitflip = (XXH_readLE64(secret+8) ^ XXH_readLE64(secret+16)) - seed; + xxh_u64 const input64 = input2 + (((xxh_u64)input1) << 32); + xxh_u64 const keyed = input64 ^ bitflip; + return XXH3_rrmxmx(keyed, len); + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_9to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(9 <= len && len <= 16); + { xxh_u64 const bitflip1 = (XXH_readLE64(secret+24) ^ XXH_readLE64(secret+32)) + seed; + xxh_u64 const bitflip2 = (XXH_readLE64(secret+40) ^ XXH_readLE64(secret+48)) - seed; + xxh_u64 const input_lo = XXH_readLE64(input) ^ bitflip1; + xxh_u64 const input_hi = XXH_readLE64(input + len - 8) ^ bitflip2; + xxh_u64 const acc = len + + XXH_swap64(input_lo) + input_hi + + XXH3_mul128_fold64(input_lo, input_hi); + return XXH3_avalanche(acc); + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_0to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(len <= 16); + { if (XXH_likely(len > 8)) return XXH3_len_9to16_64b(input, len, secret, seed); + if (XXH_likely(len >= 4)) return XXH3_len_4to8_64b(input, len, secret, seed); + if (len) return XXH3_len_1to3_64b(input, len, secret, seed); + return XXH64_avalanche(seed ^ (XXH_readLE64(secret+56) ^ XXH_readLE64(secret+64))); + } +} + +/* + * DISCLAIMER: There are known *seed-dependent* multicollisions here due to + * multiplication by zero, affecting hashes of lengths 17 to 240. + * + * However, they are very unlikely. + * + * Keep this in mind when using the unseeded XXH3_64bits() variant: As with all + * unseeded non-cryptographic hashes, it does not attempt to defend itself + * against specially crafted inputs, only random inputs. + * + * Compared to classic UMAC where a 1 in 2^31 chance of 4 consecutive bytes + * cancelling out the secret is taken an arbitrary number of times (addressed + * in XXH3_accumulate_512), this collision is very unlikely with random inputs + * and/or proper seeding: + * + * This only has a 1 in 2^63 chance of 8 consecutive bytes cancelling out, in a + * function that is only called up to 16 times per hash with up to 240 bytes of + * input. + * + * This is not too bad for a non-cryptographic hash function, especially with + * only 64 bit outputs. + * + * The 128-bit variant (which trades some speed for strength) is NOT affected + * by this, although it is always a good idea to use a proper seed if you care + * about strength. + */ +XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8* XXH_RESTRICT input, + const xxh_u8* XXH_RESTRICT secret, xxh_u64 seed64) +{ +#if defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \ + && defined(__i386__) && defined(__SSE2__) /* x86 + SSE2 */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable like XXH32 hack */ + /* + * UGLY HACK: + * GCC for x86 tends to autovectorize the 128-bit multiply, resulting in + * slower code. + * + * By forcing seed64 into a register, we disrupt the cost model and + * cause it to scalarize. See `XXH32_round()` + * + * FIXME: Clang's output is still _much_ faster -- On an AMD Ryzen 3600, + * XXH3_64bits @ len=240 runs at 4.6 GB/s with Clang 9, but 3.3 GB/s on + * GCC 9.2, despite both emitting scalar code. + * + * GCC generates much better scalar code than Clang for the rest of XXH3, + * which is why finding a more optimal codepath is an interest. + */ + XXH_COMPILER_GUARD(seed64); +#endif + { xxh_u64 const input_lo = XXH_readLE64(input); + xxh_u64 const input_hi = XXH_readLE64(input+8); + return XXH3_mul128_fold64( + input_lo ^ (XXH_readLE64(secret) + seed64), + input_hi ^ (XXH_readLE64(secret+8) - seed64) + ); + } +} + +/* For mid range keys, XXH3 uses a Mum-hash variant. */ +XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_17to128_64b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); + + { xxh_u64 acc = len * XXH_PRIME64_1; +#if XXH_SIZE_OPT >= 1 + /* Smaller and cleaner, but slightly slower. */ + unsigned int i = (unsigned int)(len - 1) / 32; + do { + acc += XXH3_mix16B(input+16 * i, secret+32*i, seed); + acc += XXH3_mix16B(input+len-16*(i+1), secret+32*i+16, seed); + } while (i-- != 0); +#else + if (len > 32) { + if (len > 64) { + if (len > 96) { + acc += XXH3_mix16B(input+48, secret+96, seed); + acc += XXH3_mix16B(input+len-64, secret+112, seed); + } + acc += XXH3_mix16B(input+32, secret+64, seed); + acc += XXH3_mix16B(input+len-48, secret+80, seed); + } + acc += XXH3_mix16B(input+16, secret+32, seed); + acc += XXH3_mix16B(input+len-32, secret+48, seed); + } + acc += XXH3_mix16B(input+0, secret+0, seed); + acc += XXH3_mix16B(input+len-16, secret+16, seed); +#endif + return XXH3_avalanche(acc); + } +} + +/*! + * @brief Maximum size of "short" key in bytes. + */ +#define XXH3_MIDSIZE_MAX 240 + +XXH_NO_INLINE XXH_PUREF XXH64_hash_t +XXH3_len_129to240_64b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + + #define XXH3_MIDSIZE_STARTOFFSET 3 + #define XXH3_MIDSIZE_LASTOFFSET 17 + + { xxh_u64 acc = len * XXH_PRIME64_1; + xxh_u64 acc_end; + unsigned int const nbRounds = (unsigned int)len / 16; + unsigned int i; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + for (i=0; i<8; i++) { + acc += XXH3_mix16B(input+(16*i), secret+(16*i), seed); + } + /* last bytes */ + acc_end = XXH3_mix16B(input + len - 16, secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, seed); + XXH_ASSERT(nbRounds >= 8); + acc = XXH3_avalanche(acc); +#if defined(__clang__) /* Clang */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Clang for ARMv7-A tries to vectorize this loop, similar to GCC x86. + * In everywhere else, it uses scalar code. + * + * For 64->128-bit multiplies, even if the NEON was 100% optimal, it + * would still be slower than UMAAL (see XXH_mult64to128). + * + * Unfortunately, Clang doesn't handle the long multiplies properly and + * converts them to the nonexistent "vmulq_u64" intrinsic, which is then + * scalarized into an ugly mess of VMOV.32 instructions. + * + * This mess is difficult to avoid without turning autovectorization + * off completely, but they are usually relatively minor and/or not + * worth it to fix. + * + * This loop is the easiest to fix, as unlike XXH32, this pragma + * _actually works_ because it is a loop vectorization instead of an + * SLP vectorization. + */ + #pragma clang loop vectorize(disable) +#endif + for (i=8 ; i < nbRounds; i++) { + /* + * Prevents clang for unrolling the acc loop and interleaving with this one. + */ + XXH_COMPILER_GUARD(acc); + acc_end += XXH3_mix16B(input+(16*i), secret+(16*(i-8)) + XXH3_MIDSIZE_STARTOFFSET, seed); + } + return XXH3_avalanche(acc + acc_end); + } +} + + +/* ======= Long Keys ======= */ + +#define XXH_STRIPE_LEN 64 +#define XXH_SECRET_CONSUME_RATE 8 /* nb of secret bytes consumed at each accumulation */ +#define XXH_ACC_NB (XXH_STRIPE_LEN / sizeof(xxh_u64)) + +#ifdef XXH_OLD_NAMES +# define STRIPE_LEN XXH_STRIPE_LEN +# define ACC_NB XXH_ACC_NB +#endif + +#ifndef XXH_PREFETCH_DIST +# ifdef __clang__ +# define XXH_PREFETCH_DIST 320 +# else +# if (XXH_VECTOR == XXH_AVX512) +# define XXH_PREFETCH_DIST 512 +# else +# define XXH_PREFETCH_DIST 384 +# endif +# endif /* __clang__ */ +#endif /* XXH_PREFETCH_DIST */ + +/* + * These macros are to generate an XXH3_accumulate() function. + * The two arguments select the name suffix and target attribute. + * + * The name of this symbol is XXH3_accumulate_() and it calls + * XXH3_accumulate_512_(). + * + * It may be useful to hand implement this function if the compiler fails to + * optimize the inline function. + */ +#define XXH3_ACCUMULATE_TEMPLATE(name) \ +void \ +XXH3_accumulate_##name(xxh_u64* XXH_RESTRICT acc, \ + const xxh_u8* XXH_RESTRICT input, \ + const xxh_u8* XXH_RESTRICT secret, \ + size_t nbStripes) \ +{ \ + size_t n; \ + for (n = 0; n < nbStripes; n++ ) { \ + const xxh_u8* const in = input + n*XXH_STRIPE_LEN; \ + XXH_PREFETCH(in + XXH_PREFETCH_DIST); \ + XXH3_accumulate_512_##name( \ + acc, \ + in, \ + secret + n*XXH_SECRET_CONSUME_RATE); \ + } \ +} + + +XXH_FORCE_INLINE void XXH_writeLE64(void* dst, xxh_u64 v64) +{ + if (!XXH_CPU_LITTLE_ENDIAN) v64 = XXH_swap64(v64); + XXH_memcpy(dst, &v64, sizeof(v64)); +} + +/* Several intrinsic functions below are supposed to accept __int64 as argument, + * as documented in https://software.intel.com/sites/landingpage/IntrinsicsGuide/ . + * However, several environments do not define __int64 type, + * requiring a workaround. + */ +#if !defined (__VMS) \ + && (defined (__cplusplus) \ + || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) + typedef int64_t xxh_i64; +#else + /* the following type must have a width of 64-bit */ + typedef long long xxh_i64; +#endif + + +/* + * XXH3_accumulate_512 is the tightest loop for long inputs, and it is the most optimized. + * + * It is a hardened version of UMAC, based off of FARSH's implementation. + * + * This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD + * implementations, and it is ridiculously fast. + * + * We harden it by mixing the original input to the accumulators as well as the product. + * + * This means that in the (relatively likely) case of a multiply by zero, the + * original input is preserved. + * + * On 128-bit inputs, we swap 64-bit pairs when we add the input to improve + * cross-pollination, as otherwise the upper and lower halves would be + * essentially independent. + * + * This doesn't matter on 64-bit hashes since they all get merged together in + * the end, so we skip the extra step. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ + +#if (XXH_VECTOR == XXH_AVX512) \ + || (defined(XXH_DISPATCH_AVX512) && XXH_DISPATCH_AVX512 != 0) + +#ifndef XXH_TARGET_AVX512 +# define XXH_TARGET_AVX512 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_accumulate_512_avx512(void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + __m512i* const xacc = (__m512i *) acc; + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); + + { + /* data_vec = input[0]; */ + __m512i const data_vec = _mm512_loadu_si512 (input); + /* key_vec = secret[0]; */ + __m512i const key_vec = _mm512_loadu_si512 (secret); + /* data_key = data_vec ^ key_vec; */ + __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m512i const data_key_lo = _mm512_srli_epi64 (data_key, 32); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m512i const product = _mm512_mul_epu32 (data_key, data_key_lo); + /* xacc[0] += swap(data_vec); */ + __m512i const data_swap = _mm512_shuffle_epi32(data_vec, (_MM_PERM_ENUM)_MM_SHUFFLE(1, 0, 3, 2)); + __m512i const sum = _mm512_add_epi64(*xacc, data_swap); + /* xacc[0] += product; */ + *xacc = _mm512_add_epi64(product, sum); + } +} +XXH_FORCE_INLINE XXH_TARGET_AVX512 XXH3_ACCUMULATE_TEMPLATE(avx512) + +/* + * XXH3_scrambleAcc: Scrambles the accumulators to improve mixing. + * + * Multiplication isn't perfect, as explained by Google in HighwayHash: + * + * // Multiplication mixes/scrambles bytes 0-7 of the 64-bit result to + * // varying degrees. In descending order of goodness, bytes + * // 3 4 2 5 1 6 0 7 have quality 228 224 164 160 100 96 36 32. + * // As expected, the upper and lower bytes are much worse. + * + * Source: https://github.com/google/highwayhash/blob/0aaf66b/highwayhash/hh_avx2.h#L291 + * + * Since our algorithm uses a pseudorandom secret to add some variance into the + * mix, we don't need to (or want to) mix as often or as much as HighwayHash does. + * + * This isn't as tight as XXH3_accumulate, but still written in SIMD to avoid + * extraction. + * + * Both XXH3_64bits and XXH3_128bits use this subroutine. + */ + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_scrambleAcc_avx512(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 63) == 0); + XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i)); + { __m512i* const xacc = (__m512i*) acc; + const __m512i prime32 = _mm512_set1_epi32((int)XXH_PRIME32_1); + + /* xacc[0] ^= (xacc[0] >> 47) */ + __m512i const acc_vec = *xacc; + __m512i const shifted = _mm512_srli_epi64 (acc_vec, 47); + /* xacc[0] ^= secret; */ + __m512i const key_vec = _mm512_loadu_si512 (secret); + __m512i const data_key = _mm512_ternarylogic_epi32(key_vec, acc_vec, shifted, 0x96 /* key_vec ^ acc_vec ^ shifted */); + + /* xacc[0] *= XXH_PRIME32_1; */ + __m512i const data_key_hi = _mm512_srli_epi64 (data_key, 32); + __m512i const prod_lo = _mm512_mul_epu32 (data_key, prime32); + __m512i const prod_hi = _mm512_mul_epu32 (data_key_hi, prime32); + *xacc = _mm512_add_epi64(prod_lo, _mm512_slli_epi64(prod_hi, 32)); + } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX512 void +XXH3_initCustomSecret_avx512(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 63) == 0); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN == 64); + XXH_ASSERT(((size_t)customSecret & 63) == 0); + (void)(&XXH_writeLE64); + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m512i); + __m512i const seed_pos = _mm512_set1_epi64((xxh_i64)seed64); + __m512i const seed = _mm512_mask_sub_epi64(seed_pos, 0xAA, _mm512_set1_epi8(0), seed_pos); + + const __m512i* const src = (const __m512i*) ((const void*) XXH3_kSecret); + __m512i* const dest = ( __m512i*) customSecret; + int i; + XXH_ASSERT(((size_t)src & 63) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 63) == 0); + for (i=0; i < nbRounds; ++i) { + dest[i] = _mm512_add_epi64(_mm512_load_si512(src + i), seed); + } } +} + +#endif + +#if (XXH_VECTOR == XXH_AVX2) \ + || (defined(XXH_DISPATCH_AVX2) && XXH_DISPATCH_AVX2 != 0) + +#ifndef XXH_TARGET_AVX2 +# define XXH_TARGET_AVX2 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void +XXH3_accumulate_512_avx2( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 31) == 0); + { __m256i* const xacc = (__m256i *) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xinput = (const __m256i *) input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xsecret = (const __m256i *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { + /* data_vec = xinput[i]; */ + __m256i const data_vec = _mm256_loadu_si256 (xinput+i); + /* key_vec = xsecret[i]; */ + __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); + /* data_key = data_vec ^ key_vec; */ + __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m256i const data_key_lo = _mm256_srli_epi64 (data_key, 32); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m256i const product = _mm256_mul_epu32 (data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m256i const data_swap = _mm256_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2)); + __m256i const sum = _mm256_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm256_add_epi64(product, sum); + } } +} +XXH_FORCE_INLINE XXH_TARGET_AVX2 XXH3_ACCUMULATE_TEMPLATE(avx2) + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void +XXH3_scrambleAcc_avx2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 31) == 0); + { __m256i* const xacc = (__m256i*) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */ + const __m256i* const xsecret = (const __m256i *) secret; + const __m256i prime32 = _mm256_set1_epi32((int)XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m256i const acc_vec = xacc[i]; + __m256i const shifted = _mm256_srli_epi64 (acc_vec, 47); + __m256i const data_vec = _mm256_xor_si256 (acc_vec, shifted); + /* xacc[i] ^= xsecret; */ + __m256i const key_vec = _mm256_loadu_si256 (xsecret+i); + __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m256i const data_key_hi = _mm256_srli_epi64 (data_key, 32); + __m256i const prod_lo = _mm256_mul_epu32 (data_key, prime32); + __m256i const prod_hi = _mm256_mul_epu32 (data_key_hi, prime32); + xacc[i] = _mm256_add_epi64(prod_lo, _mm256_slli_epi64(prod_hi, 32)); + } + } +} + +XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 31) == 0); + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE / sizeof(__m256i)) == 6); + XXH_STATIC_ASSERT(XXH_SEC_ALIGN <= 64); + (void)(&XXH_writeLE64); + XXH_PREFETCH(customSecret); + { __m256i const seed = _mm256_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64, (xxh_i64)(0U - seed64), (xxh_i64)seed64); + + const __m256i* const src = (const __m256i*) ((const void*) XXH3_kSecret); + __m256i* dest = ( __m256i*) customSecret; + +# if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + */ + XXH_COMPILER_GUARD(dest); +# endif + XXH_ASSERT(((size_t)src & 31) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dest & 31) == 0); + + /* GCC -O2 need unroll loop manually */ + dest[0] = _mm256_add_epi64(_mm256_load_si256(src+0), seed); + dest[1] = _mm256_add_epi64(_mm256_load_si256(src+1), seed); + dest[2] = _mm256_add_epi64(_mm256_load_si256(src+2), seed); + dest[3] = _mm256_add_epi64(_mm256_load_si256(src+3), seed); + dest[4] = _mm256_add_epi64(_mm256_load_si256(src+4), seed); + dest[5] = _mm256_add_epi64(_mm256_load_si256(src+5), seed); + } +} + +#endif + +/* x86dispatch always generates SSE2 */ +#if (XXH_VECTOR == XXH_SSE2) || defined(XXH_X86DISPATCH) + +#ifndef XXH_TARGET_SSE2 +# define XXH_TARGET_SSE2 /* disable attribute target */ +#endif + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void +XXH3_accumulate_512_sse2( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + /* SSE2 is just a half-scale version of the AVX2 version. */ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { __m128i* const xacc = (__m128i *) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xinput = (const __m128i *) input; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xsecret = (const __m128i *) secret; + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { + /* data_vec = xinput[i]; */ + __m128i const data_vec = _mm_loadu_si128 (xinput+i); + /* key_vec = xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128 (xsecret+i); + /* data_key = data_vec ^ key_vec; */ + __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); + /* data_key_lo = data_key >> 32; */ + __m128i const data_key_lo = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */ + __m128i const product = _mm_mul_epu32 (data_key, data_key_lo); + /* xacc[i] += swap(data_vec); */ + __m128i const data_swap = _mm_shuffle_epi32(data_vec, _MM_SHUFFLE(1,0,3,2)); + __m128i const sum = _mm_add_epi64(xacc[i], data_swap); + /* xacc[i] += product; */ + xacc[i] = _mm_add_epi64(product, sum); + } } +} +XXH_FORCE_INLINE XXH_TARGET_SSE2 XXH3_ACCUMULATE_TEMPLATE(sse2) + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void +XXH3_scrambleAcc_sse2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + { __m128i* const xacc = (__m128i*) acc; + /* Unaligned. This is mainly for pointer arithmetic, and because + * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */ + const __m128i* const xsecret = (const __m128i *) secret; + const __m128i prime32 = _mm_set1_epi32((int)XXH_PRIME32_1); + + size_t i; + for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) { + /* xacc[i] ^= (xacc[i] >> 47) */ + __m128i const acc_vec = xacc[i]; + __m128i const shifted = _mm_srli_epi64 (acc_vec, 47); + __m128i const data_vec = _mm_xor_si128 (acc_vec, shifted); + /* xacc[i] ^= xsecret[i]; */ + __m128i const key_vec = _mm_loadu_si128 (xsecret+i); + __m128i const data_key = _mm_xor_si128 (data_vec, key_vec); + + /* xacc[i] *= XXH_PRIME32_1; */ + __m128i const data_key_hi = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1)); + __m128i const prod_lo = _mm_mul_epu32 (data_key, prime32); + __m128i const prod_hi = _mm_mul_epu32 (data_key_hi, prime32); + xacc[i] = _mm_add_epi64(prod_lo, _mm_slli_epi64(prod_hi, 32)); + } + } +} + +XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + (void)(&XXH_writeLE64); + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m128i); + +# if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER < 1900 + /* MSVC 32bit mode does not support _mm_set_epi64x before 2015 */ + XXH_ALIGN(16) const xxh_i64 seed64x2[2] = { (xxh_i64)seed64, (xxh_i64)(0U - seed64) }; + __m128i const seed = _mm_load_si128((__m128i const*)seed64x2); +# else + __m128i const seed = _mm_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64); +# endif + int i; + + const void* const src16 = XXH3_kSecret; + __m128i* dst16 = (__m128i*) customSecret; +# if defined(__GNUC__) || defined(__clang__) + /* + * On GCC & Clang, marking 'dest' as modified will cause the compiler: + * - do not extract the secret from sse registers in the internal loop + * - use less common registers, and avoid pushing these reg into stack + */ + XXH_COMPILER_GUARD(dst16); +# endif + XXH_ASSERT(((size_t)src16 & 15) == 0); /* control alignment */ + XXH_ASSERT(((size_t)dst16 & 15) == 0); + + for (i=0; i < nbRounds; ++i) { + dst16[i] = _mm_add_epi64(_mm_load_si128((const __m128i *)src16+i), seed); + } } +} + +#endif + +#if (XXH_VECTOR == XXH_NEON) + +/* forward declarations for the scalar routines */ +XXH_FORCE_INLINE void +XXH3_scalarRound(void* XXH_RESTRICT acc, void const* XXH_RESTRICT input, + void const* XXH_RESTRICT secret, size_t lane); + +XXH_FORCE_INLINE void +XXH3_scalarScrambleRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT secret, size_t lane); + +/*! + * @internal + * @brief The bulk processing loop for NEON and WASM SIMD128. + * + * The NEON code path is actually partially scalar when running on AArch64. This + * is to optimize the pipelining and can have up to 15% speedup depending on the + * CPU, and it also mitigates some GCC codegen issues. + * + * @see XXH3_NEON_LANES for configuring this and details about this optimization. + * + * NEON's 32-bit to 64-bit long multiply takes a half vector of 32-bit + * integers instead of the other platforms which mask full 64-bit vectors, + * so the setup is more complicated than just shifting right. + * + * Additionally, there is an optimization for 4 lanes at once noted below. + * + * Since, as stated, the most optimal amount of lanes for Cortexes is 6, + * there needs to be *three* versions of the accumulate operation used + * for the remaining 2 lanes. + * + * WASM's SIMD128 uses SIMDe's arm_neon.h polyfill because the intrinsics overlap + * nearly perfectly. + */ + +XXH_FORCE_INLINE void +XXH3_accumulate_512_neon( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + XXH_STATIC_ASSERT(XXH3_NEON_LANES > 0 && XXH3_NEON_LANES <= XXH_ACC_NB && XXH3_NEON_LANES % 2 == 0); + { /* GCC for darwin arm64 does not like aliasing here */ + xxh_aliasing_uint64x2_t* const xacc = (xxh_aliasing_uint64x2_t*) acc; + /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. */ + uint8_t const* xinput = (const uint8_t *) input; + uint8_t const* xsecret = (const uint8_t *) secret; + + size_t i; +#ifdef __wasm_simd128__ + /* + * On WASM SIMD128, Clang emits direct address loads when XXH3_kSecret + * is constant propagated, which results in it converting it to this + * inside the loop: + * + * a = v128.load(XXH3_kSecret + 0 + $secret_offset, offset = 0) + * b = v128.load(XXH3_kSecret + 16 + $secret_offset, offset = 0) + * ... + * + * This requires a full 32-bit address immediate (and therefore a 6 byte + * instruction) as well as an add for each offset. + * + * Putting an asm guard prevents it from folding (at the cost of losing + * the alignment hint), and uses the free offset in `v128.load` instead + * of adding secret_offset each time which overall reduces code size by + * about a kilobyte and improves performance. + */ + XXH_COMPILER_GUARD(xsecret); +#endif + /* Scalar lanes use the normal scalarRound routine */ + for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { + XXH3_scalarRound(acc, input, secret, i); + } + i = 0; + /* 4 NEON lanes at a time. */ + for (; i+1 < XXH3_NEON_LANES / 2; i+=2) { + /* data_vec = xinput[i]; */ + uint64x2_t data_vec_1 = XXH_vld1q_u64(xinput + (i * 16)); + uint64x2_t data_vec_2 = XXH_vld1q_u64(xinput + ((i+1) * 16)); + /* key_vec = xsecret[i]; */ + uint64x2_t key_vec_1 = XXH_vld1q_u64(xsecret + (i * 16)); + uint64x2_t key_vec_2 = XXH_vld1q_u64(xsecret + ((i+1) * 16)); + /* data_swap = swap(data_vec) */ + uint64x2_t data_swap_1 = vextq_u64(data_vec_1, data_vec_1, 1); + uint64x2_t data_swap_2 = vextq_u64(data_vec_2, data_vec_2, 1); + /* data_key = data_vec ^ key_vec; */ + uint64x2_t data_key_1 = veorq_u64(data_vec_1, key_vec_1); + uint64x2_t data_key_2 = veorq_u64(data_vec_2, key_vec_2); + + /* + * If we reinterpret the 64x2 vectors as 32x4 vectors, we can use a + * de-interleave operation for 4 lanes in 1 step with `vuzpq_u32` to + * get one vector with the low 32 bits of each lane, and one vector + * with the high 32 bits of each lane. + * + * The intrinsic returns a double vector because the original ARMv7-a + * instruction modified both arguments in place. AArch64 and SIMD128 emit + * two instructions from this intrinsic. + * + * [ dk11L | dk11H | dk12L | dk12H ] -> [ dk11L | dk12L | dk21L | dk22L ] + * [ dk21L | dk21H | dk22L | dk22H ] -> [ dk11H | dk12H | dk21H | dk22H ] + */ + uint32x4x2_t unzipped = vuzpq_u32( + vreinterpretq_u32_u64(data_key_1), + vreinterpretq_u32_u64(data_key_2) + ); + /* data_key_lo = data_key & 0xFFFFFFFF */ + uint32x4_t data_key_lo = unzipped.val[0]; + /* data_key_hi = data_key >> 32 */ + uint32x4_t data_key_hi = unzipped.val[1]; + /* + * Then, we can split the vectors horizontally and multiply which, as for most + * widening intrinsics, have a variant that works on both high half vectors + * for free on AArch64. A similar instruction is available on SIMD128. + * + * sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi + */ + uint64x2_t sum_1 = XXH_vmlal_low_u32(data_swap_1, data_key_lo, data_key_hi); + uint64x2_t sum_2 = XXH_vmlal_high_u32(data_swap_2, data_key_lo, data_key_hi); + /* + * Clang reorders + * a += b * c; // umlal swap.2d, dkl.2s, dkh.2s + * c += a; // add acc.2d, acc.2d, swap.2d + * to + * c += a; // add acc.2d, acc.2d, swap.2d + * c += b * c; // umlal acc.2d, dkl.2s, dkh.2s + * + * While it would make sense in theory since the addition is faster, + * for reasons likely related to umlal being limited to certain NEON + * pipelines, this is worse. A compiler guard fixes this. + */ + XXH_COMPILER_GUARD_CLANG_NEON(sum_1); + XXH_COMPILER_GUARD_CLANG_NEON(sum_2); + /* xacc[i] = acc_vec + sum; */ + xacc[i] = vaddq_u64(xacc[i], sum_1); + xacc[i+1] = vaddq_u64(xacc[i+1], sum_2); + } + /* Operate on the remaining NEON lanes 2 at a time. */ + for (; i < XXH3_NEON_LANES / 2; i++) { + /* data_vec = xinput[i]; */ + uint64x2_t data_vec = XXH_vld1q_u64(xinput + (i * 16)); + /* key_vec = xsecret[i]; */ + uint64x2_t key_vec = XXH_vld1q_u64(xsecret + (i * 16)); + /* acc_vec_2 = swap(data_vec) */ + uint64x2_t data_swap = vextq_u64(data_vec, data_vec, 1); + /* data_key = data_vec ^ key_vec; */ + uint64x2_t data_key = veorq_u64(data_vec, key_vec); + /* For two lanes, just use VMOVN and VSHRN. */ + /* data_key_lo = data_key & 0xFFFFFFFF; */ + uint32x2_t data_key_lo = vmovn_u64(data_key); + /* data_key_hi = data_key >> 32; */ + uint32x2_t data_key_hi = vshrn_n_u64(data_key, 32); + /* sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi; */ + uint64x2_t sum = vmlal_u32(data_swap, data_key_lo, data_key_hi); + /* Same Clang workaround as before */ + XXH_COMPILER_GUARD_CLANG_NEON(sum); + /* xacc[i] = acc_vec + sum; */ + xacc[i] = vaddq_u64 (xacc[i], sum); + } + } +} +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(neon) + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_neon(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + + { xxh_aliasing_uint64x2_t* xacc = (xxh_aliasing_uint64x2_t*) acc; + uint8_t const* xsecret = (uint8_t const*) secret; + + size_t i; + /* WASM uses operator overloads and doesn't need these. */ +#ifndef __wasm_simd128__ + /* { prime32_1, prime32_1 } */ + uint32x2_t const kPrimeLo = vdup_n_u32(XXH_PRIME32_1); + /* { 0, prime32_1, 0, prime32_1 } */ + uint32x4_t const kPrimeHi = vreinterpretq_u32_u64(vdupq_n_u64((xxh_u64)XXH_PRIME32_1 << 32)); +#endif + + /* AArch64 uses both scalar and neon at the same time */ + for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) { + XXH3_scalarScrambleRound(acc, secret, i); + } + for (i=0; i < XXH3_NEON_LANES / 2; i++) { + /* xacc[i] ^= (xacc[i] >> 47); */ + uint64x2_t acc_vec = xacc[i]; + uint64x2_t shifted = vshrq_n_u64(acc_vec, 47); + uint64x2_t data_vec = veorq_u64(acc_vec, shifted); + + /* xacc[i] ^= xsecret[i]; */ + uint64x2_t key_vec = XXH_vld1q_u64(xsecret + (i * 16)); + uint64x2_t data_key = veorq_u64(data_vec, key_vec); + /* xacc[i] *= XXH_PRIME32_1 */ +#ifdef __wasm_simd128__ + /* SIMD128 has multiply by u64x2, use it instead of expanding and scalarizing */ + xacc[i] = data_key * XXH_PRIME32_1; +#else + /* + * Expanded version with portable NEON intrinsics + * + * lo(x) * lo(y) + (hi(x) * lo(y) << 32) + * + * prod_hi = hi(data_key) * lo(prime) << 32 + * + * Since we only need 32 bits of this multiply a trick can be used, reinterpreting the vector + * as a uint32x4_t and multiplying by { 0, prime, 0, prime } to cancel out the unwanted bits + * and avoid the shift. + */ + uint32x4_t prod_hi = vmulq_u32 (vreinterpretq_u32_u64(data_key), kPrimeHi); + /* Extract low bits for vmlal_u32 */ + uint32x2_t data_key_lo = vmovn_u64(data_key); + /* xacc[i] = prod_hi + lo(data_key) * XXH_PRIME32_1; */ + xacc[i] = vmlal_u32(vreinterpretq_u64_u32(prod_hi), data_key_lo, kPrimeLo); +#endif + } + } +} +#endif + +#if (XXH_VECTOR == XXH_VSX) + +XXH_FORCE_INLINE void +XXH3_accumulate_512_vsx( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + /* presumed aligned */ + xxh_aliasing_u64x2* const xacc = (xxh_aliasing_u64x2*) acc; + xxh_u8 const* const xinput = (xxh_u8 const*) input; /* no alignment restriction */ + xxh_u8 const* const xsecret = (xxh_u8 const*) secret; /* no alignment restriction */ + xxh_u64x2 const v32 = { 32, 32 }; + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { + /* data_vec = xinput[i]; */ + xxh_u64x2 const data_vec = XXH_vec_loadu(xinput + 16*i); + /* key_vec = xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + 16*i); + xxh_u64x2 const data_key = data_vec ^ key_vec; + /* shuffled = (data_key << 32) | (data_key >> 32); */ + xxh_u32x4 const shuffled = (xxh_u32x4)vec_rl(data_key, v32); + /* product = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)shuffled & 0xFFFFFFFF); */ + xxh_u64x2 const product = XXH_vec_mulo((xxh_u32x4)data_key, shuffled); + /* acc_vec = xacc[i]; */ + xxh_u64x2 acc_vec = xacc[i]; + acc_vec += product; + + /* swap high and low halves */ +#ifdef __s390x__ + acc_vec += vec_permi(data_vec, data_vec, 2); +#else + acc_vec += vec_xxpermdi(data_vec, data_vec, 2); +#endif + xacc[i] = acc_vec; + } +} +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(vsx) + +XXH_FORCE_INLINE void +XXH3_scrambleAcc_vsx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + XXH_ASSERT((((size_t)acc) & 15) == 0); + + { xxh_aliasing_u64x2* const xacc = (xxh_aliasing_u64x2*) acc; + const xxh_u8* const xsecret = (const xxh_u8*) secret; + /* constants */ + xxh_u64x2 const v32 = { 32, 32 }; + xxh_u64x2 const v47 = { 47, 47 }; + xxh_u32x4 const prime = { XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1 }; + size_t i; + for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) { + /* xacc[i] ^= (xacc[i] >> 47); */ + xxh_u64x2 const acc_vec = xacc[i]; + xxh_u64x2 const data_vec = acc_vec ^ (acc_vec >> v47); + + /* xacc[i] ^= xsecret[i]; */ + xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + 16*i); + xxh_u64x2 const data_key = data_vec ^ key_vec; + + /* xacc[i] *= XXH_PRIME32_1 */ + /* prod_lo = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)prime & 0xFFFFFFFF); */ + xxh_u64x2 const prod_even = XXH_vec_mule((xxh_u32x4)data_key, prime); + /* prod_hi = ((xxh_u64x2)data_key >> 32) * ((xxh_u64x2)prime >> 32); */ + xxh_u64x2 const prod_odd = XXH_vec_mulo((xxh_u32x4)data_key, prime); + xacc[i] = prod_odd + (prod_even << v32); + } } +} + +#endif + +#if (XXH_VECTOR == XXH_SVE) + +XXH_FORCE_INLINE void +XXH3_accumulate_512_sve( void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + uint64_t *xacc = (uint64_t *)acc; + const uint64_t *xinput = (const uint64_t *)(const void *)input; + const uint64_t *xsecret = (const uint64_t *)(const void *)secret; + svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1); + uint64_t element_count = svcntd(); + if (element_count >= 8) { + svbool_t mask = svptrue_pat_b64(SV_VL8); + svuint64_t vacc = svld1_u64(mask, xacc); + ACCRND(vacc, 0); + svst1_u64(mask, xacc, vacc); + } else if (element_count == 2) { /* sve128 */ + svbool_t mask = svptrue_pat_b64(SV_VL2); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 2); + svuint64_t acc2 = svld1_u64(mask, xacc + 4); + svuint64_t acc3 = svld1_u64(mask, xacc + 6); + ACCRND(acc0, 0); + ACCRND(acc1, 2); + ACCRND(acc2, 4); + ACCRND(acc3, 6); + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 2, acc1); + svst1_u64(mask, xacc + 4, acc2); + svst1_u64(mask, xacc + 6, acc3); + } else { + svbool_t mask = svptrue_pat_b64(SV_VL4); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 4); + ACCRND(acc0, 0); + ACCRND(acc1, 4); + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 4, acc1); + } +} + +XXH_FORCE_INLINE void +XXH3_accumulate_sve(xxh_u64* XXH_RESTRICT acc, + const xxh_u8* XXH_RESTRICT input, + const xxh_u8* XXH_RESTRICT secret, + size_t nbStripes) +{ + if (nbStripes != 0) { + uint64_t *xacc = (uint64_t *)acc; + const uint64_t *xinput = (const uint64_t *)(const void *)input; + const uint64_t *xsecret = (const uint64_t *)(const void *)secret; + svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1); + uint64_t element_count = svcntd(); + if (element_count >= 8) { + svbool_t mask = svptrue_pat_b64(SV_VL8); + svuint64_t vacc = svld1_u64(mask, xacc + 0); + do { + /* svprfd(svbool_t, void *, enum svfprop); */ + svprfd(mask, xinput + 128, SV_PLDL1STRM); + ACCRND(vacc, 0); + xinput += 8; + xsecret += 1; + nbStripes--; + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, vacc); + } else if (element_count == 2) { /* sve128 */ + svbool_t mask = svptrue_pat_b64(SV_VL2); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 2); + svuint64_t acc2 = svld1_u64(mask, xacc + 4); + svuint64_t acc3 = svld1_u64(mask, xacc + 6); + do { + svprfd(mask, xinput + 128, SV_PLDL1STRM); + ACCRND(acc0, 0); + ACCRND(acc1, 2); + ACCRND(acc2, 4); + ACCRND(acc3, 6); + xinput += 8; + xsecret += 1; + nbStripes--; + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 2, acc1); + svst1_u64(mask, xacc + 4, acc2); + svst1_u64(mask, xacc + 6, acc3); + } else { + svbool_t mask = svptrue_pat_b64(SV_VL4); + svuint64_t acc0 = svld1_u64(mask, xacc + 0); + svuint64_t acc1 = svld1_u64(mask, xacc + 4); + do { + svprfd(mask, xinput + 128, SV_PLDL1STRM); + ACCRND(acc0, 0); + ACCRND(acc1, 4); + xinput += 8; + xsecret += 1; + nbStripes--; + } while (nbStripes != 0); + + svst1_u64(mask, xacc + 0, acc0); + svst1_u64(mask, xacc + 4, acc1); + } + } +} + +#endif + +/* scalar variants - universal */ + +#if defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__)) +/* + * In XXH3_scalarRound(), GCC and Clang have a similar codegen issue, where they + * emit an excess mask and a full 64-bit multiply-add (MADD X-form). + * + * While this might not seem like much, as AArch64 is a 64-bit architecture, only + * big Cortex designs have a full 64-bit multiplier. + * + * On the little cores, the smaller 32-bit multiplier is used, and full 64-bit + * multiplies expand to 2-3 multiplies in microcode. This has a major penalty + * of up to 4 latency cycles and 2 stall cycles in the multiply pipeline. + * + * Thankfully, AArch64 still provides the 32-bit long multiply-add (UMADDL) which does + * not have this penalty and does the mask automatically. + */ +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, xxh_u64 acc) +{ + xxh_u64 ret; + /* note: %x = 64-bit register, %w = 32-bit register */ + __asm__("umaddl %x0, %w1, %w2, %x3" : "=r" (ret) : "r" (lhs), "r" (rhs), "r" (acc)); + return ret; +} +#else +XXH_FORCE_INLINE xxh_u64 +XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, xxh_u64 acc) +{ + return XXH_mult32to64((xxh_u32)lhs, (xxh_u32)rhs) + acc; +} +#endif + +/*! + * @internal + * @brief Scalar round for @ref XXH3_accumulate_512_scalar(). + * + * This is extracted to its own function because the NEON path uses a combination + * of NEON and scalar. + */ +XXH_FORCE_INLINE void +XXH3_scalarRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT input, + void const* XXH_RESTRICT secret, + size_t lane) +{ + xxh_u64* xacc = (xxh_u64*) acc; + xxh_u8 const* xinput = (xxh_u8 const*) input; + xxh_u8 const* xsecret = (xxh_u8 const*) secret; + XXH_ASSERT(lane < XXH_ACC_NB); + XXH_ASSERT(((size_t)acc & (XXH_ACC_ALIGN-1)) == 0); + { + xxh_u64 const data_val = XXH_readLE64(xinput + lane * 8); + xxh_u64 const data_key = data_val ^ XXH_readLE64(xsecret + lane * 8); + xacc[lane ^ 1] += data_val; /* swap adjacent lanes */ + xacc[lane] = XXH_mult32to64_add64(data_key /* & 0xFFFFFFFF */, data_key >> 32, xacc[lane]); + } +} + +/*! + * @internal + * @brief Processes a 64 byte block of data using the scalar path. + */ +XXH_FORCE_INLINE void +XXH3_accumulate_512_scalar(void* XXH_RESTRICT acc, + const void* XXH_RESTRICT input, + const void* XXH_RESTRICT secret) +{ + size_t i; + /* ARM GCC refuses to unroll this loop, resulting in a 24% slowdown on ARMv6. */ +#if defined(__GNUC__) && !defined(__clang__) \ + && (defined(__arm__) || defined(__thumb2__)) \ + && defined(__ARM_FEATURE_UNALIGNED) /* no unaligned access just wastes bytes */ \ + && XXH_SIZE_OPT <= 0 +# pragma GCC unroll 8 +#endif + for (i=0; i < XXH_ACC_NB; i++) { + XXH3_scalarRound(acc, input, secret, i); + } +} +XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(scalar) + +/*! + * @internal + * @brief Scalar scramble step for @ref XXH3_scrambleAcc_scalar(). + * + * This is extracted to its own function because the NEON path uses a combination + * of NEON and scalar. + */ +XXH_FORCE_INLINE void +XXH3_scalarScrambleRound(void* XXH_RESTRICT acc, + void const* XXH_RESTRICT secret, + size_t lane) +{ + xxh_u64* const xacc = (xxh_u64*) acc; /* presumed aligned */ + const xxh_u8* const xsecret = (const xxh_u8*) secret; /* no alignment restriction */ + XXH_ASSERT((((size_t)acc) & (XXH_ACC_ALIGN-1)) == 0); + XXH_ASSERT(lane < XXH_ACC_NB); + { + xxh_u64 const key64 = XXH_readLE64(xsecret + lane * 8); + xxh_u64 acc64 = xacc[lane]; + acc64 = XXH_xorshift64(acc64, 47); + acc64 ^= key64; + acc64 *= XXH_PRIME32_1; + xacc[lane] = acc64; + } +} + +/*! + * @internal + * @brief Scrambles the accumulators after a large chunk has been read + */ +XXH_FORCE_INLINE void +XXH3_scrambleAcc_scalar(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret) +{ + size_t i; + for (i=0; i < XXH_ACC_NB; i++) { + XXH3_scalarScrambleRound(acc, secret, i); + } +} + +XXH_FORCE_INLINE void +XXH3_initCustomSecret_scalar(void* XXH_RESTRICT customSecret, xxh_u64 seed64) +{ + /* + * We need a separate pointer for the hack below, + * which requires a non-const pointer. + * Any decent compiler will optimize this out otherwise. + */ + const xxh_u8* kSecretPtr = XXH3_kSecret; + XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0); + +#if defined(__GNUC__) && defined(__aarch64__) + /* + * UGLY HACK: + * GCC and Clang generate a bunch of MOV/MOVK pairs for aarch64, and they are + * placed sequentially, in order, at the top of the unrolled loop. + * + * While MOVK is great for generating constants (2 cycles for a 64-bit + * constant compared to 4 cycles for LDR), it fights for bandwidth with + * the arithmetic instructions. + * + * I L S + * MOVK + * MOVK + * MOVK + * MOVK + * ADD + * SUB STR + * STR + * By forcing loads from memory (as the asm line causes the compiler to assume + * that XXH3_kSecretPtr has been changed), the pipelines are used more + * efficiently: + * I L S + * LDR + * ADD LDR + * SUB STR + * STR + * + * See XXH3_NEON_LANES for details on the pipsline. + * + * XXH3_64bits_withSeed, len == 256, Snapdragon 835 + * without hack: 2654.4 MB/s + * with hack: 3202.9 MB/s + */ + XXH_COMPILER_GUARD(kSecretPtr); +#endif + { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / 16; + int i; + for (i=0; i < nbRounds; i++) { + /* + * The asm hack causes the compiler to assume that kSecretPtr aliases with + * customSecret, and on aarch64, this prevented LDP from merging two + * loads together for free. Putting the loads together before the stores + * properly generates LDP. + */ + xxh_u64 lo = XXH_readLE64(kSecretPtr + 16*i) + seed64; + xxh_u64 hi = XXH_readLE64(kSecretPtr + 16*i + 8) - seed64; + XXH_writeLE64((xxh_u8*)customSecret + 16*i, lo); + XXH_writeLE64((xxh_u8*)customSecret + 16*i + 8, hi); + } } +} + + +typedef void (*XXH3_f_accumulate)(xxh_u64* XXH_RESTRICT, const xxh_u8* XXH_RESTRICT, const xxh_u8* XXH_RESTRICT, size_t); +typedef void (*XXH3_f_scrambleAcc)(void* XXH_RESTRICT, const void*); +typedef void (*XXH3_f_initCustomSecret)(void* XXH_RESTRICT, xxh_u64); + + +#if (XXH_VECTOR == XXH_AVX512) + +#define XXH3_accumulate_512 XXH3_accumulate_512_avx512 +#define XXH3_accumulate XXH3_accumulate_avx512 +#define XXH3_scrambleAcc XXH3_scrambleAcc_avx512 +#define XXH3_initCustomSecret XXH3_initCustomSecret_avx512 + +#elif (XXH_VECTOR == XXH_AVX2) + +#define XXH3_accumulate_512 XXH3_accumulate_512_avx2 +#define XXH3_accumulate XXH3_accumulate_avx2 +#define XXH3_scrambleAcc XXH3_scrambleAcc_avx2 +#define XXH3_initCustomSecret XXH3_initCustomSecret_avx2 + +#elif (XXH_VECTOR == XXH_SSE2) + +#define XXH3_accumulate_512 XXH3_accumulate_512_sse2 +#define XXH3_accumulate XXH3_accumulate_sse2 +#define XXH3_scrambleAcc XXH3_scrambleAcc_sse2 +#define XXH3_initCustomSecret XXH3_initCustomSecret_sse2 + +#elif (XXH_VECTOR == XXH_NEON) + +#define XXH3_accumulate_512 XXH3_accumulate_512_neon +#define XXH3_accumulate XXH3_accumulate_neon +#define XXH3_scrambleAcc XXH3_scrambleAcc_neon +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#elif (XXH_VECTOR == XXH_VSX) + +#define XXH3_accumulate_512 XXH3_accumulate_512_vsx +#define XXH3_accumulate XXH3_accumulate_vsx +#define XXH3_scrambleAcc XXH3_scrambleAcc_vsx +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#elif (XXH_VECTOR == XXH_SVE) +#define XXH3_accumulate_512 XXH3_accumulate_512_sve +#define XXH3_accumulate XXH3_accumulate_sve +#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#else /* scalar */ + +#define XXH3_accumulate_512 XXH3_accumulate_512_scalar +#define XXH3_accumulate XXH3_accumulate_scalar +#define XXH3_scrambleAcc XXH3_scrambleAcc_scalar +#define XXH3_initCustomSecret XXH3_initCustomSecret_scalar + +#endif + +#if XXH_SIZE_OPT >= 1 /* don't do SIMD for initialization */ +# undef XXH3_initCustomSecret +# define XXH3_initCustomSecret XXH3_initCustomSecret_scalar +#endif + +XXH_FORCE_INLINE void +XXH3_hashLong_internal_loop(xxh_u64* XXH_RESTRICT acc, + const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + size_t const nbStripesPerBlock = (secretSize - XXH_STRIPE_LEN) / XXH_SECRET_CONSUME_RATE; + size_t const block_len = XXH_STRIPE_LEN * nbStripesPerBlock; + size_t const nb_blocks = (len - 1) / block_len; + + size_t n; + + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + + for (n = 0; n < nb_blocks; n++) { + f_acc(acc, input + n*block_len, secret, nbStripesPerBlock); + f_scramble(acc, secret + secretSize - XXH_STRIPE_LEN); + } + + /* last partial block */ + XXH_ASSERT(len > XXH_STRIPE_LEN); + { size_t const nbStripes = ((len - 1) - (block_len * nb_blocks)) / XXH_STRIPE_LEN; + XXH_ASSERT(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE)); + f_acc(acc, input + nb_blocks*block_len, secret, nbStripes); + + /* last stripe */ + { const xxh_u8* const p = input + len - XXH_STRIPE_LEN; +#define XXH_SECRET_LASTACC_START 7 /* not aligned on 8, last secret is different from acc & scrambler */ + XXH3_accumulate_512(acc, p, secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START); + } } +} + +XXH_FORCE_INLINE xxh_u64 +XXH3_mix2Accs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret) +{ + return XXH3_mul128_fold64( + acc[0] ^ XXH_readLE64(secret), + acc[1] ^ XXH_readLE64(secret+8) ); +} + +static XXH64_hash_t +XXH3_mergeAccs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 start) +{ + xxh_u64 result64 = start; + size_t i = 0; + + for (i = 0; i < 4; i++) { + result64 += XXH3_mix2Accs(acc+2*i, secret + 16*i); +#if defined(__clang__) /* Clang */ \ + && (defined(__arm__) || defined(__thumb__)) /* ARMv7 */ \ + && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \ + && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */ + /* + * UGLY HACK: + * Prevent autovectorization on Clang ARMv7-a. Exact same problem as + * the one in XXH3_len_129to240_64b. Speeds up shorter keys > 240b. + * XXH3_64bits, len == 256, Snapdragon 835: + * without hack: 2063.7 MB/s + * with hack: 2560.7 MB/s + */ + XXH_COMPILER_GUARD(result64); +#endif + } + + return XXH3_avalanche(result64); +} + +#define XXH3_INIT_ACC { XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \ + XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 } + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_internal(const void* XXH_RESTRICT input, size_t len, + const void* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; + + XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize, f_acc, f_scramble); + + /* converge into final hash */ + XXH_STATIC_ASSERT(sizeof(acc) == 64); + /* do not align on 8, so that the secret is different from the accumulator */ +#define XXH_SECRET_MERGEACCS_START 11 + XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + return XXH3_mergeAccs(acc, (const xxh_u8*)secret + XXH_SECRET_MERGEACCS_START, (xxh_u64)len * XXH_PRIME64_1); +} + +/* + * It's important for performance to transmit secret's size (when it's static) + * so that the compiler can properly optimize the vectorized loop. + * This makes a big performance difference for "medium" keys (<1 KB) when using AVX instruction set. + * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and FORCE_INLINE + * breaks -Og, this is XXH_NO_INLINE. + */ +XXH3_WITH_SECRET_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSecret(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; + return XXH3_hashLong_64b_internal(input, len, secret, secretLen, XXH3_accumulate, XXH3_scrambleAcc); +} + +/* + * It's preferable for performance that XXH3_hashLong is not inlined, + * as it results in a smaller function for small data, easier to the instruction cache. + * Note that inside this no_inline function, we do inline the internal loop, + * and provide a statically defined secret size to allow optimization of vector loop. + */ +XXH_NO_INLINE XXH_PUREF XXH64_hash_t +XXH3_hashLong_64b_default(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + return XXH3_hashLong_64b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_accumulate, XXH3_scrambleAcc); +} + +/* + * XXH3_hashLong_64b_withSeed(): + * Generate a custom key based on alteration of default XXH3_kSecret with the seed, + * and then use this key for long mode hashing. + * + * This operation is decently fast but nonetheless costs a little bit of time. + * Try to avoid it whenever possible (typically when seed==0). + * + * It's important for performance that XXH3_hashLong is not inlined. Not sure + * why (uop cache maybe?), but the difference is large and easily measurable. + */ +XXH_FORCE_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSeed_internal(const void* input, size_t len, + XXH64_hash_t seed, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble, + XXH3_f_initCustomSecret f_initSec) +{ +#if XXH_SIZE_OPT <= 0 + if (seed == 0) + return XXH3_hashLong_64b_internal(input, len, + XXH3_kSecret, sizeof(XXH3_kSecret), + f_acc, f_scramble); +#endif + { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed); + return XXH3_hashLong_64b_internal(input, len, secret, sizeof(secret), + f_acc, f_scramble); + } +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH64_hash_t +XXH3_hashLong_64b_withSeed(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed, const xxh_u8* XXH_RESTRICT secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + return XXH3_hashLong_64b_withSeed_internal(input, len, seed, + XXH3_accumulate, XXH3_scrambleAcc, XXH3_initCustomSecret); +} + + +typedef XXH64_hash_t (*XXH3_hashLong64_f)(const void* XXH_RESTRICT, size_t, + XXH64_hash_t, const xxh_u8* XXH_RESTRICT, size_t); + +XXH_FORCE_INLINE XXH64_hash_t +XXH3_64bits_internal(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, + XXH3_hashLong64_f f_hashLong) +{ + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); + /* + * If an action is to be taken if `secretLen` condition is not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + * Also, note that function signature doesn't offer room to return an error. + */ + if (len <= 16) + return XXH3_len_0to16_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + return f_hashLong(input, len, seed64, (const xxh_u8*)secret, secretLen); +} + + +/* === Public entry point === */ + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length) +{ + return XXH3_64bits_internal(input, length, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_default); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecret(XXH_NOESCAPE const void* input, size_t length, XXH_NOESCAPE const void* secret, size_t secretSize) +{ + return XXH3_64bits_internal(input, length, 0, secret, secretSize, XXH3_hashLong_64b_withSecret); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed) +{ + return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed); +} + +XXH_PUBLIC_API XXH64_hash_t +XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + if (length <= XXH3_MIDSIZE_MAX) + return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); + return XXH3_hashLong_64b_withSecret(input, length, seed, (const xxh_u8*)secret, secretSize); +} + + +/* === XXH3 streaming === */ +#ifndef XXH_NO_STREAM +/* + * Malloc's a pointer that is always aligned to align. + * + * This must be freed with `XXH_alignedFree()`. + * + * malloc typically guarantees 16 byte alignment on 64-bit systems and 8 byte + * alignment on 32-bit. This isn't enough for the 32 byte aligned loads in AVX2 + * or on 32-bit, the 16 byte aligned loads in SSE2 and NEON. + * + * This underalignment previously caused a rather obvious crash which went + * completely unnoticed due to XXH3_createState() not actually being tested. + * Credit to RedSpah for noticing this bug. + * + * The alignment is done manually: Functions like posix_memalign or _mm_malloc + * are avoided: To maintain portability, we would have to write a fallback + * like this anyways, and besides, testing for the existence of library + * functions without relying on external build tools is impossible. + * + * The method is simple: Overallocate, manually align, and store the offset + * to the original behind the returned pointer. + * + * Align must be a power of 2 and 8 <= align <= 128. + */ +static XXH_MALLOCF void* XXH_alignedMalloc(size_t s, size_t align) +{ + XXH_ASSERT(align <= 128 && align >= 8); /* range check */ + XXH_ASSERT((align & (align-1)) == 0); /* power of 2 */ + XXH_ASSERT(s != 0 && s < (s + align)); /* empty/overflow */ + { /* Overallocate to make room for manual realignment and an offset byte */ + xxh_u8* base = (xxh_u8*)XXH_malloc(s + align); + if (base != NULL) { + /* + * Get the offset needed to align this pointer. + * + * Even if the returned pointer is aligned, there will always be + * at least one byte to store the offset to the original pointer. + */ + size_t offset = align - ((size_t)base & (align - 1)); /* base % align */ + /* Add the offset for the now-aligned pointer */ + xxh_u8* ptr = base + offset; + + XXH_ASSERT((size_t)ptr % align == 0); + + /* Store the offset immediately before the returned pointer. */ + ptr[-1] = (xxh_u8)offset; + return ptr; + } + return NULL; + } +} +/* + * Frees an aligned pointer allocated by XXH_alignedMalloc(). Don't pass + * normal malloc'd pointers, XXH_alignedMalloc has a specific data layout. + */ +static void XXH_alignedFree(void* p) +{ + if (p != NULL) { + xxh_u8* ptr = (xxh_u8*)p; + /* Get the offset byte we added in XXH_malloc. */ + xxh_u8 offset = ptr[-1]; + /* Free the original malloc'd pointer */ + xxh_u8* base = ptr - offset; + XXH_free(base); + } +} +/*! @ingroup XXH3_family */ +/*! + * @brief Allocate an @ref XXH3_state_t. + * + * @return An allocated pointer of @ref XXH3_state_t on success. + * @return `NULL` on failure. + * + * @note Must be freed with XXH3_freeState(). + */ +XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void) +{ + XXH3_state_t* const state = (XXH3_state_t*)XXH_alignedMalloc(sizeof(XXH3_state_t), 64); + if (state==NULL) return NULL; + XXH3_INITSTATE(state); + return state; +} + +/*! @ingroup XXH3_family */ +/*! + * @brief Frees an @ref XXH3_state_t. + * + * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState(). + * + * @return @ref XXH_OK. + * + * @note Must be allocated with XXH3_createState(). + */ +XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr) +{ + XXH_alignedFree(statePtr); + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API void +XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOESCAPE const XXH3_state_t* src_state) +{ + XXH_memcpy(dst_state, src_state, sizeof(*dst_state)); +} + +static void +XXH3_reset_internal(XXH3_state_t* statePtr, + XXH64_hash_t seed, + const void* secret, size_t secretSize) +{ + size_t const initStart = offsetof(XXH3_state_t, bufferedSize); + size_t const initLength = offsetof(XXH3_state_t, nbStripesPerBlock) - initStart; + XXH_ASSERT(offsetof(XXH3_state_t, nbStripesPerBlock) > initStart); + XXH_ASSERT(statePtr != NULL); + /* set members from bufferedSize to nbStripesPerBlock (excluded) to 0 */ + memset((char*)statePtr + initStart, 0, initLength); + statePtr->acc[0] = XXH_PRIME32_3; + statePtr->acc[1] = XXH_PRIME64_1; + statePtr->acc[2] = XXH_PRIME64_2; + statePtr->acc[3] = XXH_PRIME64_3; + statePtr->acc[4] = XXH_PRIME64_4; + statePtr->acc[5] = XXH_PRIME32_2; + statePtr->acc[6] = XXH_PRIME64_5; + statePtr->acc[7] = XXH_PRIME32_1; + statePtr->seed = seed; + statePtr->useSeed = (seed != 0); + statePtr->extSecret = (const unsigned char*)secret; + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); + statePtr->secretLimit = secretSize - XXH_STRIPE_LEN; + statePtr->nbStripesPerBlock = statePtr->secretLimit / XXH_SECRET_CONSUME_RATE; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr) +{ + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize) +{ + if (statePtr == NULL) return XXH_ERROR; + XXH3_reset_internal(statePtr, 0, secret, secretSize); + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed) +{ + if (statePtr == NULL) return XXH_ERROR; + if (seed==0) return XXH3_64bits_reset(statePtr); + if ((seed != statePtr->seed) || (statePtr->extSecret != NULL)) + XXH3_initCustomSecret(statePtr->customSecret, seed); + XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE); + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed64) +{ + if (statePtr == NULL) return XXH_ERROR; + if (secret == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; + XXH3_reset_internal(statePtr, seed64, secret, secretSize); + statePtr->useSeed = 1; /* always, even if seed64==0 */ + return XXH_OK; +} + +/*! + * @internal + * @brief Processes a large input for XXH3_update() and XXH3_digest_long(). + * + * Unlike XXH3_hashLong_internal_loop(), this can process data that overlaps a block. + * + * @param acc Pointer to the 8 accumulator lanes + * @param nbStripesSoFarPtr In/out pointer to the number of leftover stripes in the block* + * @param nbStripesPerBlock Number of stripes in a block + * @param input Input pointer + * @param nbStripes Number of stripes to process + * @param secret Secret pointer + * @param secretLimit Offset of the last block in @p secret + * @param f_acc Pointer to an XXH3_accumulate implementation + * @param f_scramble Pointer to an XXH3_scrambleAcc implementation + * @return Pointer past the end of @p input after processing + */ +XXH_FORCE_INLINE const xxh_u8 * +XXH3_consumeStripes(xxh_u64* XXH_RESTRICT acc, + size_t* XXH_RESTRICT nbStripesSoFarPtr, size_t nbStripesPerBlock, + const xxh_u8* XXH_RESTRICT input, size_t nbStripes, + const xxh_u8* XXH_RESTRICT secret, size_t secretLimit, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + const xxh_u8* initialSecret = secret + *nbStripesSoFarPtr * XXH_SECRET_CONSUME_RATE; + /* Process full blocks */ + if (nbStripes >= (nbStripesPerBlock - *nbStripesSoFarPtr)) { + /* Process the initial partial block... */ + size_t nbStripesThisIter = nbStripesPerBlock - *nbStripesSoFarPtr; + + do { + /* Accumulate and scramble */ + f_acc(acc, input, initialSecret, nbStripesThisIter); + f_scramble(acc, secret + secretLimit); + input += nbStripesThisIter * XXH_STRIPE_LEN; + nbStripes -= nbStripesThisIter; + /* Then continue the loop with the full block size */ + nbStripesThisIter = nbStripesPerBlock; + initialSecret = secret; + } while (nbStripes >= nbStripesPerBlock); + *nbStripesSoFarPtr = 0; + } + /* Process a partial block */ + if (nbStripes > 0) { + f_acc(acc, input, initialSecret, nbStripes); + input += nbStripes * XXH_STRIPE_LEN; + *nbStripesSoFarPtr += nbStripes; + } + /* Return end pointer */ + return input; +} + +#ifndef XXH3_STREAM_USE_STACK +# if XXH_SIZE_OPT <= 0 && !defined(__clang__) /* clang doesn't need additional stack space */ +# define XXH3_STREAM_USE_STACK 1 +# endif +#endif +/* + * Both XXH3_64bits_update and XXH3_128bits_update use this routine. + */ +XXH_FORCE_INLINE XXH_errorcode +XXH3_update(XXH3_state_t* XXH_RESTRICT const state, + const xxh_u8* XXH_RESTRICT input, size_t len, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + if (input==NULL) { + XXH_ASSERT(len == 0); + return XXH_OK; + } + + XXH_ASSERT(state != NULL); + { const xxh_u8* const bEnd = input + len; + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; +#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 + /* For some reason, gcc and MSVC seem to suffer greatly + * when operating accumulators directly into state. + * Operating into stack space seems to enable proper optimization. + * clang, on the other hand, doesn't seem to need this trick */ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[8]; + XXH_memcpy(acc, state->acc, sizeof(acc)); +#else + xxh_u64* XXH_RESTRICT const acc = state->acc; +#endif + state->totalLen += len; + XXH_ASSERT(state->bufferedSize <= XXH3_INTERNALBUFFER_SIZE); + + /* small input : just fill in tmp buffer */ + if (len <= XXH3_INTERNALBUFFER_SIZE - state->bufferedSize) { + XXH_memcpy(state->buffer + state->bufferedSize, input, len); + state->bufferedSize += (XXH32_hash_t)len; + return XXH_OK; + } + + /* total input is now > XXH3_INTERNALBUFFER_SIZE */ + #define XXH3_INTERNALBUFFER_STRIPES (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN) + XXH_STATIC_ASSERT(XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN == 0); /* clean multiple */ + + /* + * Internal buffer is partially filled (always, except at beginning) + * Complete it, then consume it. + */ + if (state->bufferedSize) { + size_t const loadSize = XXH3_INTERNALBUFFER_SIZE - state->bufferedSize; + XXH_memcpy(state->buffer + state->bufferedSize, input, loadSize); + input += loadSize; + XXH3_consumeStripes(acc, + &state->nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, XXH3_INTERNALBUFFER_STRIPES, + secret, state->secretLimit, + f_acc, f_scramble); + state->bufferedSize = 0; + } + XXH_ASSERT(input < bEnd); + if (bEnd - input > XXH3_INTERNALBUFFER_SIZE) { + size_t nbStripes = (size_t)(bEnd - 1 - input) / XXH_STRIPE_LEN; + input = XXH3_consumeStripes(acc, + &state->nbStripesSoFar, state->nbStripesPerBlock, + input, nbStripes, + secret, state->secretLimit, + f_acc, f_scramble); + XXH_memcpy(state->buffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN); + + } + /* Some remaining input (always) : buffer it */ + XXH_ASSERT(input < bEnd); + XXH_ASSERT(bEnd - input <= XXH3_INTERNALBUFFER_SIZE); + XXH_ASSERT(state->bufferedSize == 0); + XXH_memcpy(state->buffer, input, (size_t)(bEnd-input)); + state->bufferedSize = (XXH32_hash_t)(bEnd-input); +#if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1 + /* save stack accumulators into state */ + XXH_memcpy(state->acc, acc, sizeof(acc)); +#endif + } + + return XXH_OK; +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_64bits_update(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len) +{ + return XXH3_update(state, (const xxh_u8*)input, len, + XXH3_accumulate, XXH3_scrambleAcc); +} + + +XXH_FORCE_INLINE void +XXH3_digest_long (XXH64_hash_t* acc, + const XXH3_state_t* state, + const unsigned char* secret) +{ + xxh_u8 lastStripe[XXH_STRIPE_LEN]; + const xxh_u8* lastStripePtr; + + /* + * Digest on a local copy. This way, the state remains unaltered, and it can + * continue ingesting more input afterwards. + */ + XXH_memcpy(acc, state->acc, sizeof(state->acc)); + if (state->bufferedSize >= XXH_STRIPE_LEN) { + /* Consume remaining stripes then point to remaining data in buffer */ + size_t const nbStripes = (state->bufferedSize - 1) / XXH_STRIPE_LEN; + size_t nbStripesSoFar = state->nbStripesSoFar; + XXH3_consumeStripes(acc, + &nbStripesSoFar, state->nbStripesPerBlock, + state->buffer, nbStripes, + secret, state->secretLimit, + XXH3_accumulate, XXH3_scrambleAcc); + lastStripePtr = state->buffer + state->bufferedSize - XXH_STRIPE_LEN; + } else { /* bufferedSize < XXH_STRIPE_LEN */ + /* Copy to temp buffer */ + size_t const catchupSize = XXH_STRIPE_LEN - state->bufferedSize; + XXH_ASSERT(state->bufferedSize > 0); /* there is always some input buffered */ + XXH_memcpy(lastStripe, state->buffer + sizeof(state->buffer) - catchupSize, catchupSize); + XXH_memcpy(lastStripe + catchupSize, state->buffer, state->bufferedSize); + lastStripePtr = lastStripe; + } + /* Last stripe */ + XXH3_accumulate_512(acc, + lastStripePtr, + secret + state->secretLimit - XXH_SECRET_LASTACC_START); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* state) +{ + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + return XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)state->totalLen * XXH_PRIME64_1); + } + /* totalLen <= XXH3_MIDSIZE_MAX: digesting a short input */ + if (state->useSeed) + return XXH3_64bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); + return XXH3_64bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); +} +#endif /* !XXH_NO_STREAM */ + + +/* ========================================== + * XXH3 128 bits (a.k.a XXH128) + * ========================================== + * XXH3's 128-bit variant has better mixing and strength than the 64-bit variant, + * even without counting the significantly larger output size. + * + * For example, extra steps are taken to avoid the seed-dependent collisions + * in 17-240 byte inputs (See XXH3_mix16B and XXH128_mix32B). + * + * This strength naturally comes at the cost of some speed, especially on short + * lengths. Note that longer hashes are about as fast as the 64-bit version + * due to it using only a slight modification of the 64-bit loop. + * + * XXH128 is also more oriented towards 64-bit machines. It is still extremely + * fast for a _128-bit_ hash on 32-bit (it usually clears XXH64). + */ + +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_1to3_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + /* A doubled version of 1to3_64b with different constants. */ + XXH_ASSERT(input != NULL); + XXH_ASSERT(1 <= len && len <= 3); + XXH_ASSERT(secret != NULL); + /* + * len = 1: combinedl = { input[0], 0x01, input[0], input[0] } + * len = 2: combinedl = { input[1], 0x02, input[0], input[1] } + * len = 3: combinedl = { input[2], 0x03, input[0], input[1] } + */ + { xxh_u8 const c1 = input[0]; + xxh_u8 const c2 = input[len >> 1]; + xxh_u8 const c3 = input[len - 1]; + xxh_u32 const combinedl = ((xxh_u32)c1 <<16) | ((xxh_u32)c2 << 24) + | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8); + xxh_u32 const combinedh = XXH_rotl32(XXH_swap32(combinedl), 13); + xxh_u64 const bitflipl = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed; + xxh_u64 const bitfliph = (XXH_readLE32(secret+8) ^ XXH_readLE32(secret+12)) - seed; + xxh_u64 const keyed_lo = (xxh_u64)combinedl ^ bitflipl; + xxh_u64 const keyed_hi = (xxh_u64)combinedh ^ bitfliph; + XXH128_hash_t h128; + h128.low64 = XXH64_avalanche(keyed_lo); + h128.high64 = XXH64_avalanche(keyed_hi); + return h128; + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_4to8_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(4 <= len && len <= 8); + seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32; + { xxh_u32 const input_lo = XXH_readLE32(input); + xxh_u32 const input_hi = XXH_readLE32(input + len - 4); + xxh_u64 const input_64 = input_lo + ((xxh_u64)input_hi << 32); + xxh_u64 const bitflip = (XXH_readLE64(secret+16) ^ XXH_readLE64(secret+24)) + seed; + xxh_u64 const keyed = input_64 ^ bitflip; + + /* Shift len to the left to ensure it is even, this avoids even multiplies. */ + XXH128_hash_t m128 = XXH_mult64to128(keyed, XXH_PRIME64_1 + (len << 2)); + + m128.high64 += (m128.low64 << 1); + m128.low64 ^= (m128.high64 >> 3); + + m128.low64 = XXH_xorshift64(m128.low64, 35); + m128.low64 *= PRIME_MX2; + m128.low64 = XXH_xorshift64(m128.low64, 28); + m128.high64 = XXH3_avalanche(m128.high64); + return m128; + } +} + +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_9to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(input != NULL); + XXH_ASSERT(secret != NULL); + XXH_ASSERT(9 <= len && len <= 16); + { xxh_u64 const bitflipl = (XXH_readLE64(secret+32) ^ XXH_readLE64(secret+40)) - seed; + xxh_u64 const bitfliph = (XXH_readLE64(secret+48) ^ XXH_readLE64(secret+56)) + seed; + xxh_u64 const input_lo = XXH_readLE64(input); + xxh_u64 input_hi = XXH_readLE64(input + len - 8); + XXH128_hash_t m128 = XXH_mult64to128(input_lo ^ input_hi ^ bitflipl, XXH_PRIME64_1); + /* + * Put len in the middle of m128 to ensure that the length gets mixed to + * both the low and high bits in the 128x64 multiply below. + */ + m128.low64 += (xxh_u64)(len - 1) << 54; + input_hi ^= bitfliph; + /* + * Add the high 32 bits of input_hi to the high 32 bits of m128, then + * add the long product of the low 32 bits of input_hi and XXH_PRIME32_2 to + * the high 64 bits of m128. + * + * The best approach to this operation is different on 32-bit and 64-bit. + */ + if (sizeof(void *) < sizeof(xxh_u64)) { /* 32-bit */ + /* + * 32-bit optimized version, which is more readable. + * + * On 32-bit, it removes an ADC and delays a dependency between the two + * halves of m128.high64, but it generates an extra mask on 64-bit. + */ + m128.high64 += (input_hi & 0xFFFFFFFF00000000ULL) + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2); + } else { + /* + * 64-bit optimized (albeit more confusing) version. + * + * Uses some properties of addition and multiplication to remove the mask: + * + * Let: + * a = input_hi.lo = (input_hi & 0x00000000FFFFFFFF) + * b = input_hi.hi = (input_hi & 0xFFFFFFFF00000000) + * c = XXH_PRIME32_2 + * + * a + (b * c) + * Inverse Property: x + y - x == y + * a + (b * (1 + c - 1)) + * Distributive Property: x * (y + z) == (x * y) + (x * z) + * a + (b * 1) + (b * (c - 1)) + * Identity Property: x * 1 == x + * a + b + (b * (c - 1)) + * + * Substitute a, b, and c: + * input_hi.hi + input_hi.lo + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) + * + * Since input_hi.hi + input_hi.lo == input_hi, we get this: + * input_hi + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1)) + */ + m128.high64 += input_hi + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2 - 1); + } + /* m128 ^= XXH_swap64(m128 >> 64); */ + m128.low64 ^= XXH_swap64(m128.high64); + + { /* 128x64 multiply: h128 = m128 * XXH_PRIME64_2; */ + XXH128_hash_t h128 = XXH_mult64to128(m128.low64, XXH_PRIME64_2); + h128.high64 += m128.high64 * XXH_PRIME64_2; + + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = XXH3_avalanche(h128.high64); + return h128; + } } +} + +/* + * Assumption: `secret` size is >= XXH3_SECRET_SIZE_MIN + */ +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_0to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed) +{ + XXH_ASSERT(len <= 16); + { if (len > 8) return XXH3_len_9to16_128b(input, len, secret, seed); + if (len >= 4) return XXH3_len_4to8_128b(input, len, secret, seed); + if (len) return XXH3_len_1to3_128b(input, len, secret, seed); + { XXH128_hash_t h128; + xxh_u64 const bitflipl = XXH_readLE64(secret+64) ^ XXH_readLE64(secret+72); + xxh_u64 const bitfliph = XXH_readLE64(secret+80) ^ XXH_readLE64(secret+88); + h128.low64 = XXH64_avalanche(seed ^ bitflipl); + h128.high64 = XXH64_avalanche( seed ^ bitfliph); + return h128; + } } +} + +/* + * A bit slower than XXH3_mix16B, but handles multiply by zero better. + */ +XXH_FORCE_INLINE XXH128_hash_t +XXH128_mix32B(XXH128_hash_t acc, const xxh_u8* input_1, const xxh_u8* input_2, + const xxh_u8* secret, XXH64_hash_t seed) +{ + acc.low64 += XXH3_mix16B (input_1, secret+0, seed); + acc.low64 ^= XXH_readLE64(input_2) + XXH_readLE64(input_2 + 8); + acc.high64 += XXH3_mix16B (input_2, secret+16, seed); + acc.high64 ^= XXH_readLE64(input_1) + XXH_readLE64(input_1 + 8); + return acc; +} + + +XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_17to128_128b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(16 < len && len <= 128); + + { XXH128_hash_t acc; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; + +#if XXH_SIZE_OPT >= 1 + { + /* Smaller, but slightly slower. */ + unsigned int i = (unsigned int)(len - 1) / 32; + do { + acc = XXH128_mix32B(acc, input+16*i, input+len-16*(i+1), secret+32*i, seed); + } while (i-- != 0); + } +#else + if (len > 32) { + if (len > 64) { + if (len > 96) { + acc = XXH128_mix32B(acc, input+48, input+len-64, secret+96, seed); + } + acc = XXH128_mix32B(acc, input+32, input+len-48, secret+64, seed); + } + acc = XXH128_mix32B(acc, input+16, input+len-32, secret+32, seed); + } + acc = XXH128_mix32B(acc, input, input+len-16, secret, seed); +#endif + { XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; + } + } +} + +XXH_NO_INLINE XXH_PUREF XXH128_hash_t +XXH3_len_129to240_128b(const xxh_u8* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH64_hash_t seed) +{ + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize; + XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX); + + { XXH128_hash_t acc; + unsigned i; + acc.low64 = len * XXH_PRIME64_1; + acc.high64 = 0; + /* + * We set as `i` as offset + 32. We do this so that unchanged + * `len` can be used as upper bound. This reaches a sweet spot + * where both x86 and aarch64 get simple agen and good codegen + * for the loop. + */ + for (i = 32; i < 160; i += 32) { + acc = XXH128_mix32B(acc, + input + i - 32, + input + i - 16, + secret + i - 32, + seed); + } + acc.low64 = XXH3_avalanche(acc.low64); + acc.high64 = XXH3_avalanche(acc.high64); + /* + * NB: `i <= len` will duplicate the last 32-bytes if + * len % 32 was zero. This is an unfortunate necessity to keep + * the hash result stable. + */ + for (i=160; i <= len; i += 32) { + acc = XXH128_mix32B(acc, + input + i - 32, + input + i - 16, + secret + XXH3_MIDSIZE_STARTOFFSET + i - 160, + seed); + } + /* last bytes */ + acc = XXH128_mix32B(acc, + input + len - 16, + input + len - 32, + secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET - 16, + (XXH64_hash_t)0 - seed); + + { XXH128_hash_t h128; + h128.low64 = acc.low64 + acc.high64; + h128.high64 = (acc.low64 * XXH_PRIME64_1) + + (acc.high64 * XXH_PRIME64_4) + + ((len - seed) * XXH_PRIME64_2); + h128.low64 = XXH3_avalanche(h128.low64); + h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64); + return h128; + } + } +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_internal(const void* XXH_RESTRICT input, size_t len, + const xxh_u8* XXH_RESTRICT secret, size_t secretSize, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble) +{ + XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC; + + XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, secret, secretSize, f_acc, f_scramble); + + /* converge into final hash */ + XXH_STATIC_ASSERT(sizeof(acc) == 64); + XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + { XXH128_hash_t h128; + h128.low64 = XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)len * XXH_PRIME64_1); + h128.high64 = XXH3_mergeAccs(acc, + secret + secretSize + - sizeof(acc) - XXH_SECRET_MERGEACCS_START, + ~((xxh_u64)len * XXH_PRIME64_2)); + return h128; + } +} + +/* + * It's important for performance that XXH3_hashLong() is not inlined. + */ +XXH_NO_INLINE XXH_PUREF XXH128_hash_t +XXH3_hashLong_128b_default(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; (void)secret; (void)secretLen; + return XXH3_hashLong_128b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_accumulate, XXH3_scrambleAcc); +} + +/* + * It's important for performance to pass @p secretLen (when it's static) + * to the compiler, so that it can properly optimize the vectorized loop. + * + * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and FORCE_INLINE + * breaks -Og, this is XXH_NO_INLINE. + */ +XXH3_WITH_SECRET_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSecret(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)seed64; + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, secretLen, + XXH3_accumulate, XXH3_scrambleAcc); +} + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSeed_internal(const void* XXH_RESTRICT input, size_t len, + XXH64_hash_t seed64, + XXH3_f_accumulate f_acc, + XXH3_f_scrambleAcc f_scramble, + XXH3_f_initCustomSecret f_initSec) +{ + if (seed64 == 0) + return XXH3_hashLong_128b_internal(input, len, + XXH3_kSecret, sizeof(XXH3_kSecret), + f_acc, f_scramble); + { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE]; + f_initSec(secret, seed64); + return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, sizeof(secret), + f_acc, f_scramble); + } +} + +/* + * It's important for performance that XXH3_hashLong is not inlined. + */ +XXH_NO_INLINE XXH128_hash_t +XXH3_hashLong_128b_withSeed(const void* input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen) +{ + (void)secret; (void)secretLen; + return XXH3_hashLong_128b_withSeed_internal(input, len, seed64, + XXH3_accumulate, XXH3_scrambleAcc, XXH3_initCustomSecret); +} + +typedef XXH128_hash_t (*XXH3_hashLong128_f)(const void* XXH_RESTRICT, size_t, + XXH64_hash_t, const void* XXH_RESTRICT, size_t); + +XXH_FORCE_INLINE XXH128_hash_t +XXH3_128bits_internal(const void* input, size_t len, + XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen, + XXH3_hashLong128_f f_hl128) +{ + XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN); + /* + * If an action is to be taken if `secret` conditions are not respected, + * it should be done here. + * For now, it's a contract pre-condition. + * Adding a check and a branch here would cost performance at every hash. + */ + if (len <= 16) + return XXH3_len_0to16_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64); + if (len <= 128) + return XXH3_len_17to128_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_len_129to240_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64); + return f_hl128(input, len, seed64, secret, secretLen); +} + + +/* === Public XXH128 API === */ + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* input, size_t len) +{ + return XXH3_128bits_internal(input, len, 0, + XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_hashLong_128b_default); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecret(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize) +{ + return XXH3_128bits_internal(input, len, 0, + (const xxh_u8*)secret, secretSize, + XXH3_hashLong_128b_withSecret); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSeed(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_128bits_internal(input, len, seed, + XXH3_kSecret, sizeof(XXH3_kSecret), + XXH3_hashLong_128b_withSeed); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + if (len <= XXH3_MIDSIZE_MAX) + return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL); + return XXH3_hashLong_128b_withSecret(input, len, seed, secret, secretSize); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH128(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed) +{ + return XXH3_128bits_withSeed(input, len, seed); +} + + +/* === XXH3 128-bit streaming === */ +#ifndef XXH_NO_STREAM +/* + * All initialization and update functions are identical to 64-bit streaming variant. + * The only difference is the finalization routine. + */ + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr) +{ + return XXH3_64bits_reset(statePtr); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize) +{ + return XXH3_64bits_reset_withSecret(statePtr, secret, secretSize); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed) +{ + return XXH3_64bits_reset_withSeed(statePtr, seed); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed) +{ + return XXH3_64bits_reset_withSecretandSeed(statePtr, secret, secretSize, seed); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_128bits_update(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len) +{ + return XXH3_64bits_update(state, input, len); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_t* state) +{ + const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret; + if (state->totalLen > XXH3_MIDSIZE_MAX) { + XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB]; + XXH3_digest_long(acc, state, secret); + XXH_ASSERT(state->secretLimit + XXH_STRIPE_LEN >= sizeof(acc) + XXH_SECRET_MERGEACCS_START); + { XXH128_hash_t h128; + h128.low64 = XXH3_mergeAccs(acc, + secret + XXH_SECRET_MERGEACCS_START, + (xxh_u64)state->totalLen * XXH_PRIME64_1); + h128.high64 = XXH3_mergeAccs(acc, + secret + state->secretLimit + XXH_STRIPE_LEN + - sizeof(acc) - XXH_SECRET_MERGEACCS_START, + ~((xxh_u64)state->totalLen * XXH_PRIME64_2)); + return h128; + } + } + /* len <= XXH3_MIDSIZE_MAX : short code */ + if (state->seed) + return XXH3_128bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed); + return XXH3_128bits_withSecret(state->buffer, (size_t)(state->totalLen), + secret, state->secretLimit + XXH_STRIPE_LEN); +} +#endif /* !XXH_NO_STREAM */ +/* 128-bit utility functions */ + +#include /* memcmp, memcpy */ + +/* return : 1 is equal, 0 if different */ +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2) +{ + /* note : XXH128_hash_t is compact, it has no padding byte */ + return !(memcmp(&h1, &h2, sizeof(h1))); +} + +/* This prototype is compatible with stdlib's qsort(). + * @return : >0 if *h128_1 > *h128_2 + * <0 if *h128_1 < *h128_2 + * =0 if *h128_1 == *h128_2 */ +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2) +{ + XXH128_hash_t const h1 = *(const XXH128_hash_t*)h128_1; + XXH128_hash_t const h2 = *(const XXH128_hash_t*)h128_2; + int const hcmp = (h1.high64 > h2.high64) - (h2.high64 > h1.high64); + /* note : bets that, in most cases, hash values are different */ + if (hcmp) return hcmp; + return (h1.low64 > h2.low64) - (h2.low64 > h1.low64); +} + + +/*====== Canonical representation ======*/ +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API void +XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH128_canonical_t) == sizeof(XXH128_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) { + hash.high64 = XXH_swap64(hash.high64); + hash.low64 = XXH_swap64(hash.low64); + } + XXH_memcpy(dst, &hash.high64, sizeof(hash.high64)); + XXH_memcpy((char*)dst + sizeof(hash.high64), &hash.low64, sizeof(hash.low64)); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH128_hash_t +XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src) +{ + XXH128_hash_t h; + h.high64 = XXH_readBE64(src); + h.low64 = XXH_readBE64(src->digest + 8); + return h; +} + + + +/* ========================================== + * Secret generators + * ========================================== + */ +#define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x)) + +XXH_FORCE_INLINE void XXH3_combine16(void* dst, XXH128_hash_t h128) +{ + XXH_writeLE64( dst, XXH_readLE64(dst) ^ h128.low64 ); + XXH_writeLE64( (char*)dst+8, XXH_readLE64((char*)dst+8) ^ h128.high64 ); +} + +/*! @ingroup XXH3_family */ +XXH_PUBLIC_API XXH_errorcode +XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer, size_t secretSize, XXH_NOESCAPE const void* customSeed, size_t customSeedSize) +{ +#if (XXH_DEBUGLEVEL >= 1) + XXH_ASSERT(secretBuffer != NULL); + XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); +#else + /* production mode, assert() are disabled */ + if (secretBuffer == NULL) return XXH_ERROR; + if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR; +#endif + + if (customSeedSize == 0) { + customSeed = XXH3_kSecret; + customSeedSize = XXH_SECRET_DEFAULT_SIZE; + } +#if (XXH_DEBUGLEVEL >= 1) + XXH_ASSERT(customSeed != NULL); +#else + if (customSeed == NULL) return XXH_ERROR; +#endif + + /* Fill secretBuffer with a copy of customSeed - repeat as needed */ + { size_t pos = 0; + while (pos < secretSize) { + size_t const toCopy = XXH_MIN((secretSize - pos), customSeedSize); + memcpy((char*)secretBuffer + pos, customSeed, toCopy); + pos += toCopy; + } } + + { size_t const nbSeg16 = secretSize / 16; + size_t n; + XXH128_canonical_t scrambler; + XXH128_canonicalFromHash(&scrambler, XXH128(customSeed, customSeedSize, 0)); + for (n=0; n +#include +#include + +#if defined(__GNUC__) && __GNUC__ >= 4 +# define ZSTD_memcpy(d,s,l) __builtin_memcpy((d),(s),(l)) +# define ZSTD_memmove(d,s,l) __builtin_memmove((d),(s),(l)) +# define ZSTD_memset(p,v,l) __builtin_memset((p),(v),(l)) +#else +# define ZSTD_memcpy(d,s,l) memcpy((d),(s),(l)) +# define ZSTD_memmove(d,s,l) memmove((d),(s),(l)) +# define ZSTD_memset(p,v,l) memset((p),(v),(l)) +#endif + +#endif /* ZSTD_DEPS_COMMON */ + +/* Need: + * ZSTD_malloc() + * ZSTD_free() + * ZSTD_calloc() + */ +#ifdef ZSTD_DEPS_NEED_MALLOC +#ifndef ZSTD_DEPS_MALLOC +#define ZSTD_DEPS_MALLOC + +#include + +#define ZSTD_malloc(s) malloc(s) +#define ZSTD_calloc(n,s) calloc((n), (s)) +#define ZSTD_free(p) free((p)) + +#endif /* ZSTD_DEPS_MALLOC */ +#endif /* ZSTD_DEPS_NEED_MALLOC */ + +/* + * Provides 64-bit math support. + * Need: + * U64 ZSTD_div64(U64 dividend, U32 divisor) + */ +#ifdef ZSTD_DEPS_NEED_MATH64 +#ifndef ZSTD_DEPS_MATH64 +#define ZSTD_DEPS_MATH64 + +#define ZSTD_div64(dividend, divisor) ((dividend) / (divisor)) + +#endif /* ZSTD_DEPS_MATH64 */ +#endif /* ZSTD_DEPS_NEED_MATH64 */ + +/* Need: + * assert() + */ +#ifdef ZSTD_DEPS_NEED_ASSERT +#ifndef ZSTD_DEPS_ASSERT +#define ZSTD_DEPS_ASSERT + +#include + +#endif /* ZSTD_DEPS_ASSERT */ +#endif /* ZSTD_DEPS_NEED_ASSERT */ + +/* Need: + * ZSTD_DEBUG_PRINT() + */ +#ifdef ZSTD_DEPS_NEED_IO +#ifndef ZSTD_DEPS_IO +#define ZSTD_DEPS_IO + +#include +#define ZSTD_DEBUG_PRINT(...) fprintf(stderr, __VA_ARGS__) + +#endif /* ZSTD_DEPS_IO */ +#endif /* ZSTD_DEPS_NEED_IO */ + +/* Only requested when is known to be present. + * Need: + * intptr_t + */ +#ifdef ZSTD_DEPS_NEED_STDINT +#ifndef ZSTD_DEPS_STDINT +#define ZSTD_DEPS_STDINT + +#include + +#endif /* ZSTD_DEPS_STDINT */ +#endif /* ZSTD_DEPS_NEED_STDINT */ diff --git a/externals/zstd/lib/common/zstd_internal.h b/externals/zstd/lib/common/zstd_internal.h new file mode 100644 index 0000000..ecb9cfb --- /dev/null +++ b/externals/zstd/lib/common/zstd_internal.h @@ -0,0 +1,392 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_CCOMMON_H_MODULE +#define ZSTD_CCOMMON_H_MODULE + +/* this module contains definitions which must be identical + * across compression, decompression and dictBuilder. + * It also contains a few functions useful to at least 2 of them + * and which benefit from being inlined */ + +/*-************************************* +* Dependencies +***************************************/ +#include "compiler.h" +#include "cpu.h" +#include "mem.h" +#include "debug.h" /* assert, DEBUGLOG, RAWLOG, g_debuglevel */ +#include "error_private.h" +#define ZSTD_STATIC_LINKING_ONLY +#include "../zstd.h" +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" +#include "huf.h" +#ifndef XXH_STATIC_LINKING_ONLY +# define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ +#endif +#include "xxhash.h" /* XXH_reset, update, digest */ +#ifndef ZSTD_NO_TRACE +# include "zstd_trace.h" +#else +# define ZSTD_TRACE 0 +#endif + +#if defined (__cplusplus) +extern "C" { +#endif + +/* ---- static assert (debug) --- */ +#define ZSTD_STATIC_ASSERT(c) DEBUG_STATIC_ASSERT(c) +#define ZSTD_isError ERR_isError /* for inlining */ +#define FSE_isError ERR_isError +#define HUF_isError ERR_isError + + +/*-************************************* +* shared macros +***************************************/ +#undef MIN +#undef MAX +#define MIN(a,b) ((a)<(b) ? (a) : (b)) +#define MAX(a,b) ((a)>(b) ? (a) : (b)) +#define BOUNDED(min,val,max) (MAX(min,MIN(val,max))) + + +/*-************************************* +* Common constants +***************************************/ +#define ZSTD_OPT_NUM (1<<12) + +#define ZSTD_REP_NUM 3 /* number of repcodes */ +static UNUSED_ATTR const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 }; + +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +#define BIT7 128 +#define BIT6 64 +#define BIT5 32 +#define BIT4 16 +#define BIT1 2 +#define BIT0 1 + +#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10 +static UNUSED_ATTR const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 }; +static UNUSED_ATTR const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 }; + +#define ZSTD_FRAMEIDSIZE 4 /* magic number size */ + +#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */ +static UNUSED_ATTR const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE; +typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e; + +#define ZSTD_FRAMECHECKSUMSIZE 4 + +#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */ +#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */) /* for a non-null block */ +#define MIN_LITERALS_FOR_4_STREAMS 6 + +typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e; + +#define LONGNBSEQ 0x7F00 + +#define MINMATCH 3 + +#define Litbits 8 +#define LitHufLog 11 +#define MaxLit ((1<= WILDCOPY_VECLEN || diff <= -WILDCOPY_VECLEN); + /* Separate out the first COPY16() call because the copy length is + * almost certain to be short, so the branches have different + * probabilities. Since it is almost certain to be short, only do + * one COPY16() in the first call. Then, do two calls per loop since + * at that point it is more likely to have a high trip count. + */ + ZSTD_copy16(op, ip); + if (16 >= length) return; + op += 16; + ip += 16; + do { + COPY16(op, ip); + COPY16(op, ip); + } + while (op < oend); + } +} + +MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + size_t const length = MIN(dstCapacity, srcSize); + if (length > 0) { + ZSTD_memcpy(dst, src, length); + } + return length; +} + +/* define "workspace is too large" as this number of times larger than needed */ +#define ZSTD_WORKSPACETOOLARGE_FACTOR 3 + +/* when workspace is continuously too large + * during at least this number of times, + * context's memory usage is considered wasteful, + * because it's sized to handle a worst case scenario which rarely happens. + * In which case, resize it down to free some memory */ +#define ZSTD_WORKSPACETOOLARGE_MAXDURATION 128 + +/* Controls whether the input/output buffer is buffered or stable. */ +typedef enum { + ZSTD_bm_buffered = 0, /* Buffer the input/output */ + ZSTD_bm_stable = 1 /* ZSTD_inBuffer/ZSTD_outBuffer is stable */ +} ZSTD_bufferMode_e; + + +/*-******************************************* +* Private declarations +*********************************************/ +typedef struct seqDef_s { + U32 offBase; /* offBase == Offset + ZSTD_REP_NUM, or repcode 1,2,3 */ + U16 litLength; + U16 mlBase; /* mlBase == matchLength - MINMATCH */ +} seqDef; + +/* Controls whether seqStore has a single "long" litLength or matchLength. See seqStore_t. */ +typedef enum { + ZSTD_llt_none = 0, /* no longLengthType */ + ZSTD_llt_literalLength = 1, /* represents a long literal */ + ZSTD_llt_matchLength = 2 /* represents a long match */ +} ZSTD_longLengthType_e; + +typedef struct { + seqDef* sequencesStart; + seqDef* sequences; /* ptr to end of sequences */ + BYTE* litStart; + BYTE* lit; /* ptr to end of literals */ + BYTE* llCode; + BYTE* mlCode; + BYTE* ofCode; + size_t maxNbSeq; + size_t maxNbLit; + + /* longLengthPos and longLengthType to allow us to represent either a single litLength or matchLength + * in the seqStore that has a value larger than U16 (if it exists). To do so, we increment + * the existing value of the litLength or matchLength by 0x10000. + */ + ZSTD_longLengthType_e longLengthType; + U32 longLengthPos; /* Index of the sequence to apply long length modification to */ +} seqStore_t; + +typedef struct { + U32 litLength; + U32 matchLength; +} ZSTD_sequenceLength; + +/** + * Returns the ZSTD_sequenceLength for the given sequences. It handles the decoding of long sequences + * indicated by longLengthPos and longLengthType, and adds MINMATCH back to matchLength. + */ +MEM_STATIC ZSTD_sequenceLength ZSTD_getSequenceLength(seqStore_t const* seqStore, seqDef const* seq) +{ + ZSTD_sequenceLength seqLen; + seqLen.litLength = seq->litLength; + seqLen.matchLength = seq->mlBase + MINMATCH; + if (seqStore->longLengthPos == (U32)(seq - seqStore->sequencesStart)) { + if (seqStore->longLengthType == ZSTD_llt_literalLength) { + seqLen.litLength += 0x10000; + } + if (seqStore->longLengthType == ZSTD_llt_matchLength) { + seqLen.matchLength += 0x10000; + } + } + return seqLen; +} + +/** + * Contains the compressed frame size and an upper-bound for the decompressed frame size. + * Note: before using `compressedSize`, check for errors using ZSTD_isError(). + * similarly, before using `decompressedBound`, check for errors using: + * `decompressedBound != ZSTD_CONTENTSIZE_ERROR` + */ +typedef struct { + size_t nbBlocks; + size_t compressedSize; + unsigned long long decompressedBound; +} ZSTD_frameSizeInfo; /* decompress & legacy */ + +const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); /* compress & dictBuilder */ +int ZSTD_seqToCodes(const seqStore_t* seqStorePtr); /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */ + + +/* ZSTD_invalidateRepCodes() : + * ensures next compression will not use repcodes from previous block. + * Note : only works with regular variant; + * do not use with extDict variant ! */ +void ZSTD_invalidateRepCodes(ZSTD_CCtx* cctx); /* zstdmt, adaptive_compression (shouldn't get this definition from here) */ + + +typedef struct { + blockType_e blockType; + U32 lastBlock; + U32 origSize; +} blockProperties_t; /* declared here for decompress and fullbench */ + +/*! ZSTD_getcBlockSize() : + * Provides the size of compressed block from block header `src` */ +/* Used by: decompress, fullbench */ +size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, + blockProperties_t* bpPtr); + +/*! ZSTD_decodeSeqHeaders() : + * decode sequence header from src */ +/* Used by: zstd_decompress_block, fullbench */ +size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, + const void* src, size_t srcSize); + +/** + * @returns true iff the CPU supports dynamic BMI2 dispatch. + */ +MEM_STATIC int ZSTD_cpuSupportsBmi2(void) +{ + ZSTD_cpuid_t cpuid = ZSTD_cpuid(); + return ZSTD_cpuid_bmi1(cpuid) && ZSTD_cpuid_bmi2(cpuid); +} + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_CCOMMON_H_MODULE */ diff --git a/externals/zstd/lib/common/zstd_trace.h b/externals/zstd/lib/common/zstd_trace.h new file mode 100644 index 0000000..da20534 --- /dev/null +++ b/externals/zstd/lib/common/zstd_trace.h @@ -0,0 +1,163 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_TRACE_H +#define ZSTD_TRACE_H + +#if defined (__cplusplus) +extern "C" { +#endif + +#include + +/* weak symbol support + * For now, enable conservatively: + * - Only GNUC + * - Only ELF + * - Only x86-64, i386 and aarch64 + * Also, explicitly disable on platforms known not to work so they aren't + * forgotten in the future. + */ +#if !defined(ZSTD_HAVE_WEAK_SYMBOLS) && \ + defined(__GNUC__) && defined(__ELF__) && \ + (defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86) || defined(__aarch64__)) && \ + !defined(__APPLE__) && !defined(_WIN32) && !defined(__MINGW32__) && \ + !defined(__CYGWIN__) && !defined(_AIX) +# define ZSTD_HAVE_WEAK_SYMBOLS 1 +#else +# define ZSTD_HAVE_WEAK_SYMBOLS 0 +#endif +#if ZSTD_HAVE_WEAK_SYMBOLS +# define ZSTD_WEAK_ATTR __attribute__((__weak__)) +#else +# define ZSTD_WEAK_ATTR +#endif + +/* Only enable tracing when weak symbols are available. */ +#ifndef ZSTD_TRACE +# define ZSTD_TRACE ZSTD_HAVE_WEAK_SYMBOLS +#endif + +#if ZSTD_TRACE + +struct ZSTD_CCtx_s; +struct ZSTD_DCtx_s; +struct ZSTD_CCtx_params_s; + +typedef struct { + /** + * ZSTD_VERSION_NUMBER + * + * This is guaranteed to be the first member of ZSTD_trace. + * Otherwise, this struct is not stable between versions. If + * the version number does not match your expectation, you + * should not interpret the rest of the struct. + */ + unsigned version; + /** + * Non-zero if streaming (de)compression is used. + */ + unsigned streaming; + /** + * The dictionary ID. + */ + unsigned dictionaryID; + /** + * Is the dictionary cold? + * Only set on decompression. + */ + unsigned dictionaryIsCold; + /** + * The dictionary size or zero if no dictionary. + */ + size_t dictionarySize; + /** + * The uncompressed size of the data. + */ + size_t uncompressedSize; + /** + * The compressed size of the data. + */ + size_t compressedSize; + /** + * The fully resolved CCtx parameters (NULL on decompression). + */ + struct ZSTD_CCtx_params_s const* params; + /** + * The ZSTD_CCtx pointer (NULL on decompression). + */ + struct ZSTD_CCtx_s const* cctx; + /** + * The ZSTD_DCtx pointer (NULL on compression). + */ + struct ZSTD_DCtx_s const* dctx; +} ZSTD_Trace; + +/** + * A tracing context. It must be 0 when tracing is disabled. + * Otherwise, any non-zero value returned by a tracing begin() + * function is presented to any subsequent calls to end(). + * + * Any non-zero value is treated as tracing is enabled and not + * interpreted by the library. + * + * Two possible uses are: + * * A timestamp for when the begin() function was called. + * * A unique key identifying the (de)compression, like the + * address of the [dc]ctx pointer if you need to track + * more information than just a timestamp. + */ +typedef unsigned long long ZSTD_TraceCtx; + +/** + * Trace the beginning of a compression call. + * @param cctx The dctx pointer for the compression. + * It can be used as a key to map begin() to end(). + * @returns Non-zero if tracing is enabled. The return value is + * passed to ZSTD_trace_compress_end(). + */ +ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_compress_begin( + struct ZSTD_CCtx_s const* cctx); + +/** + * Trace the end of a compression call. + * @param ctx The return value of ZSTD_trace_compress_begin(). + * @param trace The zstd tracing info. + */ +ZSTD_WEAK_ATTR void ZSTD_trace_compress_end( + ZSTD_TraceCtx ctx, + ZSTD_Trace const* trace); + +/** + * Trace the beginning of a decompression call. + * @param dctx The dctx pointer for the decompression. + * It can be used as a key to map begin() to end(). + * @returns Non-zero if tracing is enabled. The return value is + * passed to ZSTD_trace_compress_end(). + */ +ZSTD_WEAK_ATTR ZSTD_TraceCtx ZSTD_trace_decompress_begin( + struct ZSTD_DCtx_s const* dctx); + +/** + * Trace the end of a decompression call. + * @param ctx The return value of ZSTD_trace_decompress_begin(). + * @param trace The zstd tracing info. + */ +ZSTD_WEAK_ATTR void ZSTD_trace_decompress_end( + ZSTD_TraceCtx ctx, + ZSTD_Trace const* trace); + +#endif /* ZSTD_TRACE */ + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_TRACE_H */ diff --git a/externals/zstd/lib/decompress/huf_decompress.c b/externals/zstd/lib/decompress/huf_decompress.c new file mode 100644 index 0000000..f85dd0b --- /dev/null +++ b/externals/zstd/lib/decompress/huf_decompress.c @@ -0,0 +1,1944 @@ +/* ****************************************************************** + * huff0 huffman decoder, + * part of Finite State Entropy library + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * You can contact the author at : + * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. +****************************************************************** */ + +/* ************************************************************** +* Dependencies +****************************************************************/ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memset */ +#include "../common/compiler.h" +#include "../common/bitstream.h" /* BIT_* */ +#include "../common/fse.h" /* to compress headers */ +#include "../common/huf.h" +#include "../common/error_private.h" +#include "../common/zstd_internal.h" +#include "../common/bits.h" /* ZSTD_highbit32, ZSTD_countTrailingZeros64 */ + +/* ************************************************************** +* Constants +****************************************************************/ + +#define HUF_DECODER_FAST_TABLELOG 11 + +/* ************************************************************** +* Macros +****************************************************************/ + +#ifdef HUF_DISABLE_FAST_DECODE +# define HUF_ENABLE_FAST_DECODE 0 +#else +# define HUF_ENABLE_FAST_DECODE 1 +#endif + +/* These two optional macros force the use one way or another of the two + * Huffman decompression implementations. You can't force in both directions + * at the same time. + */ +#if defined(HUF_FORCE_DECOMPRESS_X1) && \ + defined(HUF_FORCE_DECOMPRESS_X2) +#error "Cannot force the use of the X1 and X2 decoders at the same time!" +#endif + +/* When DYNAMIC_BMI2 is enabled, fast decoders are only called when bmi2 is + * supported at runtime, so we can add the BMI2 target attribute. + * When it is disabled, we will still get BMI2 if it is enabled statically. + */ +#if DYNAMIC_BMI2 +# define HUF_FAST_BMI2_ATTRS BMI2_TARGET_ATTRIBUTE +#else +# define HUF_FAST_BMI2_ATTRS +#endif + +#ifdef __cplusplus +# define HUF_EXTERN_C extern "C" +#else +# define HUF_EXTERN_C +#endif +#define HUF_ASM_DECL HUF_EXTERN_C + +#if DYNAMIC_BMI2 +# define HUF_NEED_BMI2_FUNCTION 1 +#else +# define HUF_NEED_BMI2_FUNCTION 0 +#endif + +/* ************************************************************** +* Error Management +****************************************************************/ +#define HUF_isError ERR_isError + + +/* ************************************************************** +* Byte alignment for workSpace management +****************************************************************/ +#define HUF_ALIGN(x, a) HUF_ALIGN_MASK((x), (a) - 1) +#define HUF_ALIGN_MASK(x, mask) (((x) + (mask)) & ~(mask)) + + +/* ************************************************************** +* BMI2 Variant Wrappers +****************************************************************/ +typedef size_t (*HUF_DecompressUsingDTableFn)(void *dst, size_t dstSize, + const void *cSrc, + size_t cSrcSize, + const HUF_DTable *DTable); + +#if DYNAMIC_BMI2 + +#define HUF_DGEN(fn) \ + \ + static size_t fn##_default( \ + void* dst, size_t dstSize, \ + const void* cSrc, size_t cSrcSize, \ + const HUF_DTable* DTable) \ + { \ + return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ + } \ + \ + static BMI2_TARGET_ATTRIBUTE size_t fn##_bmi2( \ + void* dst, size_t dstSize, \ + const void* cSrc, size_t cSrcSize, \ + const HUF_DTable* DTable) \ + { \ + return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ + } \ + \ + static size_t fn(void* dst, size_t dstSize, void const* cSrc, \ + size_t cSrcSize, HUF_DTable const* DTable, int flags) \ + { \ + if (flags & HUF_flags_bmi2) { \ + return fn##_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); \ + } \ + return fn##_default(dst, dstSize, cSrc, cSrcSize, DTable); \ + } + +#else + +#define HUF_DGEN(fn) \ + static size_t fn(void* dst, size_t dstSize, void const* cSrc, \ + size_t cSrcSize, HUF_DTable const* DTable, int flags) \ + { \ + (void)flags; \ + return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ + } + +#endif + + +/*-***************************/ +/* generic DTableDesc */ +/*-***************************/ +typedef struct { BYTE maxTableLog; BYTE tableType; BYTE tableLog; BYTE reserved; } DTableDesc; + +static DTableDesc HUF_getDTableDesc(const HUF_DTable* table) +{ + DTableDesc dtd; + ZSTD_memcpy(&dtd, table, sizeof(dtd)); + return dtd; +} + +static size_t HUF_initFastDStream(BYTE const* ip) { + BYTE const lastByte = ip[7]; + size_t const bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0; + size_t const value = MEM_readLEST(ip) | 1; + assert(bitsConsumed <= 8); + assert(sizeof(size_t) == 8); + return value << bitsConsumed; +} + + +/** + * The input/output arguments to the Huffman fast decoding loop: + * + * ip [in/out] - The input pointers, must be updated to reflect what is consumed. + * op [in/out] - The output pointers, must be updated to reflect what is written. + * bits [in/out] - The bitstream containers, must be updated to reflect the current state. + * dt [in] - The decoding table. + * ilowest [in] - The beginning of the valid range of the input. Decoders may read + * down to this pointer. It may be below iend[0]. + * oend [in] - The end of the output stream. op[3] must not cross oend. + * iend [in] - The end of each input stream. ip[i] may cross iend[i], + * as long as it is above ilowest, but that indicates corruption. + */ +typedef struct { + BYTE const* ip[4]; + BYTE* op[4]; + U64 bits[4]; + void const* dt; + BYTE const* ilowest; + BYTE* oend; + BYTE const* iend[4]; +} HUF_DecompressFastArgs; + +typedef void (*HUF_DecompressFastLoopFn)(HUF_DecompressFastArgs*); + +/** + * Initializes args for the fast decoding loop. + * @returns 1 on success + * 0 if the fallback implementation should be used. + * Or an error code on failure. + */ +static size_t HUF_DecompressFastArgs_init(HUF_DecompressFastArgs* args, void* dst, size_t dstSize, void const* src, size_t srcSize, const HUF_DTable* DTable) +{ + void const* dt = DTable + 1; + U32 const dtLog = HUF_getDTableDesc(DTable).tableLog; + + const BYTE* const istart = (const BYTE*)src; + + BYTE* const oend = ZSTD_maybeNullPtrAdd((BYTE*)dst, dstSize); + + /* The fast decoding loop assumes 64-bit little-endian. + * This condition is false on x32. + */ + if (!MEM_isLittleEndian() || MEM_32bits()) + return 0; + + /* Avoid nullptr addition */ + if (dstSize == 0) + return 0; + assert(dst != NULL); + + /* strict minimum : jump table + 1 byte per stream */ + if (srcSize < 10) + return ERROR(corruption_detected); + + /* Must have at least 8 bytes per stream because we don't handle initializing smaller bit containers. + * If table log is not correct at this point, fallback to the old decoder. + * On small inputs we don't have enough data to trigger the fast loop, so use the old decoder. + */ + if (dtLog != HUF_DECODER_FAST_TABLELOG) + return 0; + + /* Read the jump table. */ + { + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = srcSize - (length1 + length2 + length3 + 6); + args->iend[0] = istart + 6; /* jumpTable */ + args->iend[1] = args->iend[0] + length1; + args->iend[2] = args->iend[1] + length2; + args->iend[3] = args->iend[2] + length3; + + /* HUF_initFastDStream() requires this, and this small of an input + * won't benefit from the ASM loop anyways. + */ + if (length1 < 8 || length2 < 8 || length3 < 8 || length4 < 8) + return 0; + if (length4 > srcSize) return ERROR(corruption_detected); /* overflow */ + } + /* ip[] contains the position that is currently loaded into bits[]. */ + args->ip[0] = args->iend[1] - sizeof(U64); + args->ip[1] = args->iend[2] - sizeof(U64); + args->ip[2] = args->iend[3] - sizeof(U64); + args->ip[3] = (BYTE const*)src + srcSize - sizeof(U64); + + /* op[] contains the output pointers. */ + args->op[0] = (BYTE*)dst; + args->op[1] = args->op[0] + (dstSize+3)/4; + args->op[2] = args->op[1] + (dstSize+3)/4; + args->op[3] = args->op[2] + (dstSize+3)/4; + + /* No point to call the ASM loop for tiny outputs. */ + if (args->op[3] >= oend) + return 0; + + /* bits[] is the bit container. + * It is read from the MSB down to the LSB. + * It is shifted left as it is read, and zeros are + * shifted in. After the lowest valid bit a 1 is + * set, so that CountTrailingZeros(bits[]) can be used + * to count how many bits we've consumed. + */ + args->bits[0] = HUF_initFastDStream(args->ip[0]); + args->bits[1] = HUF_initFastDStream(args->ip[1]); + args->bits[2] = HUF_initFastDStream(args->ip[2]); + args->bits[3] = HUF_initFastDStream(args->ip[3]); + + /* The decoders must be sure to never read beyond ilowest. + * This is lower than iend[0], but allowing decoders to read + * down to ilowest can allow an extra iteration or two in the + * fast loop. + */ + args->ilowest = istart; + + args->oend = oend; + args->dt = dt; + + return 1; +} + +static size_t HUF_initRemainingDStream(BIT_DStream_t* bit, HUF_DecompressFastArgs const* args, int stream, BYTE* segmentEnd) +{ + /* Validate that we haven't overwritten. */ + if (args->op[stream] > segmentEnd) + return ERROR(corruption_detected); + /* Validate that we haven't read beyond iend[]. + * Note that ip[] may be < iend[] because the MSB is + * the next bit to read, and we may have consumed 100% + * of the stream, so down to iend[i] - 8 is valid. + */ + if (args->ip[stream] < args->iend[stream] - 8) + return ERROR(corruption_detected); + + /* Construct the BIT_DStream_t. */ + assert(sizeof(size_t) == 8); + bit->bitContainer = MEM_readLEST(args->ip[stream]); + bit->bitsConsumed = ZSTD_countTrailingZeros64(args->bits[stream]); + bit->start = (const char*)args->ilowest; + bit->limitPtr = bit->start + sizeof(size_t); + bit->ptr = (const char*)args->ip[stream]; + + return 0; +} + +/* Calls X(N) for each stream 0, 1, 2, 3. */ +#define HUF_4X_FOR_EACH_STREAM(X) \ + do { \ + X(0); \ + X(1); \ + X(2); \ + X(3); \ + } while (0) + +/* Calls X(N, var) for each stream 0, 1, 2, 3. */ +#define HUF_4X_FOR_EACH_STREAM_WITH_VAR(X, var) \ + do { \ + X(0, (var)); \ + X(1, (var)); \ + X(2, (var)); \ + X(3, (var)); \ + } while (0) + + +#ifndef HUF_FORCE_DECOMPRESS_X2 + +/*-***************************/ +/* single-symbol decoding */ +/*-***************************/ +typedef struct { BYTE nbBits; BYTE byte; } HUF_DEltX1; /* single-symbol decoding */ + +/** + * Packs 4 HUF_DEltX1 structs into a U64. This is used to lay down 4 entries at + * a time. + */ +static U64 HUF_DEltX1_set4(BYTE symbol, BYTE nbBits) { + U64 D4; + if (MEM_isLittleEndian()) { + D4 = (U64)((symbol << 8) + nbBits); + } else { + D4 = (U64)(symbol + (nbBits << 8)); + } + assert(D4 < (1U << 16)); + D4 *= 0x0001000100010001ULL; + return D4; +} + +/** + * Increase the tableLog to targetTableLog and rescales the stats. + * If tableLog > targetTableLog this is a no-op. + * @returns New tableLog + */ +static U32 HUF_rescaleStats(BYTE* huffWeight, U32* rankVal, U32 nbSymbols, U32 tableLog, U32 targetTableLog) +{ + if (tableLog > targetTableLog) + return tableLog; + if (tableLog < targetTableLog) { + U32 const scale = targetTableLog - tableLog; + U32 s; + /* Increase the weight for all non-zero probability symbols by scale. */ + for (s = 0; s < nbSymbols; ++s) { + huffWeight[s] += (BYTE)((huffWeight[s] == 0) ? 0 : scale); + } + /* Update rankVal to reflect the new weights. + * All weights except 0 get moved to weight + scale. + * Weights [1, scale] are empty. + */ + for (s = targetTableLog; s > scale; --s) { + rankVal[s] = rankVal[s - scale]; + } + for (s = scale; s > 0; --s) { + rankVal[s] = 0; + } + } + return targetTableLog; +} + +typedef struct { + U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; + U32 rankStart[HUF_TABLELOG_ABSOLUTEMAX + 1]; + U32 statsWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; + BYTE symbols[HUF_SYMBOLVALUE_MAX + 1]; + BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; +} HUF_ReadDTableX1_Workspace; + +size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags) +{ + U32 tableLog = 0; + U32 nbSymbols = 0; + size_t iSize; + void* const dtPtr = DTable + 1; + HUF_DEltX1* const dt = (HUF_DEltX1*)dtPtr; + HUF_ReadDTableX1_Workspace* wksp = (HUF_ReadDTableX1_Workspace*)workSpace; + + DEBUG_STATIC_ASSERT(HUF_DECOMPRESS_WORKSPACE_SIZE >= sizeof(*wksp)); + if (sizeof(*wksp) > wkspSize) return ERROR(tableLog_tooLarge); + + DEBUG_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable)); + /* ZSTD_memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */ + + iSize = HUF_readStats_wksp(wksp->huffWeight, HUF_SYMBOLVALUE_MAX + 1, wksp->rankVal, &nbSymbols, &tableLog, src, srcSize, wksp->statsWksp, sizeof(wksp->statsWksp), flags); + if (HUF_isError(iSize)) return iSize; + + + /* Table header */ + { DTableDesc dtd = HUF_getDTableDesc(DTable); + U32 const maxTableLog = dtd.maxTableLog + 1; + U32 const targetTableLog = MIN(maxTableLog, HUF_DECODER_FAST_TABLELOG); + tableLog = HUF_rescaleStats(wksp->huffWeight, wksp->rankVal, nbSymbols, tableLog, targetTableLog); + if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge); /* DTable too small, Huffman tree cannot fit in */ + dtd.tableType = 0; + dtd.tableLog = (BYTE)tableLog; + ZSTD_memcpy(DTable, &dtd, sizeof(dtd)); + } + + /* Compute symbols and rankStart given rankVal: + * + * rankVal already contains the number of values of each weight. + * + * symbols contains the symbols ordered by weight. First are the rankVal[0] + * weight 0 symbols, followed by the rankVal[1] weight 1 symbols, and so on. + * symbols[0] is filled (but unused) to avoid a branch. + * + * rankStart contains the offset where each rank belongs in the DTable. + * rankStart[0] is not filled because there are no entries in the table for + * weight 0. + */ + { int n; + U32 nextRankStart = 0; + int const unroll = 4; + int const nLimit = (int)nbSymbols - unroll + 1; + for (n=0; n<(int)tableLog+1; n++) { + U32 const curr = nextRankStart; + nextRankStart += wksp->rankVal[n]; + wksp->rankStart[n] = curr; + } + for (n=0; n < nLimit; n += unroll) { + int u; + for (u=0; u < unroll; ++u) { + size_t const w = wksp->huffWeight[n+u]; + wksp->symbols[wksp->rankStart[w]++] = (BYTE)(n+u); + } + } + for (; n < (int)nbSymbols; ++n) { + size_t const w = wksp->huffWeight[n]; + wksp->symbols[wksp->rankStart[w]++] = (BYTE)n; + } + } + + /* fill DTable + * We fill all entries of each weight in order. + * That way length is a constant for each iteration of the outer loop. + * We can switch based on the length to a different inner loop which is + * optimized for that particular case. + */ + { U32 w; + int symbol = wksp->rankVal[0]; + int rankStart = 0; + for (w=1; wrankVal[w]; + int const length = (1 << w) >> 1; + int uStart = rankStart; + BYTE const nbBits = (BYTE)(tableLog + 1 - w); + int s; + int u; + switch (length) { + case 1: + for (s=0; ssymbols[symbol + s]; + D.nbBits = nbBits; + dt[uStart] = D; + uStart += 1; + } + break; + case 2: + for (s=0; ssymbols[symbol + s]; + D.nbBits = nbBits; + dt[uStart+0] = D; + dt[uStart+1] = D; + uStart += 2; + } + break; + case 4: + for (s=0; ssymbols[symbol + s], nbBits); + MEM_write64(dt + uStart, D4); + uStart += 4; + } + break; + case 8: + for (s=0; ssymbols[symbol + s], nbBits); + MEM_write64(dt + uStart, D4); + MEM_write64(dt + uStart + 4, D4); + uStart += 8; + } + break; + default: + for (s=0; ssymbols[symbol + s], nbBits); + for (u=0; u < length; u += 16) { + MEM_write64(dt + uStart + u + 0, D4); + MEM_write64(dt + uStart + u + 4, D4); + MEM_write64(dt + uStart + u + 8, D4); + MEM_write64(dt + uStart + u + 12, D4); + } + assert(u == length); + uStart += length; + } + break; + } + symbol += symbolCount; + rankStart += symbolCount * length; + } + } + return iSize; +} + +FORCE_INLINE_TEMPLATE BYTE +HUF_decodeSymbolX1(BIT_DStream_t* Dstream, const HUF_DEltX1* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */ + BYTE const c = dt[val].byte; + BIT_skipBits(Dstream, dt[val].nbBits); + return c; +} + +#define HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr) \ + do { *ptr++ = HUF_decodeSymbolX1(DStreamPtr, dt, dtLog); } while (0) + +#define HUF_DECODE_SYMBOLX1_1(ptr, DStreamPtr) \ + do { \ + if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ + HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr); \ + } while (0) + +#define HUF_DECODE_SYMBOLX1_2(ptr, DStreamPtr) \ + do { \ + if (MEM_64bits()) \ + HUF_DECODE_SYMBOLX1_0(ptr, DStreamPtr); \ + } while (0) + +HINT_INLINE size_t +HUF_decodeStreamX1(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX1* const dt, const U32 dtLog) +{ + BYTE* const pStart = p; + + /* up to 4 symbols at a time */ + if ((pEnd - p) > 3) { + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-3)) { + HUF_DECODE_SYMBOLX1_2(p, bitDPtr); + HUF_DECODE_SYMBOLX1_1(p, bitDPtr); + HUF_DECODE_SYMBOLX1_2(p, bitDPtr); + HUF_DECODE_SYMBOLX1_0(p, bitDPtr); + } + } else { + BIT_reloadDStream(bitDPtr); + } + + /* [0-3] symbols remaining */ + if (MEM_32bits()) + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd)) + HUF_DECODE_SYMBOLX1_0(p, bitDPtr); + + /* no more data to retrieve from bitstream, no need to reload */ + while (p < pEnd) + HUF_DECODE_SYMBOLX1_0(p, bitDPtr); + + return (size_t)(pEnd-pStart); +} + +FORCE_INLINE_TEMPLATE size_t +HUF_decompress1X1_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + BYTE* op = (BYTE*)dst; + BYTE* const oend = ZSTD_maybeNullPtrAdd(op, dstSize); + const void* dtPtr = DTable + 1; + const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr; + BIT_DStream_t bitD; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + + CHECK_F( BIT_initDStream(&bitD, cSrc, cSrcSize) ); + + HUF_decodeStreamX1(op, &bitD, oend, dt, dtLog); + + if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); + + return dstSize; +} + +/* HUF_decompress4X1_usingDTable_internal_body(): + * Conditions : + * @dstSize >= 6 + */ +FORCE_INLINE_TEMPLATE size_t +HUF_decompress4X1_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + /* Check */ + if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + if (dstSize < 6) return ERROR(corruption_detected); /* stream 4-split doesn't work */ + + { const BYTE* const istart = (const BYTE*) cSrc; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + BYTE* const olimit = oend - 3; + const void* const dtPtr = DTable + 1; + const HUF_DEltX1* const dt = (const HUF_DEltX1*)dtPtr; + + /* Init */ + BIT_DStream_t bitD1; + BIT_DStream_t bitD2; + BIT_DStream_t bitD3; + BIT_DStream_t bitD4; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); + const BYTE* const istart1 = istart + 6; /* jumpTable */ + const BYTE* const istart2 = istart1 + length1; + const BYTE* const istart3 = istart2 + length2; + const BYTE* const istart4 = istart3 + length3; + const size_t segmentSize = (dstSize+3) / 4; + BYTE* const opStart2 = ostart + segmentSize; + BYTE* const opStart3 = opStart2 + segmentSize; + BYTE* const opStart4 = opStart3 + segmentSize; + BYTE* op1 = ostart; + BYTE* op2 = opStart2; + BYTE* op3 = opStart3; + BYTE* op4 = opStart4; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + U32 endSignal = 1; + + if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + if (opStart4 > oend) return ERROR(corruption_detected); /* overflow */ + assert(dstSize >= 6); /* validated above */ + CHECK_F( BIT_initDStream(&bitD1, istart1, length1) ); + CHECK_F( BIT_initDStream(&bitD2, istart2, length2) ); + CHECK_F( BIT_initDStream(&bitD3, istart3, length3) ); + CHECK_F( BIT_initDStream(&bitD4, istart4, length4) ); + + /* up to 16 symbols per loop (4 symbols per stream) in 64-bit mode */ + if ((size_t)(oend - op4) >= sizeof(size_t)) { + for ( ; (endSignal) & (op4 < olimit) ; ) { + HUF_DECODE_SYMBOLX1_2(op1, &bitD1); + HUF_DECODE_SYMBOLX1_2(op2, &bitD2); + HUF_DECODE_SYMBOLX1_2(op3, &bitD3); + HUF_DECODE_SYMBOLX1_2(op4, &bitD4); + HUF_DECODE_SYMBOLX1_1(op1, &bitD1); + HUF_DECODE_SYMBOLX1_1(op2, &bitD2); + HUF_DECODE_SYMBOLX1_1(op3, &bitD3); + HUF_DECODE_SYMBOLX1_1(op4, &bitD4); + HUF_DECODE_SYMBOLX1_2(op1, &bitD1); + HUF_DECODE_SYMBOLX1_2(op2, &bitD2); + HUF_DECODE_SYMBOLX1_2(op3, &bitD3); + HUF_DECODE_SYMBOLX1_2(op4, &bitD4); + HUF_DECODE_SYMBOLX1_0(op1, &bitD1); + HUF_DECODE_SYMBOLX1_0(op2, &bitD2); + HUF_DECODE_SYMBOLX1_0(op3, &bitD3); + HUF_DECODE_SYMBOLX1_0(op4, &bitD4); + endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished; + } + } + + /* check corruption */ + /* note : should not be necessary : op# advance in lock step, and we control op4. + * but curiously, binary generated by gcc 7.2 & 7.3 with -mbmi2 runs faster when >=1 test is present */ + if (op1 > opStart2) return ERROR(corruption_detected); + if (op2 > opStart3) return ERROR(corruption_detected); + if (op3 > opStart4) return ERROR(corruption_detected); + /* note : op4 supposed already verified within main loop */ + + /* finish bitStreams one by one */ + HUF_decodeStreamX1(op1, &bitD1, opStart2, dt, dtLog); + HUF_decodeStreamX1(op2, &bitD2, opStart3, dt, dtLog); + HUF_decodeStreamX1(op3, &bitD3, opStart4, dt, dtLog); + HUF_decodeStreamX1(op4, &bitD4, oend, dt, dtLog); + + /* check */ + { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); + if (!endCheck) return ERROR(corruption_detected); } + + /* decoded size */ + return dstSize; + } +} + +#if HUF_NEED_BMI2_FUNCTION +static BMI2_TARGET_ATTRIBUTE +size_t HUF_decompress4X1_usingDTable_internal_bmi2(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X1_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} +#endif + +static +size_t HUF_decompress4X1_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X1_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 + +HUF_ASM_DECL void HUF_decompress4X1_usingDTable_internal_fast_asm_loop(HUF_DecompressFastArgs* args) ZSTDLIB_HIDDEN; + +#endif + +static HUF_FAST_BMI2_ATTRS +void HUF_decompress4X1_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs* args) +{ + U64 bits[4]; + BYTE const* ip[4]; + BYTE* op[4]; + U16 const* const dtable = (U16 const*)args->dt; + BYTE* const oend = args->oend; + BYTE const* const ilowest = args->ilowest; + + /* Copy the arguments to local variables */ + ZSTD_memcpy(&bits, &args->bits, sizeof(bits)); + ZSTD_memcpy((void*)(&ip), &args->ip, sizeof(ip)); + ZSTD_memcpy(&op, &args->op, sizeof(op)); + + assert(MEM_isLittleEndian()); + assert(!MEM_32bits()); + + for (;;) { + BYTE* olimit; + int stream; + + /* Assert loop preconditions */ +#ifndef NDEBUG + for (stream = 0; stream < 4; ++stream) { + assert(op[stream] <= (stream == 3 ? oend : op[stream + 1])); + assert(ip[stream] >= ilowest); + } +#endif + /* Compute olimit */ + { + /* Each iteration produces 5 output symbols per stream */ + size_t const oiters = (size_t)(oend - op[3]) / 5; + /* Each iteration consumes up to 11 bits * 5 = 55 bits < 7 bytes + * per stream. + */ + size_t const iiters = (size_t)(ip[0] - ilowest) / 7; + /* We can safely run iters iterations before running bounds checks */ + size_t const iters = MIN(oiters, iiters); + size_t const symbols = iters * 5; + + /* We can simply check that op[3] < olimit, instead of checking all + * of our bounds, since we can't hit the other bounds until we've run + * iters iterations, which only happens when op[3] == olimit. + */ + olimit = op[3] + symbols; + + /* Exit fast decoding loop once we reach the end. */ + if (op[3] == olimit) + break; + + /* Exit the decoding loop if any input pointer has crossed the + * previous one. This indicates corruption, and a precondition + * to our loop is that ip[i] >= ip[0]. + */ + for (stream = 1; stream < 4; ++stream) { + if (ip[stream] < ip[stream - 1]) + goto _out; + } + } + +#ifndef NDEBUG + for (stream = 1; stream < 4; ++stream) { + assert(ip[stream] >= ip[stream - 1]); + } +#endif + +#define HUF_4X1_DECODE_SYMBOL(_stream, _symbol) \ + do { \ + int const index = (int)(bits[(_stream)] >> 53); \ + int const entry = (int)dtable[index]; \ + bits[(_stream)] <<= (entry & 0x3F); \ + op[(_stream)][(_symbol)] = (BYTE)((entry >> 8) & 0xFF); \ + } while (0) + +#define HUF_4X1_RELOAD_STREAM(_stream) \ + do { \ + int const ctz = ZSTD_countTrailingZeros64(bits[(_stream)]); \ + int const nbBits = ctz & 7; \ + int const nbBytes = ctz >> 3; \ + op[(_stream)] += 5; \ + ip[(_stream)] -= nbBytes; \ + bits[(_stream)] = MEM_read64(ip[(_stream)]) | 1; \ + bits[(_stream)] <<= nbBits; \ + } while (0) + + /* Manually unroll the loop because compilers don't consistently + * unroll the inner loops, which destroys performance. + */ + do { + /* Decode 5 symbols in each of the 4 streams */ + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 0); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 1); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 2); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 3); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X1_DECODE_SYMBOL, 4); + + /* Reload each of the 4 the bitstreams */ + HUF_4X_FOR_EACH_STREAM(HUF_4X1_RELOAD_STREAM); + } while (op[3] < olimit); + +#undef HUF_4X1_DECODE_SYMBOL +#undef HUF_4X1_RELOAD_STREAM + } + +_out: + + /* Save the final values of each of the state variables back to args. */ + ZSTD_memcpy(&args->bits, &bits, sizeof(bits)); + ZSTD_memcpy((void*)(&args->ip), &ip, sizeof(ip)); + ZSTD_memcpy(&args->op, &op, sizeof(op)); +} + +/** + * @returns @p dstSize on success (>= 6) + * 0 if the fallback implementation should be used + * An error if an error occurred + */ +static HUF_FAST_BMI2_ATTRS +size_t +HUF_decompress4X1_usingDTable_internal_fast( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable, + HUF_DecompressFastLoopFn loopFn) +{ + void const* dt = DTable + 1; + BYTE const* const ilowest = (BYTE const*)cSrc; + BYTE* const oend = ZSTD_maybeNullPtrAdd((BYTE*)dst, dstSize); + HUF_DecompressFastArgs args; + { size_t const ret = HUF_DecompressFastArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable); + FORWARD_IF_ERROR(ret, "Failed to init fast loop args"); + if (ret == 0) + return 0; + } + + assert(args.ip[0] >= args.ilowest); + loopFn(&args); + + /* Our loop guarantees that ip[] >= ilowest and that we haven't + * overwritten any op[]. + */ + assert(args.ip[0] >= ilowest); + assert(args.ip[0] >= ilowest); + assert(args.ip[1] >= ilowest); + assert(args.ip[2] >= ilowest); + assert(args.ip[3] >= ilowest); + assert(args.op[3] <= oend); + + assert(ilowest == args.ilowest); + assert(ilowest + 6 == args.iend[0]); + (void)ilowest; + + /* finish bit streams one by one. */ + { size_t const segmentSize = (dstSize+3) / 4; + BYTE* segmentEnd = (BYTE*)dst; + int i; + for (i = 0; i < 4; ++i) { + BIT_DStream_t bit; + if (segmentSize <= (size_t)(oend - segmentEnd)) + segmentEnd += segmentSize; + else + segmentEnd = oend; + FORWARD_IF_ERROR(HUF_initRemainingDStream(&bit, &args, i, segmentEnd), "corruption"); + /* Decompress and validate that we've produced exactly the expected length. */ + args.op[i] += HUF_decodeStreamX1(args.op[i], &bit, segmentEnd, (HUF_DEltX1 const*)dt, HUF_DECODER_FAST_TABLELOG); + if (args.op[i] != segmentEnd) return ERROR(corruption_detected); + } + } + + /* decoded size */ + assert(dstSize != 0); + return dstSize; +} + +HUF_DGEN(HUF_decompress1X1_usingDTable_internal) + +static size_t HUF_decompress4X1_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable, int flags) +{ + HUF_DecompressUsingDTableFn fallbackFn = HUF_decompress4X1_usingDTable_internal_default; + HUF_DecompressFastLoopFn loopFn = HUF_decompress4X1_usingDTable_internal_fast_c_loop; + +#if DYNAMIC_BMI2 + if (flags & HUF_flags_bmi2) { + fallbackFn = HUF_decompress4X1_usingDTable_internal_bmi2; +# if ZSTD_ENABLE_ASM_X86_64_BMI2 + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X1_usingDTable_internal_fast_asm_loop; + } +# endif + } else { + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); + } +#endif + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__) + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X1_usingDTable_internal_fast_asm_loop; + } +#endif + + if (HUF_ENABLE_FAST_DECODE && !(flags & HUF_flags_disableFast)) { + size_t const ret = HUF_decompress4X1_usingDTable_internal_fast(dst, dstSize, cSrc, cSrcSize, DTable, loopFn); + if (ret != 0) + return ret; + } + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); +} + +static size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize, int flags) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX1_wksp(dctx, cSrc, cSrcSize, workSpace, wkspSize, flags); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress4X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags); +} + +#endif /* HUF_FORCE_DECOMPRESS_X2 */ + + +#ifndef HUF_FORCE_DECOMPRESS_X1 + +/* *************************/ +/* double-symbols decoding */ +/* *************************/ + +typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX2; /* double-symbols decoding */ +typedef struct { BYTE symbol; } sortedSymbol_t; +typedef U32 rankValCol_t[HUF_TABLELOG_MAX + 1]; +typedef rankValCol_t rankVal_t[HUF_TABLELOG_MAX]; + +/** + * Constructs a HUF_DEltX2 in a U32. + */ +static U32 HUF_buildDEltX2U32(U32 symbol, U32 nbBits, U32 baseSeq, int level) +{ + U32 seq; + DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, sequence) == 0); + DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, nbBits) == 2); + DEBUG_STATIC_ASSERT(offsetof(HUF_DEltX2, length) == 3); + DEBUG_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(U32)); + if (MEM_isLittleEndian()) { + seq = level == 1 ? symbol : (baseSeq + (symbol << 8)); + return seq + (nbBits << 16) + ((U32)level << 24); + } else { + seq = level == 1 ? (symbol << 8) : ((baseSeq << 8) + symbol); + return (seq << 16) + (nbBits << 8) + (U32)level; + } +} + +/** + * Constructs a HUF_DEltX2. + */ +static HUF_DEltX2 HUF_buildDEltX2(U32 symbol, U32 nbBits, U32 baseSeq, int level) +{ + HUF_DEltX2 DElt; + U32 const val = HUF_buildDEltX2U32(symbol, nbBits, baseSeq, level); + DEBUG_STATIC_ASSERT(sizeof(DElt) == sizeof(val)); + ZSTD_memcpy(&DElt, &val, sizeof(val)); + return DElt; +} + +/** + * Constructs 2 HUF_DEltX2s and packs them into a U64. + */ +static U64 HUF_buildDEltX2U64(U32 symbol, U32 nbBits, U16 baseSeq, int level) +{ + U32 DElt = HUF_buildDEltX2U32(symbol, nbBits, baseSeq, level); + return (U64)DElt + ((U64)DElt << 32); +} + +/** + * Fills the DTable rank with all the symbols from [begin, end) that are each + * nbBits long. + * + * @param DTableRank The start of the rank in the DTable. + * @param begin The first symbol to fill (inclusive). + * @param end The last symbol to fill (exclusive). + * @param nbBits Each symbol is nbBits long. + * @param tableLog The table log. + * @param baseSeq If level == 1 { 0 } else { the first level symbol } + * @param level The level in the table. Must be 1 or 2. + */ +static void HUF_fillDTableX2ForWeight( + HUF_DEltX2* DTableRank, + sortedSymbol_t const* begin, sortedSymbol_t const* end, + U32 nbBits, U32 tableLog, + U16 baseSeq, int const level) +{ + U32 const length = 1U << ((tableLog - nbBits) & 0x1F /* quiet static-analyzer */); + const sortedSymbol_t* ptr; + assert(level >= 1 && level <= 2); + switch (length) { + case 1: + for (ptr = begin; ptr != end; ++ptr) { + HUF_DEltX2 const DElt = HUF_buildDEltX2(ptr->symbol, nbBits, baseSeq, level); + *DTableRank++ = DElt; + } + break; + case 2: + for (ptr = begin; ptr != end; ++ptr) { + HUF_DEltX2 const DElt = HUF_buildDEltX2(ptr->symbol, nbBits, baseSeq, level); + DTableRank[0] = DElt; + DTableRank[1] = DElt; + DTableRank += 2; + } + break; + case 4: + for (ptr = begin; ptr != end; ++ptr) { + U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level); + ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2)); + DTableRank += 4; + } + break; + case 8: + for (ptr = begin; ptr != end; ++ptr) { + U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level); + ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 4, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 6, &DEltX2, sizeof(DEltX2)); + DTableRank += 8; + } + break; + default: + for (ptr = begin; ptr != end; ++ptr) { + U64 const DEltX2 = HUF_buildDEltX2U64(ptr->symbol, nbBits, baseSeq, level); + HUF_DEltX2* const DTableRankEnd = DTableRank + length; + for (; DTableRank != DTableRankEnd; DTableRank += 8) { + ZSTD_memcpy(DTableRank + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 2, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 4, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTableRank + 6, &DEltX2, sizeof(DEltX2)); + } + } + break; + } +} + +/* HUF_fillDTableX2Level2() : + * `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */ +static void HUF_fillDTableX2Level2(HUF_DEltX2* DTable, U32 targetLog, const U32 consumedBits, + const U32* rankVal, const int minWeight, const int maxWeight1, + const sortedSymbol_t* sortedSymbols, U32 const* rankStart, + U32 nbBitsBaseline, U16 baseSeq) +{ + /* Fill skipped values (all positions up to rankVal[minWeight]). + * These are positions only get a single symbol because the combined weight + * is too large. + */ + if (minWeight>1) { + U32 const length = 1U << ((targetLog - consumedBits) & 0x1F /* quiet static-analyzer */); + U64 const DEltX2 = HUF_buildDEltX2U64(baseSeq, consumedBits, /* baseSeq */ 0, /* level */ 1); + int const skipSize = rankVal[minWeight]; + assert(length > 1); + assert((U32)skipSize < length); + switch (length) { + case 2: + assert(skipSize == 1); + ZSTD_memcpy(DTable, &DEltX2, sizeof(DEltX2)); + break; + case 4: + assert(skipSize <= 4); + ZSTD_memcpy(DTable + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + 2, &DEltX2, sizeof(DEltX2)); + break; + default: + { + int i; + for (i = 0; i < skipSize; i += 8) { + ZSTD_memcpy(DTable + i + 0, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + i + 2, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + i + 4, &DEltX2, sizeof(DEltX2)); + ZSTD_memcpy(DTable + i + 6, &DEltX2, sizeof(DEltX2)); + } + } + } + } + + /* Fill each of the second level symbols by weight. */ + { + int w; + for (w = minWeight; w < maxWeight1; ++w) { + int const begin = rankStart[w]; + int const end = rankStart[w+1]; + U32 const nbBits = nbBitsBaseline - w; + U32 const totalBits = nbBits + consumedBits; + HUF_fillDTableX2ForWeight( + DTable + rankVal[w], + sortedSymbols + begin, sortedSymbols + end, + totalBits, targetLog, + baseSeq, /* level */ 2); + } + } +} + +static void HUF_fillDTableX2(HUF_DEltX2* DTable, const U32 targetLog, + const sortedSymbol_t* sortedList, + const U32* rankStart, rankValCol_t* rankValOrigin, const U32 maxWeight, + const U32 nbBitsBaseline) +{ + U32* const rankVal = rankValOrigin[0]; + const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */ + const U32 minBits = nbBitsBaseline - maxWeight; + int w; + int const wEnd = (int)maxWeight + 1; + + /* Fill DTable in order of weight. */ + for (w = 1; w < wEnd; ++w) { + int const begin = (int)rankStart[w]; + int const end = (int)rankStart[w+1]; + U32 const nbBits = nbBitsBaseline - w; + + if (targetLog-nbBits >= minBits) { + /* Enough room for a second symbol. */ + int start = rankVal[w]; + U32 const length = 1U << ((targetLog - nbBits) & 0x1F /* quiet static-analyzer */); + int minWeight = nbBits + scaleLog; + int s; + if (minWeight < 1) minWeight = 1; + /* Fill the DTable for every symbol of weight w. + * These symbols get at least 1 second symbol. + */ + for (s = begin; s != end; ++s) { + HUF_fillDTableX2Level2( + DTable + start, targetLog, nbBits, + rankValOrigin[nbBits], minWeight, wEnd, + sortedList, rankStart, + nbBitsBaseline, sortedList[s].symbol); + start += length; + } + } else { + /* Only a single symbol. */ + HUF_fillDTableX2ForWeight( + DTable + rankVal[w], + sortedList + begin, sortedList + end, + nbBits, targetLog, + /* baseSeq */ 0, /* level */ 1); + } + } +} + +typedef struct { + rankValCol_t rankVal[HUF_TABLELOG_MAX]; + U32 rankStats[HUF_TABLELOG_MAX + 1]; + U32 rankStart0[HUF_TABLELOG_MAX + 3]; + sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1]; + BYTE weightList[HUF_SYMBOLVALUE_MAX + 1]; + U32 calleeWksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; +} HUF_ReadDTableX2_Workspace; + +size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, + const void* src, size_t srcSize, + void* workSpace, size_t wkspSize, int flags) +{ + U32 tableLog, maxW, nbSymbols; + DTableDesc dtd = HUF_getDTableDesc(DTable); + U32 maxTableLog = dtd.maxTableLog; + size_t iSize; + void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */ + HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr; + U32 *rankStart; + + HUF_ReadDTableX2_Workspace* const wksp = (HUF_ReadDTableX2_Workspace*)workSpace; + + if (sizeof(*wksp) > wkspSize) return ERROR(GENERIC); + + rankStart = wksp->rankStart0 + 1; + ZSTD_memset(wksp->rankStats, 0, sizeof(wksp->rankStats)); + ZSTD_memset(wksp->rankStart0, 0, sizeof(wksp->rankStart0)); + + DEBUG_STATIC_ASSERT(sizeof(HUF_DEltX2) == sizeof(HUF_DTable)); /* if compiler fails here, assertion is wrong */ + if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); + /* ZSTD_memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */ + + iSize = HUF_readStats_wksp(wksp->weightList, HUF_SYMBOLVALUE_MAX + 1, wksp->rankStats, &nbSymbols, &tableLog, src, srcSize, wksp->calleeWksp, sizeof(wksp->calleeWksp), flags); + if (HUF_isError(iSize)) return iSize; + + /* check result */ + if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */ + if (tableLog <= HUF_DECODER_FAST_TABLELOG && maxTableLog > HUF_DECODER_FAST_TABLELOG) maxTableLog = HUF_DECODER_FAST_TABLELOG; + + /* find maxWeight */ + for (maxW = tableLog; wksp->rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */ + + /* Get start index of each weight */ + { U32 w, nextRankStart = 0; + for (w=1; wrankStats[w]; + rankStart[w] = curr; + } + rankStart[0] = nextRankStart; /* put all 0w symbols at the end of sorted list*/ + rankStart[maxW+1] = nextRankStart; + } + + /* sort symbols by weight */ + { U32 s; + for (s=0; sweightList[s]; + U32 const r = rankStart[w]++; + wksp->sortedSymbol[r].symbol = (BYTE)s; + } + rankStart[0] = 0; /* forget 0w symbols; this is beginning of weight(1) */ + } + + /* Build rankVal */ + { U32* const rankVal0 = wksp->rankVal[0]; + { int const rescale = (maxTableLog-tableLog) - 1; /* tableLog <= maxTableLog */ + U32 nextRankVal = 0; + U32 w; + for (w=1; wrankStats[w] << (w+rescale); + rankVal0[w] = curr; + } } + { U32 const minBits = tableLog+1 - maxW; + U32 consumed; + for (consumed = minBits; consumed < maxTableLog - minBits + 1; consumed++) { + U32* const rankValPtr = wksp->rankVal[consumed]; + U32 w; + for (w = 1; w < maxW+1; w++) { + rankValPtr[w] = rankVal0[w] >> consumed; + } } } } + + HUF_fillDTableX2(dt, maxTableLog, + wksp->sortedSymbol, + wksp->rankStart0, wksp->rankVal, maxW, + tableLog+1); + + dtd.tableLog = (BYTE)maxTableLog; + dtd.tableType = 1; + ZSTD_memcpy(DTable, &dtd, sizeof(dtd)); + return iSize; +} + + +FORCE_INLINE_TEMPLATE U32 +HUF_decodeSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + ZSTD_memcpy(op, &dt[val].sequence, 2); + BIT_skipBits(DStream, dt[val].nbBits); + return dt[val].length; +} + +FORCE_INLINE_TEMPLATE U32 +HUF_decodeLastSymbolX2(void* op, BIT_DStream_t* DStream, const HUF_DEltX2* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + ZSTD_memcpy(op, &dt[val].sequence, 1); + if (dt[val].length==1) { + BIT_skipBits(DStream, dt[val].nbBits); + } else { + if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) { + BIT_skipBits(DStream, dt[val].nbBits); + if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8)) + /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */ + DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); + } + } + return 1; +} + +#define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \ + do { ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog); } while (0) + +#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \ + do { \ + if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ + ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog); \ + } while (0) + +#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \ + do { \ + if (MEM_64bits()) \ + ptr += HUF_decodeSymbolX2(ptr, DStreamPtr, dt, dtLog); \ + } while (0) + +HINT_INLINE size_t +HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, + const HUF_DEltX2* const dt, const U32 dtLog) +{ + BYTE* const pStart = p; + + /* up to 8 symbols at a time */ + if ((size_t)(pEnd - p) >= sizeof(bitDPtr->bitContainer)) { + if (dtLog <= 11 && MEM_64bits()) { + /* up to 10 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-9)) { + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + } + } else { + /* up to 8 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-(sizeof(bitDPtr->bitContainer)-1))) { + HUF_DECODE_SYMBOLX2_2(p, bitDPtr); + HUF_DECODE_SYMBOLX2_1(p, bitDPtr); + HUF_DECODE_SYMBOLX2_2(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + } + } + } else { + BIT_reloadDStream(bitDPtr); + } + + /* closer to end : up to 2 symbols at a time */ + if ((size_t)(pEnd - p) >= 2) { + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd-2)) + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + + while (p <= pEnd-2) + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); /* no need to reload : reached the end of DStream */ + } + + if (p < pEnd) + p += HUF_decodeLastSymbolX2(p, bitDPtr, dt, dtLog); + + return p-pStart; +} + +FORCE_INLINE_TEMPLATE size_t +HUF_decompress1X2_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + BIT_DStream_t bitD; + + /* Init */ + CHECK_F( BIT_initDStream(&bitD, cSrc, cSrcSize) ); + + /* decode */ + { BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ZSTD_maybeNullPtrAdd(ostart, dstSize); + const void* const dtPtr = DTable+1; /* force compiler to not use strict-aliasing */ + const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + HUF_decodeStreamX2(ostart, &bitD, oend, dt, dtd.tableLog); + } + + /* check */ + if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); + + /* decoded size */ + return dstSize; +} + +/* HUF_decompress4X2_usingDTable_internal_body(): + * Conditions: + * @dstSize >= 6 + */ +FORCE_INLINE_TEMPLATE size_t +HUF_decompress4X2_usingDTable_internal_body( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + if (dstSize < 6) return ERROR(corruption_detected); /* stream 4-split doesn't work */ + + { const BYTE* const istart = (const BYTE*) cSrc; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + BYTE* const olimit = oend - (sizeof(size_t)-1); + const void* const dtPtr = DTable+1; + const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; + + /* Init */ + BIT_DStream_t bitD1; + BIT_DStream_t bitD2; + BIT_DStream_t bitD3; + BIT_DStream_t bitD4; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); + const BYTE* const istart1 = istart + 6; /* jumpTable */ + const BYTE* const istart2 = istart1 + length1; + const BYTE* const istart3 = istart2 + length2; + const BYTE* const istart4 = istart3 + length3; + size_t const segmentSize = (dstSize+3) / 4; + BYTE* const opStart2 = ostart + segmentSize; + BYTE* const opStart3 = opStart2 + segmentSize; + BYTE* const opStart4 = opStart3 + segmentSize; + BYTE* op1 = ostart; + BYTE* op2 = opStart2; + BYTE* op3 = opStart3; + BYTE* op4 = opStart4; + U32 endSignal = 1; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + + if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + if (opStart4 > oend) return ERROR(corruption_detected); /* overflow */ + assert(dstSize >= 6 /* validated above */); + CHECK_F( BIT_initDStream(&bitD1, istart1, length1) ); + CHECK_F( BIT_initDStream(&bitD2, istart2, length2) ); + CHECK_F( BIT_initDStream(&bitD3, istart3, length3) ); + CHECK_F( BIT_initDStream(&bitD4, istart4, length4) ); + + /* 16-32 symbols per loop (4-8 symbols per stream) */ + if ((size_t)(oend - op4) >= sizeof(size_t)) { + for ( ; (endSignal) & (op4 < olimit); ) { +#if defined(__clang__) && (defined(__x86_64__) || defined(__i386__)) + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_1(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_0(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_1(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_0(op2, &bitD2); + endSignal &= BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished; + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_1(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_0(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_1(op4, &bitD4); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_0(op4, &bitD4); + endSignal &= BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished; + endSignal &= BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished; +#else + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_1(op1, &bitD1); + HUF_DECODE_SYMBOLX2_1(op2, &bitD2); + HUF_DECODE_SYMBOLX2_1(op3, &bitD3); + HUF_DECODE_SYMBOLX2_1(op4, &bitD4); + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_0(op1, &bitD1); + HUF_DECODE_SYMBOLX2_0(op2, &bitD2); + HUF_DECODE_SYMBOLX2_0(op3, &bitD3); + HUF_DECODE_SYMBOLX2_0(op4, &bitD4); + endSignal = (U32)LIKELY((U32) + (BIT_reloadDStreamFast(&bitD1) == BIT_DStream_unfinished) + & (BIT_reloadDStreamFast(&bitD2) == BIT_DStream_unfinished) + & (BIT_reloadDStreamFast(&bitD3) == BIT_DStream_unfinished) + & (BIT_reloadDStreamFast(&bitD4) == BIT_DStream_unfinished)); +#endif + } + } + + /* check corruption */ + if (op1 > opStart2) return ERROR(corruption_detected); + if (op2 > opStart3) return ERROR(corruption_detected); + if (op3 > opStart4) return ERROR(corruption_detected); + /* note : op4 already verified within main loop */ + + /* finish bitStreams one by one */ + HUF_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog); + HUF_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog); + HUF_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog); + HUF_decodeStreamX2(op4, &bitD4, oend, dt, dtLog); + + /* check */ + { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); + if (!endCheck) return ERROR(corruption_detected); } + + /* decoded size */ + return dstSize; + } +} + +#if HUF_NEED_BMI2_FUNCTION +static BMI2_TARGET_ATTRIBUTE +size_t HUF_decompress4X2_usingDTable_internal_bmi2(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X2_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} +#endif + +static +size_t HUF_decompress4X2_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable) { + return HUF_decompress4X2_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); +} + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 + +HUF_ASM_DECL void HUF_decompress4X2_usingDTable_internal_fast_asm_loop(HUF_DecompressFastArgs* args) ZSTDLIB_HIDDEN; + +#endif + +static HUF_FAST_BMI2_ATTRS +void HUF_decompress4X2_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs* args) +{ + U64 bits[4]; + BYTE const* ip[4]; + BYTE* op[4]; + BYTE* oend[4]; + HUF_DEltX2 const* const dtable = (HUF_DEltX2 const*)args->dt; + BYTE const* const ilowest = args->ilowest; + + /* Copy the arguments to local registers. */ + ZSTD_memcpy(&bits, &args->bits, sizeof(bits)); + ZSTD_memcpy((void*)(&ip), &args->ip, sizeof(ip)); + ZSTD_memcpy(&op, &args->op, sizeof(op)); + + oend[0] = op[1]; + oend[1] = op[2]; + oend[2] = op[3]; + oend[3] = args->oend; + + assert(MEM_isLittleEndian()); + assert(!MEM_32bits()); + + for (;;) { + BYTE* olimit; + int stream; + + /* Assert loop preconditions */ +#ifndef NDEBUG + for (stream = 0; stream < 4; ++stream) { + assert(op[stream] <= oend[stream]); + assert(ip[stream] >= ilowest); + } +#endif + /* Compute olimit */ + { + /* Each loop does 5 table lookups for each of the 4 streams. + * Each table lookup consumes up to 11 bits of input, and produces + * up to 2 bytes of output. + */ + /* We can consume up to 7 bytes of input per iteration per stream. + * We also know that each input pointer is >= ip[0]. So we can run + * iters loops before running out of input. + */ + size_t iters = (size_t)(ip[0] - ilowest) / 7; + /* Each iteration can produce up to 10 bytes of output per stream. + * Each output stream my advance at different rates. So take the + * minimum number of safe iterations among all the output streams. + */ + for (stream = 0; stream < 4; ++stream) { + size_t const oiters = (size_t)(oend[stream] - op[stream]) / 10; + iters = MIN(iters, oiters); + } + + /* Each iteration produces at least 5 output symbols. So until + * op[3] crosses olimit, we know we haven't executed iters + * iterations yet. This saves us maintaining an iters counter, + * at the expense of computing the remaining # of iterations + * more frequently. + */ + olimit = op[3] + (iters * 5); + + /* Exit the fast decoding loop once we reach the end. */ + if (op[3] == olimit) + break; + + /* Exit the decoding loop if any input pointer has crossed the + * previous one. This indicates corruption, and a precondition + * to our loop is that ip[i] >= ip[0]. + */ + for (stream = 1; stream < 4; ++stream) { + if (ip[stream] < ip[stream - 1]) + goto _out; + } + } + +#ifndef NDEBUG + for (stream = 1; stream < 4; ++stream) { + assert(ip[stream] >= ip[stream - 1]); + } +#endif + +#define HUF_4X2_DECODE_SYMBOL(_stream, _decode3) \ + do { \ + if ((_decode3) || (_stream) != 3) { \ + int const index = (int)(bits[(_stream)] >> 53); \ + HUF_DEltX2 const entry = dtable[index]; \ + MEM_write16(op[(_stream)], entry.sequence); \ + bits[(_stream)] <<= (entry.nbBits) & 0x3F; \ + op[(_stream)] += (entry.length); \ + } \ + } while (0) + +#define HUF_4X2_RELOAD_STREAM(_stream) \ + do { \ + HUF_4X2_DECODE_SYMBOL(3, 1); \ + { \ + int const ctz = ZSTD_countTrailingZeros64(bits[(_stream)]); \ + int const nbBits = ctz & 7; \ + int const nbBytes = ctz >> 3; \ + ip[(_stream)] -= nbBytes; \ + bits[(_stream)] = MEM_read64(ip[(_stream)]) | 1; \ + bits[(_stream)] <<= nbBits; \ + } \ + } while (0) + + /* Manually unroll the loop because compilers don't consistently + * unroll the inner loops, which destroys performance. + */ + do { + /* Decode 5 symbols from each of the first 3 streams. + * The final stream will be decoded during the reload phase + * to reduce register pressure. + */ + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0); + HUF_4X_FOR_EACH_STREAM_WITH_VAR(HUF_4X2_DECODE_SYMBOL, 0); + + /* Decode one symbol from the final stream */ + HUF_4X2_DECODE_SYMBOL(3, 1); + + /* Decode 4 symbols from the final stream & reload bitstreams. + * The final stream is reloaded last, meaning that all 5 symbols + * are decoded from the final stream before it is reloaded. + */ + HUF_4X_FOR_EACH_STREAM(HUF_4X2_RELOAD_STREAM); + } while (op[3] < olimit); + } + +#undef HUF_4X2_DECODE_SYMBOL +#undef HUF_4X2_RELOAD_STREAM + +_out: + + /* Save the final values of each of the state variables back to args. */ + ZSTD_memcpy(&args->bits, &bits, sizeof(bits)); + ZSTD_memcpy((void*)(&args->ip), &ip, sizeof(ip)); + ZSTD_memcpy(&args->op, &op, sizeof(op)); +} + + +static HUF_FAST_BMI2_ATTRS size_t +HUF_decompress4X2_usingDTable_internal_fast( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable, + HUF_DecompressFastLoopFn loopFn) { + void const* dt = DTable + 1; + const BYTE* const ilowest = (const BYTE*)cSrc; + BYTE* const oend = ZSTD_maybeNullPtrAdd((BYTE*)dst, dstSize); + HUF_DecompressFastArgs args; + { + size_t const ret = HUF_DecompressFastArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable); + FORWARD_IF_ERROR(ret, "Failed to init asm args"); + if (ret == 0) + return 0; + } + + assert(args.ip[0] >= args.ilowest); + loopFn(&args); + + /* note : op4 already verified within main loop */ + assert(args.ip[0] >= ilowest); + assert(args.ip[1] >= ilowest); + assert(args.ip[2] >= ilowest); + assert(args.ip[3] >= ilowest); + assert(args.op[3] <= oend); + + assert(ilowest == args.ilowest); + assert(ilowest + 6 == args.iend[0]); + (void)ilowest; + + /* finish bitStreams one by one */ + { + size_t const segmentSize = (dstSize+3) / 4; + BYTE* segmentEnd = (BYTE*)dst; + int i; + for (i = 0; i < 4; ++i) { + BIT_DStream_t bit; + if (segmentSize <= (size_t)(oend - segmentEnd)) + segmentEnd += segmentSize; + else + segmentEnd = oend; + FORWARD_IF_ERROR(HUF_initRemainingDStream(&bit, &args, i, segmentEnd), "corruption"); + args.op[i] += HUF_decodeStreamX2(args.op[i], &bit, segmentEnd, (HUF_DEltX2 const*)dt, HUF_DECODER_FAST_TABLELOG); + if (args.op[i] != segmentEnd) + return ERROR(corruption_detected); + } + } + + /* decoded size */ + return dstSize; +} + +static size_t HUF_decompress4X2_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc, + size_t cSrcSize, HUF_DTable const* DTable, int flags) +{ + HUF_DecompressUsingDTableFn fallbackFn = HUF_decompress4X2_usingDTable_internal_default; + HUF_DecompressFastLoopFn loopFn = HUF_decompress4X2_usingDTable_internal_fast_c_loop; + +#if DYNAMIC_BMI2 + if (flags & HUF_flags_bmi2) { + fallbackFn = HUF_decompress4X2_usingDTable_internal_bmi2; +# if ZSTD_ENABLE_ASM_X86_64_BMI2 + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X2_usingDTable_internal_fast_asm_loop; + } +# endif + } else { + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); + } +#endif + +#if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__) + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X2_usingDTable_internal_fast_asm_loop; + } +#endif + + if (HUF_ENABLE_FAST_DECODE && !(flags & HUF_flags_disableFast)) { + size_t const ret = HUF_decompress4X2_usingDTable_internal_fast(dst, dstSize, cSrc, cSrcSize, DTable, loopFn); + if (ret != 0) + return ret; + } + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); +} + +HUF_DGEN(HUF_decompress1X2_usingDTable_internal) + +size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize, int flags) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX2_wksp(DCtx, cSrc, cSrcSize, + workSpace, wkspSize, flags); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, flags); +} + +static size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize, int flags) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t hSize = HUF_readDTableX2_wksp(dctx, cSrc, cSrcSize, + workSpace, wkspSize, flags); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags); +} + +#endif /* HUF_FORCE_DECOMPRESS_X1 */ + + +/* ***********************************/ +/* Universal decompression selectors */ +/* ***********************************/ + + +#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2) +typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t; +static const algo_time_t algoTime[16 /* Quantization */][2 /* single, double */] = +{ + /* single, double, quad */ + {{0,0}, {1,1}}, /* Q==0 : impossible */ + {{0,0}, {1,1}}, /* Q==1 : impossible */ + {{ 150,216}, { 381,119}}, /* Q == 2 : 12-18% */ + {{ 170,205}, { 514,112}}, /* Q == 3 : 18-25% */ + {{ 177,199}, { 539,110}}, /* Q == 4 : 25-32% */ + {{ 197,194}, { 644,107}}, /* Q == 5 : 32-38% */ + {{ 221,192}, { 735,107}}, /* Q == 6 : 38-44% */ + {{ 256,189}, { 881,106}}, /* Q == 7 : 44-50% */ + {{ 359,188}, {1167,109}}, /* Q == 8 : 50-56% */ + {{ 582,187}, {1570,114}}, /* Q == 9 : 56-62% */ + {{ 688,187}, {1712,122}}, /* Q ==10 : 62-69% */ + {{ 825,186}, {1965,136}}, /* Q ==11 : 69-75% */ + {{ 976,185}, {2131,150}}, /* Q ==12 : 75-81% */ + {{1180,186}, {2070,175}}, /* Q ==13 : 81-87% */ + {{1377,185}, {1731,202}}, /* Q ==14 : 87-93% */ + {{1412,185}, {1695,202}}, /* Q ==15 : 93-99% */ +}; +#endif + +/** HUF_selectDecoder() : + * Tells which decoder is likely to decode faster, + * based on a set of pre-computed metrics. + * @return : 0==HUF_decompress4X1, 1==HUF_decompress4X2 . + * Assumption : 0 < dstSize <= 128 KB */ +U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize) +{ + assert(dstSize > 0); + assert(dstSize <= 128*1024); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dstSize; + (void)cSrcSize; + return 0; +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dstSize; + (void)cSrcSize; + return 1; +#else + /* decoder timing evaluation */ + { U32 const Q = (cSrcSize >= dstSize) ? 15 : (U32)(cSrcSize * 16 / dstSize); /* Q < 16 */ + U32 const D256 = (U32)(dstSize >> 8); + U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256); + U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256); + DTime1 += DTime1 >> 5; /* small advantage to algorithm using less memory, to reduce cache eviction */ + return DTime1 < DTime0; + } +#endif +} + +size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + void* workSpace, size_t wkspSize, int flags) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ + if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ + if (cSrcSize == 1) { ZSTD_memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)algoNb; + assert(algoNb == 0); + return HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize, flags); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)algoNb; + assert(algoNb == 1); + return HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize, flags); +#else + return algoNb ? HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize, flags): + HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc, + cSrcSize, workSpace, wkspSize, flags); +#endif + } +} + + +size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags) +{ + DTableDesc const dtd = HUF_getDTableDesc(DTable); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dtd; + assert(dtd.tableType == 0); + return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dtd; + assert(dtd.tableType == 1); + return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#else + return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags) : + HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#endif +} + +#ifndef HUF_FORCE_DECOMPRESS_X2 +size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX1_wksp(dctx, cSrc, cSrcSize, workSpace, wkspSize, flags); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags); +} +#endif + +size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags) +{ + DTableDesc const dtd = HUF_getDTableDesc(DTable); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)dtd; + assert(dtd.tableType == 0); + return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)dtd; + assert(dtd.tableType == 1); + return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#else + return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags) : + HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); +#endif +} + +size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize == 0) return ERROR(corruption_detected); + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); +#if defined(HUF_FORCE_DECOMPRESS_X1) + (void)algoNb; + assert(algoNb == 0); + return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags); +#elif defined(HUF_FORCE_DECOMPRESS_X2) + (void)algoNb; + assert(algoNb == 1); + return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags); +#else + return algoNb ? HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags) : + HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags); +#endif + } +} diff --git a/externals/zstd/lib/decompress/zstd_ddict.c b/externals/zstd/lib/decompress/zstd_ddict.c new file mode 100644 index 0000000..309ec0d --- /dev/null +++ b/externals/zstd/lib/decompress/zstd_ddict.c @@ -0,0 +1,244 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* zstd_ddict.c : + * concentrates all logic that needs to know the internals of ZSTD_DDict object */ + +/*-******************************************************* +* Dependencies +*********************************************************/ +#include "../common/allocations.h" /* ZSTD_customMalloc, ZSTD_customFree */ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */ +#include "../common/cpu.h" /* bmi2 */ +#include "../common/mem.h" /* low level memory routines */ +#define FSE_STATIC_LINKING_ONLY +#include "../common/fse.h" +#include "../common/huf.h" +#include "zstd_decompress_internal.h" +#include "zstd_ddict.h" + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) +# include "../legacy/zstd_legacy.h" +#endif + + + +/*-******************************************************* +* Types +*********************************************************/ +struct ZSTD_DDict_s { + void* dictBuffer; + const void* dictContent; + size_t dictSize; + ZSTD_entropyDTables_t entropy; + U32 dictID; + U32 entropyPresent; + ZSTD_customMem cMem; +}; /* typedef'd to ZSTD_DDict within "zstd.h" */ + +const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict) +{ + assert(ddict != NULL); + return ddict->dictContent; +} + +size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict) +{ + assert(ddict != NULL); + return ddict->dictSize; +} + +void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) +{ + DEBUGLOG(4, "ZSTD_copyDDictParameters"); + assert(dctx != NULL); + assert(ddict != NULL); + dctx->dictID = ddict->dictID; + dctx->prefixStart = ddict->dictContent; + dctx->virtualStart = ddict->dictContent; + dctx->dictEnd = (const BYTE*)ddict->dictContent + ddict->dictSize; + dctx->previousDstEnd = dctx->dictEnd; +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + dctx->dictContentBeginForFuzzing = dctx->prefixStart; + dctx->dictContentEndForFuzzing = dctx->previousDstEnd; +#endif + if (ddict->entropyPresent) { + dctx->litEntropy = 1; + dctx->fseEntropy = 1; + dctx->LLTptr = ddict->entropy.LLTable; + dctx->MLTptr = ddict->entropy.MLTable; + dctx->OFTptr = ddict->entropy.OFTable; + dctx->HUFptr = ddict->entropy.hufTable; + dctx->entropy.rep[0] = ddict->entropy.rep[0]; + dctx->entropy.rep[1] = ddict->entropy.rep[1]; + dctx->entropy.rep[2] = ddict->entropy.rep[2]; + } else { + dctx->litEntropy = 0; + dctx->fseEntropy = 0; + } +} + + +static size_t +ZSTD_loadEntropy_intoDDict(ZSTD_DDict* ddict, + ZSTD_dictContentType_e dictContentType) +{ + ddict->dictID = 0; + ddict->entropyPresent = 0; + if (dictContentType == ZSTD_dct_rawContent) return 0; + + if (ddict->dictSize < 8) { + if (dictContentType == ZSTD_dct_fullDict) + return ERROR(dictionary_corrupted); /* only accept specified dictionaries */ + return 0; /* pure content mode */ + } + { U32 const magic = MEM_readLE32(ddict->dictContent); + if (magic != ZSTD_MAGIC_DICTIONARY) { + if (dictContentType == ZSTD_dct_fullDict) + return ERROR(dictionary_corrupted); /* only accept specified dictionaries */ + return 0; /* pure content mode */ + } + } + ddict->dictID = MEM_readLE32((const char*)ddict->dictContent + ZSTD_FRAMEIDSIZE); + + /* load entropy tables */ + RETURN_ERROR_IF(ZSTD_isError(ZSTD_loadDEntropy( + &ddict->entropy, ddict->dictContent, ddict->dictSize)), + dictionary_corrupted, ""); + ddict->entropyPresent = 1; + return 0; +} + + +static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType) +{ + if ((dictLoadMethod == ZSTD_dlm_byRef) || (!dict) || (!dictSize)) { + ddict->dictBuffer = NULL; + ddict->dictContent = dict; + if (!dict) dictSize = 0; + } else { + void* const internalBuffer = ZSTD_customMalloc(dictSize, ddict->cMem); + ddict->dictBuffer = internalBuffer; + ddict->dictContent = internalBuffer; + if (!internalBuffer) return ERROR(memory_allocation); + ZSTD_memcpy(internalBuffer, dict, dictSize); + } + ddict->dictSize = dictSize; + ddict->entropy.hufTable[0] = (HUF_DTable)((ZSTD_HUFFDTABLE_CAPACITY_LOG)*0x1000001); /* cover both little and big endian */ + + /* parse dictionary content */ + FORWARD_IF_ERROR( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) , ""); + + return 0; +} + +ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_customMem customMem) +{ + if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; + + { ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_customMalloc(sizeof(ZSTD_DDict), customMem); + if (ddict == NULL) return NULL; + ddict->cMem = customMem; + { size_t const initResult = ZSTD_initDDict_internal(ddict, + dict, dictSize, + dictLoadMethod, dictContentType); + if (ZSTD_isError(initResult)) { + ZSTD_freeDDict(ddict); + return NULL; + } } + return ddict; + } +} + +/*! ZSTD_createDDict() : +* Create a digested dictionary, to start decompression without startup delay. +* `dict` content is copied inside DDict. +* Consequently, `dict` can be released after `ZSTD_DDict` creation */ +ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize) +{ + ZSTD_customMem const allocator = { NULL, NULL, NULL }; + return ZSTD_createDDict_advanced(dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto, allocator); +} + +/*! ZSTD_createDDict_byReference() : + * Create a digested dictionary, to start decompression without startup delay. + * Dictionary content is simply referenced, it will be accessed during decompression. + * Warning : dictBuffer must outlive DDict (DDict must be freed before dictBuffer) */ +ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize) +{ + ZSTD_customMem const allocator = { NULL, NULL, NULL }; + return ZSTD_createDDict_advanced(dictBuffer, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto, allocator); +} + + +const ZSTD_DDict* ZSTD_initStaticDDict( + void* sBuffer, size_t sBufferSize, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType) +{ + size_t const neededSpace = sizeof(ZSTD_DDict) + + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); + ZSTD_DDict* const ddict = (ZSTD_DDict*)sBuffer; + assert(sBuffer != NULL); + assert(dict != NULL); + if ((size_t)sBuffer & 7) return NULL; /* 8-aligned */ + if (sBufferSize < neededSpace) return NULL; + if (dictLoadMethod == ZSTD_dlm_byCopy) { + ZSTD_memcpy(ddict+1, dict, dictSize); /* local copy */ + dict = ddict+1; + } + if (ZSTD_isError( ZSTD_initDDict_internal(ddict, + dict, dictSize, + ZSTD_dlm_byRef, dictContentType) )) + return NULL; + return ddict; +} + + +size_t ZSTD_freeDDict(ZSTD_DDict* ddict) +{ + if (ddict==NULL) return 0; /* support free on NULL */ + { ZSTD_customMem const cMem = ddict->cMem; + ZSTD_customFree(ddict->dictBuffer, cMem); + ZSTD_customFree(ddict, cMem); + return 0; + } +} + +/*! ZSTD_estimateDDictSize() : + * Estimate amount of memory that will be needed to create a dictionary for decompression. + * Note : dictionary created by reference using ZSTD_dlm_byRef are smaller */ +size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod) +{ + return sizeof(ZSTD_DDict) + (dictLoadMethod == ZSTD_dlm_byRef ? 0 : dictSize); +} + +size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict) +{ + if (ddict==NULL) return 0; /* support sizeof on NULL */ + return sizeof(*ddict) + (ddict->dictBuffer ? ddict->dictSize : 0) ; +} + +/*! ZSTD_getDictID_fromDDict() : + * Provides the dictID of the dictionary loaded into `ddict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) +{ + if (ddict==NULL) return 0; + return ddict->dictID; +} diff --git a/externals/zstd/lib/decompress/zstd_ddict.h b/externals/zstd/lib/decompress/zstd_ddict.h new file mode 100644 index 0000000..c4ca887 --- /dev/null +++ b/externals/zstd/lib/decompress/zstd_ddict.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +#ifndef ZSTD_DDICT_H +#define ZSTD_DDICT_H + +/*-******************************************************* + * Dependencies + *********************************************************/ +#include "../common/zstd_deps.h" /* size_t */ +#include "../zstd.h" /* ZSTD_DDict, and several public functions */ + + +/*-******************************************************* + * Interface + *********************************************************/ + +/* note: several prototypes are already published in `zstd.h` : + * ZSTD_createDDict() + * ZSTD_createDDict_byReference() + * ZSTD_createDDict_advanced() + * ZSTD_freeDDict() + * ZSTD_initStaticDDict() + * ZSTD_sizeof_DDict() + * ZSTD_estimateDDictSize() + * ZSTD_getDictID_fromDict() + */ + +const void* ZSTD_DDict_dictContent(const ZSTD_DDict* ddict); +size_t ZSTD_DDict_dictSize(const ZSTD_DDict* ddict); + +void ZSTD_copyDDictParameters(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); + + + +#endif /* ZSTD_DDICT_H */ diff --git a/externals/zstd/lib/decompress/zstd_decompress.c b/externals/zstd/lib/decompress/zstd_decompress.c new file mode 100644 index 0000000..2f03cf7 --- /dev/null +++ b/externals/zstd/lib/decompress/zstd_decompress.c @@ -0,0 +1,2407 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/* *************************************************************** +* Tuning parameters +*****************************************************************/ +/*! + * HEAPMODE : + * Select how default decompression function ZSTD_decompress() allocates its context, + * on stack (0), or into heap (1, default; requires malloc()). + * Note that functions with explicit context such as ZSTD_decompressDCtx() are unaffected. + */ +#ifndef ZSTD_HEAPMODE +# define ZSTD_HEAPMODE 1 +#endif + +/*! +* LEGACY_SUPPORT : +* if set to 1+, ZSTD_decompress() can decode older formats (v0.1+) +*/ +#ifndef ZSTD_LEGACY_SUPPORT +# define ZSTD_LEGACY_SUPPORT 0 +#endif + +/*! + * MAXWINDOWSIZE_DEFAULT : + * maximum window size accepted by DStream __by default__. + * Frames requiring more memory will be rejected. + * It's possible to set a different limit using ZSTD_DCtx_setMaxWindowSize(). + */ +#ifndef ZSTD_MAXWINDOWSIZE_DEFAULT +# define ZSTD_MAXWINDOWSIZE_DEFAULT (((U32)1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) + 1) +#endif + +/*! + * NO_FORWARD_PROGRESS_MAX : + * maximum allowed nb of calls to ZSTD_decompressStream() + * without any forward progress + * (defined as: no byte read from input, and no byte flushed to output) + * before triggering an error. + */ +#ifndef ZSTD_NO_FORWARD_PROGRESS_MAX +# define ZSTD_NO_FORWARD_PROGRESS_MAX 16 +#endif + + +/*-******************************************************* +* Dependencies +*********************************************************/ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */ +#include "../common/allocations.h" /* ZSTD_customMalloc, ZSTD_customCalloc, ZSTD_customFree */ +#include "../common/error_private.h" +#include "../common/zstd_internal.h" /* blockProperties_t */ +#include "../common/mem.h" /* low level memory routines */ +#include "../common/bits.h" /* ZSTD_highbit32 */ +#define FSE_STATIC_LINKING_ONLY +#include "../common/fse.h" +#include "../common/huf.h" +#include "../common/xxhash.h" /* XXH64_reset, XXH64_update, XXH64_digest, XXH64 */ +#include "zstd_decompress_internal.h" /* ZSTD_DCtx */ +#include "zstd_ddict.h" /* ZSTD_DDictDictContent */ +#include "zstd_decompress_block.h" /* ZSTD_decompressBlock_internal */ + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) +# include "../legacy/zstd_legacy.h" +#endif + + + +/************************************* + * Multiple DDicts Hashset internals * + *************************************/ + +#define DDICT_HASHSET_MAX_LOAD_FACTOR_COUNT_MULT 4 +#define DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT 3 /* These two constants represent SIZE_MULT/COUNT_MULT load factor without using a float. + * Currently, that means a 0.75 load factor. + * So, if count * COUNT_MULT / size * SIZE_MULT != 0, then we've exceeded + * the load factor of the ddict hash set. + */ + +#define DDICT_HASHSET_TABLE_BASE_SIZE 64 +#define DDICT_HASHSET_RESIZE_FACTOR 2 + +/* Hash function to determine starting position of dict insertion within the table + * Returns an index between [0, hashSet->ddictPtrTableSize] + */ +static size_t ZSTD_DDictHashSet_getIndex(const ZSTD_DDictHashSet* hashSet, U32 dictID) { + const U64 hash = XXH64(&dictID, sizeof(U32), 0); + /* DDict ptr table size is a multiple of 2, use size - 1 as mask to get index within [0, hashSet->ddictPtrTableSize) */ + return hash & (hashSet->ddictPtrTableSize - 1); +} + +/* Adds DDict to a hashset without resizing it. + * If inserting a DDict with a dictID that already exists in the set, replaces the one in the set. + * Returns 0 if successful, or a zstd error code if something went wrong. + */ +static size_t ZSTD_DDictHashSet_emplaceDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict) { + const U32 dictID = ZSTD_getDictID_fromDDict(ddict); + size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID); + const size_t idxRangeMask = hashSet->ddictPtrTableSize - 1; + RETURN_ERROR_IF(hashSet->ddictPtrCount == hashSet->ddictPtrTableSize, GENERIC, "Hash set is full!"); + DEBUGLOG(4, "Hashed index: for dictID: %u is %zu", dictID, idx); + while (hashSet->ddictPtrTable[idx] != NULL) { + /* Replace existing ddict if inserting ddict with same dictID */ + if (ZSTD_getDictID_fromDDict(hashSet->ddictPtrTable[idx]) == dictID) { + DEBUGLOG(4, "DictID already exists, replacing rather than adding"); + hashSet->ddictPtrTable[idx] = ddict; + return 0; + } + idx &= idxRangeMask; + idx++; + } + DEBUGLOG(4, "Final idx after probing for dictID %u is: %zu", dictID, idx); + hashSet->ddictPtrTable[idx] = ddict; + hashSet->ddictPtrCount++; + return 0; +} + +/* Expands hash table by factor of DDICT_HASHSET_RESIZE_FACTOR and + * rehashes all values, allocates new table, frees old table. + * Returns 0 on success, otherwise a zstd error code. + */ +static size_t ZSTD_DDictHashSet_expand(ZSTD_DDictHashSet* hashSet, ZSTD_customMem customMem) { + size_t newTableSize = hashSet->ddictPtrTableSize * DDICT_HASHSET_RESIZE_FACTOR; + const ZSTD_DDict** newTable = (const ZSTD_DDict**)ZSTD_customCalloc(sizeof(ZSTD_DDict*) * newTableSize, customMem); + const ZSTD_DDict** oldTable = hashSet->ddictPtrTable; + size_t oldTableSize = hashSet->ddictPtrTableSize; + size_t i; + + DEBUGLOG(4, "Expanding DDict hash table! Old size: %zu new size: %zu", oldTableSize, newTableSize); + RETURN_ERROR_IF(!newTable, memory_allocation, "Expanded hashset allocation failed!"); + hashSet->ddictPtrTable = newTable; + hashSet->ddictPtrTableSize = newTableSize; + hashSet->ddictPtrCount = 0; + for (i = 0; i < oldTableSize; ++i) { + if (oldTable[i] != NULL) { + FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, oldTable[i]), ""); + } + } + ZSTD_customFree((void*)oldTable, customMem); + DEBUGLOG(4, "Finished re-hash"); + return 0; +} + +/* Fetches a DDict with the given dictID + * Returns the ZSTD_DDict* with the requested dictID. If it doesn't exist, then returns NULL. + */ +static const ZSTD_DDict* ZSTD_DDictHashSet_getDDict(ZSTD_DDictHashSet* hashSet, U32 dictID) { + size_t idx = ZSTD_DDictHashSet_getIndex(hashSet, dictID); + const size_t idxRangeMask = hashSet->ddictPtrTableSize - 1; + DEBUGLOG(4, "Hashed index: for dictID: %u is %zu", dictID, idx); + for (;;) { + size_t currDictID = ZSTD_getDictID_fromDDict(hashSet->ddictPtrTable[idx]); + if (currDictID == dictID || currDictID == 0) { + /* currDictID == 0 implies a NULL ddict entry */ + break; + } else { + idx &= idxRangeMask; /* Goes to start of table when we reach the end */ + idx++; + } + } + DEBUGLOG(4, "Final idx after probing for dictID %u is: %zu", dictID, idx); + return hashSet->ddictPtrTable[idx]; +} + +/* Allocates space for and returns a ddict hash set + * The hash set's ZSTD_DDict* table has all values automatically set to NULL to begin with. + * Returns NULL if allocation failed. + */ +static ZSTD_DDictHashSet* ZSTD_createDDictHashSet(ZSTD_customMem customMem) { + ZSTD_DDictHashSet* ret = (ZSTD_DDictHashSet*)ZSTD_customMalloc(sizeof(ZSTD_DDictHashSet), customMem); + DEBUGLOG(4, "Allocating new hash set"); + if (!ret) + return NULL; + ret->ddictPtrTable = (const ZSTD_DDict**)ZSTD_customCalloc(DDICT_HASHSET_TABLE_BASE_SIZE * sizeof(ZSTD_DDict*), customMem); + if (!ret->ddictPtrTable) { + ZSTD_customFree(ret, customMem); + return NULL; + } + ret->ddictPtrTableSize = DDICT_HASHSET_TABLE_BASE_SIZE; + ret->ddictPtrCount = 0; + return ret; +} + +/* Frees the table of ZSTD_DDict* within a hashset, then frees the hashset itself. + * Note: The ZSTD_DDict* within the table are NOT freed. + */ +static void ZSTD_freeDDictHashSet(ZSTD_DDictHashSet* hashSet, ZSTD_customMem customMem) { + DEBUGLOG(4, "Freeing ddict hash set"); + if (hashSet && hashSet->ddictPtrTable) { + ZSTD_customFree((void*)hashSet->ddictPtrTable, customMem); + } + if (hashSet) { + ZSTD_customFree(hashSet, customMem); + } +} + +/* Public function: Adds a DDict into the ZSTD_DDictHashSet, possibly triggering a resize of the hash set. + * Returns 0 on success, or a ZSTD error. + */ +static size_t ZSTD_DDictHashSet_addDDict(ZSTD_DDictHashSet* hashSet, const ZSTD_DDict* ddict, ZSTD_customMem customMem) { + DEBUGLOG(4, "Adding dict ID: %u to hashset with - Count: %zu Tablesize: %zu", ZSTD_getDictID_fromDDict(ddict), hashSet->ddictPtrCount, hashSet->ddictPtrTableSize); + if (hashSet->ddictPtrCount * DDICT_HASHSET_MAX_LOAD_FACTOR_COUNT_MULT / hashSet->ddictPtrTableSize * DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT != 0) { + FORWARD_IF_ERROR(ZSTD_DDictHashSet_expand(hashSet, customMem), ""); + } + FORWARD_IF_ERROR(ZSTD_DDictHashSet_emplaceDDict(hashSet, ddict), ""); + return 0; +} + +/*-************************************************************* +* Context management +***************************************************************/ +size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) +{ + if (dctx==NULL) return 0; /* support sizeof NULL */ + return sizeof(*dctx) + + ZSTD_sizeof_DDict(dctx->ddictLocal) + + dctx->inBuffSize + dctx->outBuffSize; +} + +size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); } + + +static size_t ZSTD_startingInputLength(ZSTD_format_e format) +{ + size_t const startingInputLength = ZSTD_FRAMEHEADERSIZE_PREFIX(format); + /* only supports formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless */ + assert( (format == ZSTD_f_zstd1) || (format == ZSTD_f_zstd1_magicless) ); + return startingInputLength; +} + +static void ZSTD_DCtx_resetParameters(ZSTD_DCtx* dctx) +{ + assert(dctx->streamStage == zdss_init); + dctx->format = ZSTD_f_zstd1; + dctx->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; + dctx->outBufferMode = ZSTD_bm_buffered; + dctx->forceIgnoreChecksum = ZSTD_d_validateChecksum; + dctx->refMultipleDDicts = ZSTD_rmd_refSingleDDict; + dctx->disableHufAsm = 0; + dctx->maxBlockSizeParam = 0; +} + +static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx) +{ + dctx->staticSize = 0; + dctx->ddict = NULL; + dctx->ddictLocal = NULL; + dctx->dictEnd = NULL; + dctx->ddictIsCold = 0; + dctx->dictUses = ZSTD_dont_use; + dctx->inBuff = NULL; + dctx->inBuffSize = 0; + dctx->outBuffSize = 0; + dctx->streamStage = zdss_init; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + dctx->legacyContext = NULL; + dctx->previousLegacyVersion = 0; +#endif + dctx->noForwardProgress = 0; + dctx->oversizedDuration = 0; + dctx->isFrameDecompression = 1; +#if DYNAMIC_BMI2 + dctx->bmi2 = ZSTD_cpuSupportsBmi2(); +#endif + dctx->ddictSet = NULL; + ZSTD_DCtx_resetParameters(dctx); +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + dctx->dictContentEndForFuzzing = NULL; +#endif +} + +ZSTD_DCtx* ZSTD_initStaticDCtx(void *workspace, size_t workspaceSize) +{ + ZSTD_DCtx* const dctx = (ZSTD_DCtx*) workspace; + + if ((size_t)workspace & 7) return NULL; /* 8-aligned */ + if (workspaceSize < sizeof(ZSTD_DCtx)) return NULL; /* minimum size */ + + ZSTD_initDCtx_internal(dctx); + dctx->staticSize = workspaceSize; + dctx->inBuff = (char*)(dctx+1); + return dctx; +} + +static ZSTD_DCtx* ZSTD_createDCtx_internal(ZSTD_customMem customMem) { + if ((!customMem.customAlloc) ^ (!customMem.customFree)) return NULL; + + { ZSTD_DCtx* const dctx = (ZSTD_DCtx*)ZSTD_customMalloc(sizeof(*dctx), customMem); + if (!dctx) return NULL; + dctx->customMem = customMem; + ZSTD_initDCtx_internal(dctx); + return dctx; + } +} + +ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) +{ + return ZSTD_createDCtx_internal(customMem); +} + +ZSTD_DCtx* ZSTD_createDCtx(void) +{ + DEBUGLOG(3, "ZSTD_createDCtx"); + return ZSTD_createDCtx_internal(ZSTD_defaultCMem); +} + +static void ZSTD_clearDict(ZSTD_DCtx* dctx) +{ + ZSTD_freeDDict(dctx->ddictLocal); + dctx->ddictLocal = NULL; + dctx->ddict = NULL; + dctx->dictUses = ZSTD_dont_use; +} + +size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx) +{ + if (dctx==NULL) return 0; /* support free on NULL */ + RETURN_ERROR_IF(dctx->staticSize, memory_allocation, "not compatible with static DCtx"); + { ZSTD_customMem const cMem = dctx->customMem; + ZSTD_clearDict(dctx); + ZSTD_customFree(dctx->inBuff, cMem); + dctx->inBuff = NULL; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (dctx->legacyContext) + ZSTD_freeLegacyStreamContext(dctx->legacyContext, dctx->previousLegacyVersion); +#endif + if (dctx->ddictSet) { + ZSTD_freeDDictHashSet(dctx->ddictSet, cMem); + dctx->ddictSet = NULL; + } + ZSTD_customFree(dctx, cMem); + return 0; + } +} + +/* no longer useful */ +void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) +{ + size_t const toCopy = (size_t)((char*)(&dstDCtx->inBuff) - (char*)dstDCtx); + ZSTD_memcpy(dstDCtx, srcDCtx, toCopy); /* no need to copy workspace */ +} + +/* Given a dctx with a digested frame params, re-selects the correct ZSTD_DDict based on + * the requested dict ID from the frame. If there exists a reference to the correct ZSTD_DDict, then + * accordingly sets the ddict to be used to decompress the frame. + * + * If no DDict is found, then no action is taken, and the ZSTD_DCtx::ddict remains as-is. + * + * ZSTD_d_refMultipleDDicts must be enabled for this function to be called. + */ +static void ZSTD_DCtx_selectFrameDDict(ZSTD_DCtx* dctx) { + assert(dctx->refMultipleDDicts && dctx->ddictSet); + DEBUGLOG(4, "Adjusting DDict based on requested dict ID from frame"); + if (dctx->ddict) { + const ZSTD_DDict* frameDDict = ZSTD_DDictHashSet_getDDict(dctx->ddictSet, dctx->fParams.dictID); + if (frameDDict) { + DEBUGLOG(4, "DDict found!"); + ZSTD_clearDict(dctx); + dctx->dictID = dctx->fParams.dictID; + dctx->ddict = frameDDict; + dctx->dictUses = ZSTD_use_indefinitely; + } + } +} + + +/*-************************************************************* + * Frame header decoding + ***************************************************************/ + +/*! ZSTD_isFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier. + * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. + * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled. + * Note 3 : Skippable Frame Identifiers are considered valid. */ +unsigned ZSTD_isFrame(const void* buffer, size_t size) +{ + if (size < ZSTD_FRAMEIDSIZE) return 0; + { U32 const magic = MEM_readLE32(buffer); + if (magic == ZSTD_MAGICNUMBER) return 1; + if ((magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) return 1; + } +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (ZSTD_isLegacy(buffer, size)) return 1; +#endif + return 0; +} + +/*! ZSTD_isSkippableFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame. + * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. + */ +unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size) +{ + if (size < ZSTD_FRAMEIDSIZE) return 0; + { U32 const magic = MEM_readLE32(buffer); + if ((magic & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) return 1; + } + return 0; +} + +/** ZSTD_frameHeaderSize_internal() : + * srcSize must be large enough to reach header size fields. + * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless. + * @return : size of the Frame Header + * or an error code, which can be tested with ZSTD_isError() */ +static size_t ZSTD_frameHeaderSize_internal(const void* src, size_t srcSize, ZSTD_format_e format) +{ + size_t const minInputSize = ZSTD_startingInputLength(format); + RETURN_ERROR_IF(srcSize < minInputSize, srcSize_wrong, ""); + + { BYTE const fhd = ((const BYTE*)src)[minInputSize-1]; + U32 const dictID= fhd & 3; + U32 const singleSegment = (fhd >> 5) & 1; + U32 const fcsId = fhd >> 6; + return minInputSize + !singleSegment + + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId] + + (singleSegment && !fcsId); + } +} + +/** ZSTD_frameHeaderSize() : + * srcSize must be >= ZSTD_frameHeaderSize_prefix. + * @return : size of the Frame Header, + * or an error code (if srcSize is too small) */ +size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) +{ + return ZSTD_frameHeaderSize_internal(src, srcSize, ZSTD_f_zstd1); +} + + +/** ZSTD_getFrameHeader_advanced() : + * decode Frame Header, or require larger `srcSize`. + * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless + * @return : 0, `zfhPtr` is correctly filled, + * >0, `srcSize` is too small, value is wanted `srcSize` amount, +** or an error code, which can be tested using ZSTD_isError() */ +size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format) +{ + const BYTE* ip = (const BYTE*)src; + size_t const minInputSize = ZSTD_startingInputLength(format); + + DEBUGLOG(5, "ZSTD_getFrameHeader_advanced: minInputSize = %zu, srcSize = %zu", minInputSize, srcSize); + + if (srcSize > 0) { + /* note : technically could be considered an assert(), since it's an invalid entry */ + RETURN_ERROR_IF(src==NULL, GENERIC, "invalid parameter : src==NULL, but srcSize>0"); + } + if (srcSize < minInputSize) { + if (srcSize > 0 && format != ZSTD_f_zstd1_magicless) { + /* when receiving less than @minInputSize bytes, + * control these bytes at least correspond to a supported magic number + * in order to error out early if they don't. + **/ + size_t const toCopy = MIN(4, srcSize); + unsigned char hbuf[4]; MEM_writeLE32(hbuf, ZSTD_MAGICNUMBER); + assert(src != NULL); + ZSTD_memcpy(hbuf, src, toCopy); + if ( MEM_readLE32(hbuf) != ZSTD_MAGICNUMBER ) { + /* not a zstd frame : let's check if it's a skippable frame */ + MEM_writeLE32(hbuf, ZSTD_MAGIC_SKIPPABLE_START); + ZSTD_memcpy(hbuf, src, toCopy); + if ((MEM_readLE32(hbuf) & ZSTD_MAGIC_SKIPPABLE_MASK) != ZSTD_MAGIC_SKIPPABLE_START) { + RETURN_ERROR(prefix_unknown, + "first bytes don't correspond to any supported magic number"); + } } } + return minInputSize; + } + + ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr)); /* not strictly necessary, but static analyzers may not understand that zfhPtr will be read only if return value is zero, since they are 2 different signals */ + if ( (format != ZSTD_f_zstd1_magicless) + && (MEM_readLE32(src) != ZSTD_MAGICNUMBER) ) { + if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + /* skippable frame */ + if (srcSize < ZSTD_SKIPPABLEHEADERSIZE) + return ZSTD_SKIPPABLEHEADERSIZE; /* magic number + frame length */ + ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr)); + zfhPtr->frameContentSize = MEM_readLE32((const char *)src + ZSTD_FRAMEIDSIZE); + zfhPtr->frameType = ZSTD_skippableFrame; + return 0; + } + RETURN_ERROR(prefix_unknown, ""); + } + + /* ensure there is enough `srcSize` to fully read/decode frame header */ + { size_t const fhsize = ZSTD_frameHeaderSize_internal(src, srcSize, format); + if (srcSize < fhsize) return fhsize; + zfhPtr->headerSize = (U32)fhsize; + } + + { BYTE const fhdByte = ip[minInputSize-1]; + size_t pos = minInputSize; + U32 const dictIDSizeCode = fhdByte&3; + U32 const checksumFlag = (fhdByte>>2)&1; + U32 const singleSegment = (fhdByte>>5)&1; + U32 const fcsID = fhdByte>>6; + U64 windowSize = 0; + U32 dictID = 0; + U64 frameContentSize = ZSTD_CONTENTSIZE_UNKNOWN; + RETURN_ERROR_IF((fhdByte & 0x08) != 0, frameParameter_unsupported, + "reserved bits, must be zero"); + + if (!singleSegment) { + BYTE const wlByte = ip[pos++]; + U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN; + RETURN_ERROR_IF(windowLog > ZSTD_WINDOWLOG_MAX, frameParameter_windowTooLarge, ""); + windowSize = (1ULL << windowLog); + windowSize += (windowSize >> 3) * (wlByte&7); + } + switch(dictIDSizeCode) + { + default: + assert(0); /* impossible */ + ZSTD_FALLTHROUGH; + case 0 : break; + case 1 : dictID = ip[pos]; pos++; break; + case 2 : dictID = MEM_readLE16(ip+pos); pos+=2; break; + case 3 : dictID = MEM_readLE32(ip+pos); pos+=4; break; + } + switch(fcsID) + { + default: + assert(0); /* impossible */ + ZSTD_FALLTHROUGH; + case 0 : if (singleSegment) frameContentSize = ip[pos]; break; + case 1 : frameContentSize = MEM_readLE16(ip+pos)+256; break; + case 2 : frameContentSize = MEM_readLE32(ip+pos); break; + case 3 : frameContentSize = MEM_readLE64(ip+pos); break; + } + if (singleSegment) windowSize = frameContentSize; + + zfhPtr->frameType = ZSTD_frame; + zfhPtr->frameContentSize = frameContentSize; + zfhPtr->windowSize = windowSize; + zfhPtr->blockSizeMax = (unsigned) MIN(windowSize, ZSTD_BLOCKSIZE_MAX); + zfhPtr->dictID = dictID; + zfhPtr->checksumFlag = checksumFlag; + } + return 0; +} + +/** ZSTD_getFrameHeader() : + * decode Frame Header, or require larger `srcSize`. + * note : this function does not consume input, it only reads it. + * @return : 0, `zfhPtr` is correctly filled, + * >0, `srcSize` is too small, value is wanted `srcSize` amount, + * or an error code, which can be tested using ZSTD_isError() */ +size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize) +{ + return ZSTD_getFrameHeader_advanced(zfhPtr, src, srcSize, ZSTD_f_zstd1); +} + +/** ZSTD_getFrameContentSize() : + * compatible with legacy mode + * @return : decompressed size of the single frame pointed to be `src` if known, otherwise + * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined + * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) */ +unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize) +{ +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (ZSTD_isLegacy(src, srcSize)) { + unsigned long long const ret = ZSTD_getDecompressedSize_legacy(src, srcSize); + return ret == 0 ? ZSTD_CONTENTSIZE_UNKNOWN : ret; + } +#endif + { ZSTD_frameHeader zfh; + if (ZSTD_getFrameHeader(&zfh, src, srcSize) != 0) + return ZSTD_CONTENTSIZE_ERROR; + if (zfh.frameType == ZSTD_skippableFrame) { + return 0; + } else { + return zfh.frameContentSize; + } } +} + +static size_t readSkippableFrameSize(void const* src, size_t srcSize) +{ + size_t const skippableHeaderSize = ZSTD_SKIPPABLEHEADERSIZE; + U32 sizeU32; + + RETURN_ERROR_IF(srcSize < ZSTD_SKIPPABLEHEADERSIZE, srcSize_wrong, ""); + + sizeU32 = MEM_readLE32((BYTE const*)src + ZSTD_FRAMEIDSIZE); + RETURN_ERROR_IF((U32)(sizeU32 + ZSTD_SKIPPABLEHEADERSIZE) < sizeU32, + frameParameter_unsupported, ""); + { size_t const skippableSize = skippableHeaderSize + sizeU32; + RETURN_ERROR_IF(skippableSize > srcSize, srcSize_wrong, ""); + return skippableSize; + } +} + +/*! ZSTD_readSkippableFrame() : + * Retrieves content of a skippable frame, and writes it to dst buffer. + * + * The parameter magicVariant will receive the magicVariant that was supplied when the frame was written, + * i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. This can be NULL if the caller is not interested + * in the magicVariant. + * + * Returns an error if destination buffer is not large enough, or if this is not a valid skippable frame. + * + * @return : number of bytes written or a ZSTD error. + */ +size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, + unsigned* magicVariant, /* optional, can be NULL */ + const void* src, size_t srcSize) +{ + RETURN_ERROR_IF(srcSize < ZSTD_SKIPPABLEHEADERSIZE, srcSize_wrong, ""); + + { U32 const magicNumber = MEM_readLE32(src); + size_t skippableFrameSize = readSkippableFrameSize(src, srcSize); + size_t skippableContentSize = skippableFrameSize - ZSTD_SKIPPABLEHEADERSIZE; + + /* check input validity */ + RETURN_ERROR_IF(!ZSTD_isSkippableFrame(src, srcSize), frameParameter_unsupported, ""); + RETURN_ERROR_IF(skippableFrameSize < ZSTD_SKIPPABLEHEADERSIZE || skippableFrameSize > srcSize, srcSize_wrong, ""); + RETURN_ERROR_IF(skippableContentSize > dstCapacity, dstSize_tooSmall, ""); + + /* deliver payload */ + if (skippableContentSize > 0 && dst != NULL) + ZSTD_memcpy(dst, (const BYTE *)src + ZSTD_SKIPPABLEHEADERSIZE, skippableContentSize); + if (magicVariant != NULL) + *magicVariant = magicNumber - ZSTD_MAGIC_SKIPPABLE_START; + return skippableContentSize; + } +} + +/** ZSTD_findDecompressedSize() : + * `srcSize` must be the exact length of some number of ZSTD compressed and/or + * skippable frames + * note: compatible with legacy mode + * @return : decompressed size of the frames contained */ +unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) +{ + unsigned long long totalDstSize = 0; + + while (srcSize >= ZSTD_startingInputLength(ZSTD_f_zstd1)) { + U32 const magicNumber = MEM_readLE32(src); + + if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + size_t const skippableSize = readSkippableFrameSize(src, srcSize); + if (ZSTD_isError(skippableSize)) return ZSTD_CONTENTSIZE_ERROR; + assert(skippableSize <= srcSize); + + src = (const BYTE *)src + skippableSize; + srcSize -= skippableSize; + continue; + } + + { unsigned long long const fcs = ZSTD_getFrameContentSize(src, srcSize); + if (fcs >= ZSTD_CONTENTSIZE_ERROR) return fcs; + + if (totalDstSize + fcs < totalDstSize) + return ZSTD_CONTENTSIZE_ERROR; /* check for overflow */ + totalDstSize += fcs; + } + /* skip to next frame */ + { size_t const frameSrcSize = ZSTD_findFrameCompressedSize(src, srcSize); + if (ZSTD_isError(frameSrcSize)) return ZSTD_CONTENTSIZE_ERROR; + assert(frameSrcSize <= srcSize); + + src = (const BYTE *)src + frameSrcSize; + srcSize -= frameSrcSize; + } + } /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */ + + if (srcSize) return ZSTD_CONTENTSIZE_ERROR; + + return totalDstSize; +} + +/** ZSTD_getDecompressedSize() : + * compatible with legacy mode + * @return : decompressed size if known, 0 otherwise + note : 0 can mean any of the following : + - frame content is empty + - decompressed size field is not present in frame header + - frame header unknown / not supported + - frame header not complete (`srcSize` too small) */ +unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize) +{ + unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize); + ZSTD_STATIC_ASSERT(ZSTD_CONTENTSIZE_ERROR < ZSTD_CONTENTSIZE_UNKNOWN); + return (ret >= ZSTD_CONTENTSIZE_ERROR) ? 0 : ret; +} + + +/** ZSTD_decodeFrameHeader() : + * `headerSize` must be the size provided by ZSTD_frameHeaderSize(). + * If multiple DDict references are enabled, also will choose the correct DDict to use. + * @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */ +static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t headerSize) +{ + size_t const result = ZSTD_getFrameHeader_advanced(&(dctx->fParams), src, headerSize, dctx->format); + if (ZSTD_isError(result)) return result; /* invalid header */ + RETURN_ERROR_IF(result>0, srcSize_wrong, "headerSize too small"); + + /* Reference DDict requested by frame if dctx references multiple ddicts */ + if (dctx->refMultipleDDicts == ZSTD_rmd_refMultipleDDicts && dctx->ddictSet) { + ZSTD_DCtx_selectFrameDDict(dctx); + } + +#ifndef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + /* Skip the dictID check in fuzzing mode, because it makes the search + * harder. + */ + RETURN_ERROR_IF(dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID), + dictionary_wrong, ""); +#endif + dctx->validateChecksum = (dctx->fParams.checksumFlag && !dctx->forceIgnoreChecksum) ? 1 : 0; + if (dctx->validateChecksum) XXH64_reset(&dctx->xxhState, 0); + dctx->processedCSize += headerSize; + return 0; +} + +static ZSTD_frameSizeInfo ZSTD_errorFrameSizeInfo(size_t ret) +{ + ZSTD_frameSizeInfo frameSizeInfo; + frameSizeInfo.compressedSize = ret; + frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR; + return frameSizeInfo; +} + +static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize, ZSTD_format_e format) +{ + ZSTD_frameSizeInfo frameSizeInfo; + ZSTD_memset(&frameSizeInfo, 0, sizeof(ZSTD_frameSizeInfo)); + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (format == ZSTD_f_zstd1 && ZSTD_isLegacy(src, srcSize)) + return ZSTD_findFrameSizeInfoLegacy(src, srcSize); +#endif + + if (format == ZSTD_f_zstd1 && (srcSize >= ZSTD_SKIPPABLEHEADERSIZE) + && (MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + frameSizeInfo.compressedSize = readSkippableFrameSize(src, srcSize); + assert(ZSTD_isError(frameSizeInfo.compressedSize) || + frameSizeInfo.compressedSize <= srcSize); + return frameSizeInfo; + } else { + const BYTE* ip = (const BYTE*)src; + const BYTE* const ipstart = ip; + size_t remainingSize = srcSize; + size_t nbBlocks = 0; + ZSTD_frameHeader zfh; + + /* Extract Frame Header */ + { size_t const ret = ZSTD_getFrameHeader_advanced(&zfh, src, srcSize, format); + if (ZSTD_isError(ret)) + return ZSTD_errorFrameSizeInfo(ret); + if (ret > 0) + return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong)); + } + + ip += zfh.headerSize; + remainingSize -= zfh.headerSize; + + /* Iterate over each block */ + while (1) { + blockProperties_t blockProperties; + size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties); + if (ZSTD_isError(cBlockSize)) + return ZSTD_errorFrameSizeInfo(cBlockSize); + + if (ZSTD_blockHeaderSize + cBlockSize > remainingSize) + return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong)); + + ip += ZSTD_blockHeaderSize + cBlockSize; + remainingSize -= ZSTD_blockHeaderSize + cBlockSize; + nbBlocks++; + + if (blockProperties.lastBlock) break; + } + + /* Final frame content checksum */ + if (zfh.checksumFlag) { + if (remainingSize < 4) + return ZSTD_errorFrameSizeInfo(ERROR(srcSize_wrong)); + ip += 4; + } + + frameSizeInfo.nbBlocks = nbBlocks; + frameSizeInfo.compressedSize = (size_t)(ip - ipstart); + frameSizeInfo.decompressedBound = (zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) + ? zfh.frameContentSize + : (unsigned long long)nbBlocks * zfh.blockSizeMax; + return frameSizeInfo; + } +} + +static size_t ZSTD_findFrameCompressedSize_advanced(const void *src, size_t srcSize, ZSTD_format_e format) { + ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize, format); + return frameSizeInfo.compressedSize; +} + +/** ZSTD_findFrameCompressedSize() : + * See docs in zstd.h + * Note: compatible with legacy mode */ +size_t ZSTD_findFrameCompressedSize(const void *src, size_t srcSize) +{ + return ZSTD_findFrameCompressedSize_advanced(src, srcSize, ZSTD_f_zstd1); +} + +/** ZSTD_decompressBound() : + * compatible with legacy mode + * `src` must point to the start of a ZSTD frame or a skippeable frame + * `srcSize` must be at least as large as the frame contained + * @return : the maximum decompressed size of the compressed source + */ +unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize) +{ + unsigned long long bound = 0; + /* Iterate over each frame */ + while (srcSize > 0) { + ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize, ZSTD_f_zstd1); + size_t const compressedSize = frameSizeInfo.compressedSize; + unsigned long long const decompressedBound = frameSizeInfo.decompressedBound; + if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR) + return ZSTD_CONTENTSIZE_ERROR; + assert(srcSize >= compressedSize); + src = (const BYTE*)src + compressedSize; + srcSize -= compressedSize; + bound += decompressedBound; + } + return bound; +} + +size_t ZSTD_decompressionMargin(void const* src, size_t srcSize) +{ + size_t margin = 0; + unsigned maxBlockSize = 0; + + /* Iterate over each frame */ + while (srcSize > 0) { + ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize, ZSTD_f_zstd1); + size_t const compressedSize = frameSizeInfo.compressedSize; + unsigned long long const decompressedBound = frameSizeInfo.decompressedBound; + ZSTD_frameHeader zfh; + + FORWARD_IF_ERROR(ZSTD_getFrameHeader(&zfh, src, srcSize), ""); + if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR) + return ERROR(corruption_detected); + + if (zfh.frameType == ZSTD_frame) { + /* Add the frame header to our margin */ + margin += zfh.headerSize; + /* Add the checksum to our margin */ + margin += zfh.checksumFlag ? 4 : 0; + /* Add 3 bytes per block */ + margin += 3 * frameSizeInfo.nbBlocks; + + /* Compute the max block size */ + maxBlockSize = MAX(maxBlockSize, zfh.blockSizeMax); + } else { + assert(zfh.frameType == ZSTD_skippableFrame); + /* Add the entire skippable frame size to our margin. */ + margin += compressedSize; + } + + assert(srcSize >= compressedSize); + src = (const BYTE*)src + compressedSize; + srcSize -= compressedSize; + } + + /* Add the max block size back to the margin. */ + margin += maxBlockSize; + + return margin; +} + +/*-************************************************************* + * Frame decoding + ***************************************************************/ + +/** ZSTD_insertBlock() : + * insert `src` block into `dctx` history. Useful to track uncompressed blocks. */ +size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize) +{ + DEBUGLOG(5, "ZSTD_insertBlock: %u bytes", (unsigned)blockSize); + ZSTD_checkContinuity(dctx, blockStart, blockSize); + dctx->previousDstEnd = (const char*)blockStart + blockSize; + return blockSize; +} + + +static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_copyRawBlock"); + RETURN_ERROR_IF(srcSize > dstCapacity, dstSize_tooSmall, ""); + if (dst == NULL) { + if (srcSize == 0) return 0; + RETURN_ERROR(dstBuffer_null, ""); + } + ZSTD_memmove(dst, src, srcSize); + return srcSize; +} + +static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity, + BYTE b, + size_t regenSize) +{ + RETURN_ERROR_IF(regenSize > dstCapacity, dstSize_tooSmall, ""); + if (dst == NULL) { + if (regenSize == 0) return 0; + RETURN_ERROR(dstBuffer_null, ""); + } + ZSTD_memset(dst, b, regenSize); + return regenSize; +} + +static void ZSTD_DCtx_trace_end(ZSTD_DCtx const* dctx, U64 uncompressedSize, U64 compressedSize, unsigned streaming) +{ +#if ZSTD_TRACE + if (dctx->traceCtx && ZSTD_trace_decompress_end != NULL) { + ZSTD_Trace trace; + ZSTD_memset(&trace, 0, sizeof(trace)); + trace.version = ZSTD_VERSION_NUMBER; + trace.streaming = streaming; + if (dctx->ddict) { + trace.dictionaryID = ZSTD_getDictID_fromDDict(dctx->ddict); + trace.dictionarySize = ZSTD_DDict_dictSize(dctx->ddict); + trace.dictionaryIsCold = dctx->ddictIsCold; + } + trace.uncompressedSize = (size_t)uncompressedSize; + trace.compressedSize = (size_t)compressedSize; + trace.dctx = dctx; + ZSTD_trace_decompress_end(dctx->traceCtx, &trace); + } +#else + (void)dctx; + (void)uncompressedSize; + (void)compressedSize; + (void)streaming; +#endif +} + + +/*! ZSTD_decompressFrame() : + * @dctx must be properly initialized + * will update *srcPtr and *srcSizePtr, + * to make *srcPtr progress by one frame. */ +static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void** srcPtr, size_t *srcSizePtr) +{ + const BYTE* const istart = (const BYTE*)(*srcPtr); + const BYTE* ip = istart; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = dstCapacity != 0 ? ostart + dstCapacity : ostart; + BYTE* op = ostart; + size_t remainingSrcSize = *srcSizePtr; + + DEBUGLOG(4, "ZSTD_decompressFrame (srcSize:%i)", (int)*srcSizePtr); + + /* check */ + RETURN_ERROR_IF( + remainingSrcSize < ZSTD_FRAMEHEADERSIZE_MIN(dctx->format)+ZSTD_blockHeaderSize, + srcSize_wrong, ""); + + /* Frame Header */ + { size_t const frameHeaderSize = ZSTD_frameHeaderSize_internal( + ip, ZSTD_FRAMEHEADERSIZE_PREFIX(dctx->format), dctx->format); + if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize; + RETURN_ERROR_IF(remainingSrcSize < frameHeaderSize+ZSTD_blockHeaderSize, + srcSize_wrong, ""); + FORWARD_IF_ERROR( ZSTD_decodeFrameHeader(dctx, ip, frameHeaderSize) , ""); + ip += frameHeaderSize; remainingSrcSize -= frameHeaderSize; + } + + /* Shrink the blockSizeMax if enabled */ + if (dctx->maxBlockSizeParam != 0) + dctx->fParams.blockSizeMax = MIN(dctx->fParams.blockSizeMax, (unsigned)dctx->maxBlockSizeParam); + + /* Loop on each block */ + while (1) { + BYTE* oBlockEnd = oend; + size_t decodedSize; + blockProperties_t blockProperties; + size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSrcSize, &blockProperties); + if (ZSTD_isError(cBlockSize)) return cBlockSize; + + ip += ZSTD_blockHeaderSize; + remainingSrcSize -= ZSTD_blockHeaderSize; + RETURN_ERROR_IF(cBlockSize > remainingSrcSize, srcSize_wrong, ""); + + if (ip >= op && ip < oBlockEnd) { + /* We are decompressing in-place. Limit the output pointer so that we + * don't overwrite the block that we are currently reading. This will + * fail decompression if the input & output pointers aren't spaced + * far enough apart. + * + * This is important to set, even when the pointers are far enough + * apart, because ZSTD_decompressBlock_internal() can decide to store + * literals in the output buffer, after the block it is decompressing. + * Since we don't want anything to overwrite our input, we have to tell + * ZSTD_decompressBlock_internal to never write past ip. + * + * See ZSTD_allocateLiteralsBuffer() for reference. + */ + oBlockEnd = op + (ip - op); + } + + switch(blockProperties.blockType) + { + case bt_compressed: + assert(dctx->isFrameDecompression == 1); + decodedSize = ZSTD_decompressBlock_internal(dctx, op, (size_t)(oBlockEnd-op), ip, cBlockSize, not_streaming); + break; + case bt_raw : + /* Use oend instead of oBlockEnd because this function is safe to overlap. It uses memmove. */ + decodedSize = ZSTD_copyRawBlock(op, (size_t)(oend-op), ip, cBlockSize); + break; + case bt_rle : + decodedSize = ZSTD_setRleBlock(op, (size_t)(oBlockEnd-op), *ip, blockProperties.origSize); + break; + case bt_reserved : + default: + RETURN_ERROR(corruption_detected, "invalid block type"); + } + FORWARD_IF_ERROR(decodedSize, "Block decompression failure"); + DEBUGLOG(5, "Decompressed block of dSize = %u", (unsigned)decodedSize); + if (dctx->validateChecksum) { + XXH64_update(&dctx->xxhState, op, decodedSize); + } + if (decodedSize) /* support dst = NULL,0 */ { + op += decodedSize; + } + assert(ip != NULL); + ip += cBlockSize; + remainingSrcSize -= cBlockSize; + if (blockProperties.lastBlock) break; + } + + if (dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) { + RETURN_ERROR_IF((U64)(op-ostart) != dctx->fParams.frameContentSize, + corruption_detected, ""); + } + if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */ + RETURN_ERROR_IF(remainingSrcSize<4, checksum_wrong, ""); + if (!dctx->forceIgnoreChecksum) { + U32 const checkCalc = (U32)XXH64_digest(&dctx->xxhState); + U32 checkRead; + checkRead = MEM_readLE32(ip); + RETURN_ERROR_IF(checkRead != checkCalc, checksum_wrong, ""); + } + ip += 4; + remainingSrcSize -= 4; + } + ZSTD_DCtx_trace_end(dctx, (U64)(op-ostart), (U64)(ip-istart), /* streaming */ 0); + /* Allow caller to get size read */ + DEBUGLOG(4, "ZSTD_decompressFrame: decompressed frame of size %zi, consuming %zi bytes of input", op-ostart, ip - (const BYTE*)*srcPtr); + *srcPtr = ip; + *srcSizePtr = remainingSrcSize; + return (size_t)(op-ostart); +} + +static +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize, + const ZSTD_DDict* ddict) +{ + void* const dststart = dst; + int moreThan1Frame = 0; + + DEBUGLOG(5, "ZSTD_decompressMultiFrame"); + assert(dict==NULL || ddict==NULL); /* either dict or ddict set, not both */ + + if (ddict) { + dict = ZSTD_DDict_dictContent(ddict); + dictSize = ZSTD_DDict_dictSize(ddict); + } + + while (srcSize >= ZSTD_startingInputLength(dctx->format)) { + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (dctx->format == ZSTD_f_zstd1 && ZSTD_isLegacy(src, srcSize)) { + size_t decodedSize; + size_t const frameSize = ZSTD_findFrameCompressedSizeLegacy(src, srcSize); + if (ZSTD_isError(frameSize)) return frameSize; + RETURN_ERROR_IF(dctx->staticSize, memory_allocation, + "legacy support is not compatible with static dctx"); + + decodedSize = ZSTD_decompressLegacy(dst, dstCapacity, src, frameSize, dict, dictSize); + if (ZSTD_isError(decodedSize)) return decodedSize; + + { + unsigned long long const expectedSize = ZSTD_getFrameContentSize(src, srcSize); + RETURN_ERROR_IF(expectedSize == ZSTD_CONTENTSIZE_ERROR, corruption_detected, "Corrupted frame header!"); + if (expectedSize != ZSTD_CONTENTSIZE_UNKNOWN) { + RETURN_ERROR_IF(expectedSize != decodedSize, corruption_detected, + "Frame header size does not match decoded size!"); + } + } + + assert(decodedSize <= dstCapacity); + dst = (BYTE*)dst + decodedSize; + dstCapacity -= decodedSize; + + src = (const BYTE*)src + frameSize; + srcSize -= frameSize; + + continue; + } +#endif + + if (dctx->format == ZSTD_f_zstd1 && srcSize >= 4) { + U32 const magicNumber = MEM_readLE32(src); + DEBUGLOG(5, "reading magic number %08X", (unsigned)magicNumber); + if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + /* skippable frame detected : skip it */ + size_t const skippableSize = readSkippableFrameSize(src, srcSize); + FORWARD_IF_ERROR(skippableSize, "invalid skippable frame"); + assert(skippableSize <= srcSize); + + src = (const BYTE *)src + skippableSize; + srcSize -= skippableSize; + continue; /* check next frame */ + } } + + if (ddict) { + /* we were called from ZSTD_decompress_usingDDict */ + FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(dctx, ddict), ""); + } else { + /* this will initialize correctly with no dict if dict == NULL, so + * use this in all cases but ddict */ + FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDict(dctx, dict, dictSize), ""); + } + ZSTD_checkContinuity(dctx, dst, dstCapacity); + + { const size_t res = ZSTD_decompressFrame(dctx, dst, dstCapacity, + &src, &srcSize); + RETURN_ERROR_IF( + (ZSTD_getErrorCode(res) == ZSTD_error_prefix_unknown) + && (moreThan1Frame==1), + srcSize_wrong, + "At least one frame successfully completed, " + "but following bytes are garbage: " + "it's more likely to be a srcSize error, " + "specifying more input bytes than size of frame(s). " + "Note: one could be unlucky, it might be a corruption error instead, " + "happening right at the place where we expect zstd magic bytes. " + "But this is _much_ less likely than a srcSize field error."); + if (ZSTD_isError(res)) return res; + assert(res <= dstCapacity); + if (res != 0) + dst = (BYTE*)dst + res; + dstCapacity -= res; + } + moreThan1Frame = 1; + } /* while (srcSize >= ZSTD_frameHeaderSize_prefix) */ + + RETURN_ERROR_IF(srcSize, srcSize_wrong, "input not entirely consumed"); + + return (size_t)((BYTE*)dst - (BYTE*)dststart); +} + +size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize) +{ + return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, dict, dictSize, NULL); +} + + +static ZSTD_DDict const* ZSTD_getDDict(ZSTD_DCtx* dctx) +{ + switch (dctx->dictUses) { + default: + assert(0 /* Impossible */); + ZSTD_FALLTHROUGH; + case ZSTD_dont_use: + ZSTD_clearDict(dctx); + return NULL; + case ZSTD_use_indefinitely: + return dctx->ddict; + case ZSTD_use_once: + dctx->dictUses = ZSTD_dont_use; + return dctx->ddict; + } +} + +size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + return ZSTD_decompress_usingDDict(dctx, dst, dstCapacity, src, srcSize, ZSTD_getDDict(dctx)); +} + + +size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ +#if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE>=1) + size_t regenSize; + ZSTD_DCtx* const dctx = ZSTD_createDCtx_internal(ZSTD_defaultCMem); + RETURN_ERROR_IF(dctx==NULL, memory_allocation, "NULL pointer!"); + regenSize = ZSTD_decompressDCtx(dctx, dst, dstCapacity, src, srcSize); + ZSTD_freeDCtx(dctx); + return regenSize; +#else /* stack mode */ + ZSTD_DCtx dctx; + ZSTD_initDCtx_internal(&dctx); + return ZSTD_decompressDCtx(&dctx, dst, dstCapacity, src, srcSize); +#endif +} + + +/*-************************************** +* Advanced Streaming Decompression API +* Bufferless and synchronous +****************************************/ +size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) { return dctx->expected; } + +/** + * Similar to ZSTD_nextSrcSizeToDecompress(), but when a block input can be streamed, we + * allow taking a partial block as the input. Currently only raw uncompressed blocks can + * be streamed. + * + * For blocks that can be streamed, this allows us to reduce the latency until we produce + * output, and avoid copying the input. + * + * @param inputSize - The total amount of input that the caller currently has. + */ +static size_t ZSTD_nextSrcSizeToDecompressWithInputSize(ZSTD_DCtx* dctx, size_t inputSize) { + if (!(dctx->stage == ZSTDds_decompressBlock || dctx->stage == ZSTDds_decompressLastBlock)) + return dctx->expected; + if (dctx->bType != bt_raw) + return dctx->expected; + return BOUNDED(1, inputSize, dctx->expected); +} + +ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) { + switch(dctx->stage) + { + default: /* should not happen */ + assert(0); + ZSTD_FALLTHROUGH; + case ZSTDds_getFrameHeaderSize: + ZSTD_FALLTHROUGH; + case ZSTDds_decodeFrameHeader: + return ZSTDnit_frameHeader; + case ZSTDds_decodeBlockHeader: + return ZSTDnit_blockHeader; + case ZSTDds_decompressBlock: + return ZSTDnit_block; + case ZSTDds_decompressLastBlock: + return ZSTDnit_lastBlock; + case ZSTDds_checkChecksum: + return ZSTDnit_checksum; + case ZSTDds_decodeSkippableHeader: + ZSTD_FALLTHROUGH; + case ZSTDds_skipFrame: + return ZSTDnit_skippableFrame; + } +} + +static int ZSTD_isSkipFrame(ZSTD_DCtx* dctx) { return dctx->stage == ZSTDds_skipFrame; } + +/** ZSTD_decompressContinue() : + * srcSize : must be the exact nb of bytes expected (see ZSTD_nextSrcSizeToDecompress()) + * @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity) + * or an error code, which can be tested using ZSTD_isError() */ +size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + DEBUGLOG(5, "ZSTD_decompressContinue (srcSize:%u)", (unsigned)srcSize); + /* Sanity check */ + RETURN_ERROR_IF(srcSize != ZSTD_nextSrcSizeToDecompressWithInputSize(dctx, srcSize), srcSize_wrong, "not allowed"); + ZSTD_checkContinuity(dctx, dst, dstCapacity); + + dctx->processedCSize += srcSize; + + switch (dctx->stage) + { + case ZSTDds_getFrameHeaderSize : + assert(src != NULL); + if (dctx->format == ZSTD_f_zstd1) { /* allows header */ + assert(srcSize >= ZSTD_FRAMEIDSIZE); /* to read skippable magic number */ + if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ + ZSTD_memcpy(dctx->headerBuffer, src, srcSize); + dctx->expected = ZSTD_SKIPPABLEHEADERSIZE - srcSize; /* remaining to load to get full skippable frame header */ + dctx->stage = ZSTDds_decodeSkippableHeader; + return 0; + } } + dctx->headerSize = ZSTD_frameHeaderSize_internal(src, srcSize, dctx->format); + if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize; + ZSTD_memcpy(dctx->headerBuffer, src, srcSize); + dctx->expected = dctx->headerSize - srcSize; + dctx->stage = ZSTDds_decodeFrameHeader; + return 0; + + case ZSTDds_decodeFrameHeader: + assert(src != NULL); + ZSTD_memcpy(dctx->headerBuffer + (dctx->headerSize - srcSize), src, srcSize); + FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize), ""); + dctx->expected = ZSTD_blockHeaderSize; + dctx->stage = ZSTDds_decodeBlockHeader; + return 0; + + case ZSTDds_decodeBlockHeader: + { blockProperties_t bp; + size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp); + if (ZSTD_isError(cBlockSize)) return cBlockSize; + RETURN_ERROR_IF(cBlockSize > dctx->fParams.blockSizeMax, corruption_detected, "Block Size Exceeds Maximum"); + dctx->expected = cBlockSize; + dctx->bType = bp.blockType; + dctx->rleSize = bp.origSize; + if (cBlockSize) { + dctx->stage = bp.lastBlock ? ZSTDds_decompressLastBlock : ZSTDds_decompressBlock; + return 0; + } + /* empty block */ + if (bp.lastBlock) { + if (dctx->fParams.checksumFlag) { + dctx->expected = 4; + dctx->stage = ZSTDds_checkChecksum; + } else { + dctx->expected = 0; /* end of frame */ + dctx->stage = ZSTDds_getFrameHeaderSize; + } + } else { + dctx->expected = ZSTD_blockHeaderSize; /* jump to next header */ + dctx->stage = ZSTDds_decodeBlockHeader; + } + return 0; + } + + case ZSTDds_decompressLastBlock: + case ZSTDds_decompressBlock: + DEBUGLOG(5, "ZSTD_decompressContinue: case ZSTDds_decompressBlock"); + { size_t rSize; + switch(dctx->bType) + { + case bt_compressed: + DEBUGLOG(5, "ZSTD_decompressContinue: case bt_compressed"); + assert(dctx->isFrameDecompression == 1); + rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, is_streaming); + dctx->expected = 0; /* Streaming not supported */ + break; + case bt_raw : + assert(srcSize <= dctx->expected); + rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize); + FORWARD_IF_ERROR(rSize, "ZSTD_copyRawBlock failed"); + assert(rSize == srcSize); + dctx->expected -= rSize; + break; + case bt_rle : + rSize = ZSTD_setRleBlock(dst, dstCapacity, *(const BYTE*)src, dctx->rleSize); + dctx->expected = 0; /* Streaming not supported */ + break; + case bt_reserved : /* should never happen */ + default: + RETURN_ERROR(corruption_detected, "invalid block type"); + } + FORWARD_IF_ERROR(rSize, ""); + RETURN_ERROR_IF(rSize > dctx->fParams.blockSizeMax, corruption_detected, "Decompressed Block Size Exceeds Maximum"); + DEBUGLOG(5, "ZSTD_decompressContinue: decoded size from block : %u", (unsigned)rSize); + dctx->decodedSize += rSize; + if (dctx->validateChecksum) XXH64_update(&dctx->xxhState, dst, rSize); + dctx->previousDstEnd = (char*)dst + rSize; + + /* Stay on the same stage until we are finished streaming the block. */ + if (dctx->expected > 0) { + return rSize; + } + + if (dctx->stage == ZSTDds_decompressLastBlock) { /* end of frame */ + DEBUGLOG(4, "ZSTD_decompressContinue: decoded size from frame : %u", (unsigned)dctx->decodedSize); + RETURN_ERROR_IF( + dctx->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN + && dctx->decodedSize != dctx->fParams.frameContentSize, + corruption_detected, ""); + if (dctx->fParams.checksumFlag) { /* another round for frame checksum */ + dctx->expected = 4; + dctx->stage = ZSTDds_checkChecksum; + } else { + ZSTD_DCtx_trace_end(dctx, dctx->decodedSize, dctx->processedCSize, /* streaming */ 1); + dctx->expected = 0; /* ends here */ + dctx->stage = ZSTDds_getFrameHeaderSize; + } + } else { + dctx->stage = ZSTDds_decodeBlockHeader; + dctx->expected = ZSTD_blockHeaderSize; + } + return rSize; + } + + case ZSTDds_checkChecksum: + assert(srcSize == 4); /* guaranteed by dctx->expected */ + { + if (dctx->validateChecksum) { + U32 const h32 = (U32)XXH64_digest(&dctx->xxhState); + U32 const check32 = MEM_readLE32(src); + DEBUGLOG(4, "ZSTD_decompressContinue: checksum : calculated %08X :: %08X read", (unsigned)h32, (unsigned)check32); + RETURN_ERROR_IF(check32 != h32, checksum_wrong, ""); + } + ZSTD_DCtx_trace_end(dctx, dctx->decodedSize, dctx->processedCSize, /* streaming */ 1); + dctx->expected = 0; + dctx->stage = ZSTDds_getFrameHeaderSize; + return 0; + } + + case ZSTDds_decodeSkippableHeader: + assert(src != NULL); + assert(srcSize <= ZSTD_SKIPPABLEHEADERSIZE); + assert(dctx->format != ZSTD_f_zstd1_magicless); + ZSTD_memcpy(dctx->headerBuffer + (ZSTD_SKIPPABLEHEADERSIZE - srcSize), src, srcSize); /* complete skippable header */ + dctx->expected = MEM_readLE32(dctx->headerBuffer + ZSTD_FRAMEIDSIZE); /* note : dctx->expected can grow seriously large, beyond local buffer size */ + dctx->stage = ZSTDds_skipFrame; + return 0; + + case ZSTDds_skipFrame: + dctx->expected = 0; + dctx->stage = ZSTDds_getFrameHeaderSize; + return 0; + + default: + assert(0); /* impossible */ + RETURN_ERROR(GENERIC, "impossible to reach"); /* some compilers require default to do something */ + } +} + + +static size_t ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + dctx->dictEnd = dctx->previousDstEnd; + dctx->virtualStart = (const char*)dict - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart)); + dctx->prefixStart = dict; + dctx->previousDstEnd = (const char*)dict + dictSize; +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + dctx->dictContentBeginForFuzzing = dctx->prefixStart; + dctx->dictContentEndForFuzzing = dctx->previousDstEnd; +#endif + return 0; +} + +/*! ZSTD_loadDEntropy() : + * dict : must point at beginning of a valid zstd dictionary. + * @return : size of entropy tables read */ +size_t +ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, + const void* const dict, size_t const dictSize) +{ + const BYTE* dictPtr = (const BYTE*)dict; + const BYTE* const dictEnd = dictPtr + dictSize; + + RETURN_ERROR_IF(dictSize <= 8, dictionary_corrupted, "dict is too small"); + assert(MEM_readLE32(dict) == ZSTD_MAGIC_DICTIONARY); /* dict must be valid */ + dictPtr += 8; /* skip header = magic + dictID */ + + ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, OFTable) == offsetof(ZSTD_entropyDTables_t, LLTable) + sizeof(entropy->LLTable)); + ZSTD_STATIC_ASSERT(offsetof(ZSTD_entropyDTables_t, MLTable) == offsetof(ZSTD_entropyDTables_t, OFTable) + sizeof(entropy->OFTable)); + ZSTD_STATIC_ASSERT(sizeof(entropy->LLTable) + sizeof(entropy->OFTable) + sizeof(entropy->MLTable) >= HUF_DECOMPRESS_WORKSPACE_SIZE); + { void* const workspace = &entropy->LLTable; /* use fse tables as temporary workspace; implies fse tables are grouped together */ + size_t const workspaceSize = sizeof(entropy->LLTable) + sizeof(entropy->OFTable) + sizeof(entropy->MLTable); +#ifdef HUF_FORCE_DECOMPRESS_X1 + /* in minimal huffman, we always use X1 variants */ + size_t const hSize = HUF_readDTableX1_wksp(entropy->hufTable, + dictPtr, dictEnd - dictPtr, + workspace, workspaceSize, /* flags */ 0); +#else + size_t const hSize = HUF_readDTableX2_wksp(entropy->hufTable, + dictPtr, (size_t)(dictEnd - dictPtr), + workspace, workspaceSize, /* flags */ 0); +#endif + RETURN_ERROR_IF(HUF_isError(hSize), dictionary_corrupted, ""); + dictPtr += hSize; + } + + { short offcodeNCount[MaxOff+1]; + unsigned offcodeMaxValue = MaxOff, offcodeLog; + size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, (size_t)(dictEnd-dictPtr)); + RETURN_ERROR_IF(FSE_isError(offcodeHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(offcodeMaxValue > MaxOff, dictionary_corrupted, ""); + RETURN_ERROR_IF(offcodeLog > OffFSELog, dictionary_corrupted, ""); + ZSTD_buildFSETable( entropy->OFTable, + offcodeNCount, offcodeMaxValue, + OF_base, OF_bits, + offcodeLog, + entropy->workspace, sizeof(entropy->workspace), + /* bmi2 */0); + dictPtr += offcodeHeaderSize; + } + + { short matchlengthNCount[MaxML+1]; + unsigned matchlengthMaxValue = MaxML, matchlengthLog; + size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, (size_t)(dictEnd-dictPtr)); + RETURN_ERROR_IF(FSE_isError(matchlengthHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(matchlengthMaxValue > MaxML, dictionary_corrupted, ""); + RETURN_ERROR_IF(matchlengthLog > MLFSELog, dictionary_corrupted, ""); + ZSTD_buildFSETable( entropy->MLTable, + matchlengthNCount, matchlengthMaxValue, + ML_base, ML_bits, + matchlengthLog, + entropy->workspace, sizeof(entropy->workspace), + /* bmi2 */ 0); + dictPtr += matchlengthHeaderSize; + } + + { short litlengthNCount[MaxLL+1]; + unsigned litlengthMaxValue = MaxLL, litlengthLog; + size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, (size_t)(dictEnd-dictPtr)); + RETURN_ERROR_IF(FSE_isError(litlengthHeaderSize), dictionary_corrupted, ""); + RETURN_ERROR_IF(litlengthMaxValue > MaxLL, dictionary_corrupted, ""); + RETURN_ERROR_IF(litlengthLog > LLFSELog, dictionary_corrupted, ""); + ZSTD_buildFSETable( entropy->LLTable, + litlengthNCount, litlengthMaxValue, + LL_base, LL_bits, + litlengthLog, + entropy->workspace, sizeof(entropy->workspace), + /* bmi2 */ 0); + dictPtr += litlengthHeaderSize; + } + + RETURN_ERROR_IF(dictPtr+12 > dictEnd, dictionary_corrupted, ""); + { int i; + size_t const dictContentSize = (size_t)(dictEnd - (dictPtr+12)); + for (i=0; i<3; i++) { + U32 const rep = MEM_readLE32(dictPtr); dictPtr += 4; + RETURN_ERROR_IF(rep==0 || rep > dictContentSize, + dictionary_corrupted, ""); + entropy->rep[i] = rep; + } } + + return (size_t)(dictPtr - (const BYTE*)dict); +} + +static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + if (dictSize < 8) return ZSTD_refDictContent(dctx, dict, dictSize); + { U32 const magic = MEM_readLE32(dict); + if (magic != ZSTD_MAGIC_DICTIONARY) { + return ZSTD_refDictContent(dctx, dict, dictSize); /* pure content mode */ + } } + dctx->dictID = MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE); + + /* load entropy tables */ + { size_t const eSize = ZSTD_loadDEntropy(&dctx->entropy, dict, dictSize); + RETURN_ERROR_IF(ZSTD_isError(eSize), dictionary_corrupted, ""); + dict = (const char*)dict + eSize; + dictSize -= eSize; + } + dctx->litEntropy = dctx->fseEntropy = 1; + + /* reference dictionary content */ + return ZSTD_refDictContent(dctx, dict, dictSize); +} + +size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) +{ + assert(dctx != NULL); +#if ZSTD_TRACE + dctx->traceCtx = (ZSTD_trace_decompress_begin != NULL) ? ZSTD_trace_decompress_begin(dctx) : 0; +#endif + dctx->expected = ZSTD_startingInputLength(dctx->format); /* dctx->format must be properly set */ + dctx->stage = ZSTDds_getFrameHeaderSize; + dctx->processedCSize = 0; + dctx->decodedSize = 0; + dctx->previousDstEnd = NULL; + dctx->prefixStart = NULL; + dctx->virtualStart = NULL; + dctx->dictEnd = NULL; + dctx->entropy.hufTable[0] = (HUF_DTable)((ZSTD_HUFFDTABLE_CAPACITY_LOG)*0x1000001); /* cover both little and big endian */ + dctx->litEntropy = dctx->fseEntropy = 0; + dctx->dictID = 0; + dctx->bType = bt_reserved; + dctx->isFrameDecompression = 1; + ZSTD_STATIC_ASSERT(sizeof(dctx->entropy.rep) == sizeof(repStartValue)); + ZSTD_memcpy(dctx->entropy.rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */ + dctx->LLTptr = dctx->entropy.LLTable; + dctx->MLTptr = dctx->entropy.MLTable; + dctx->OFTptr = dctx->entropy.OFTable; + dctx->HUFptr = dctx->entropy.hufTable; + return 0; +} + +size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) , ""); + if (dict && dictSize) + RETURN_ERROR_IF( + ZSTD_isError(ZSTD_decompress_insertDictionary(dctx, dict, dictSize)), + dictionary_corrupted, ""); + return 0; +} + + +/* ====== ZSTD_DDict ====== */ + +size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) +{ + DEBUGLOG(4, "ZSTD_decompressBegin_usingDDict"); + assert(dctx != NULL); + if (ddict) { + const char* const dictStart = (const char*)ZSTD_DDict_dictContent(ddict); + size_t const dictSize = ZSTD_DDict_dictSize(ddict); + const void* const dictEnd = dictStart + dictSize; + dctx->ddictIsCold = (dctx->dictEnd != dictEnd); + DEBUGLOG(4, "DDict is %s", + dctx->ddictIsCold ? "~cold~" : "hot!"); + } + FORWARD_IF_ERROR( ZSTD_decompressBegin(dctx) , ""); + if (ddict) { /* NULL ddict is equivalent to no dictionary */ + ZSTD_copyDDictParameters(dctx, ddict); + } + return 0; +} + +/*! ZSTD_getDictID_fromDict() : + * Provides the dictID stored within dictionary. + * if @return == 0, the dictionary is not conformant with Zstandard specification. + * It can still be loaded, but as a content-only dictionary. */ +unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize) +{ + if (dictSize < 8) return 0; + if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) return 0; + return MEM_readLE32((const char*)dict + ZSTD_FRAMEIDSIZE); +} + +/*! ZSTD_getDictID_fromFrame() : + * Provides the dictID required to decompress frame stored within `src`. + * If @return == 0, the dictID could not be decoded. + * This could for one of the following reasons : + * - The frame does not require a dictionary (most common case). + * - The frame was built with dictID intentionally removed. + * Needed dictionary is a hidden piece of information. + * Note : this use case also happens when using a non-conformant dictionary. + * - `srcSize` is too small, and as a result, frame header could not be decoded. + * Note : possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`. + * - This is not a Zstandard frame. + * When identifying the exact failure cause, it's possible to use + * ZSTD_getFrameHeader(), which will provide a more precise error code. */ +unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize) +{ + ZSTD_frameHeader zfp = { 0, 0, 0, ZSTD_frame, 0, 0, 0, 0, 0 }; + size_t const hError = ZSTD_getFrameHeader(&zfp, src, srcSize); + if (ZSTD_isError(hError)) return 0; + return zfp.dictID; +} + + +/*! ZSTD_decompress_usingDDict() : +* Decompression using a pre-digested Dictionary +* Use dictionary without significant overhead. */ +size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_DDict* ddict) +{ + /* pass content and size in case legacy frames are encountered */ + return ZSTD_decompressMultiFrame(dctx, dst, dstCapacity, src, srcSize, + NULL, 0, + ddict); +} + + +/*===================================== +* Streaming decompression +*====================================*/ + +ZSTD_DStream* ZSTD_createDStream(void) +{ + DEBUGLOG(3, "ZSTD_createDStream"); + return ZSTD_createDCtx_internal(ZSTD_defaultCMem); +} + +ZSTD_DStream* ZSTD_initStaticDStream(void *workspace, size_t workspaceSize) +{ + return ZSTD_initStaticDCtx(workspace, workspaceSize); +} + +ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem) +{ + return ZSTD_createDCtx_internal(customMem); +} + +size_t ZSTD_freeDStream(ZSTD_DStream* zds) +{ + return ZSTD_freeDCtx(zds); +} + + +/* *** Initialization *** */ + +size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize; } +size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_MAX; } + +size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType) +{ + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); + ZSTD_clearDict(dctx); + if (dict && dictSize != 0) { + dctx->ddictLocal = ZSTD_createDDict_advanced(dict, dictSize, dictLoadMethod, dictContentType, dctx->customMem); + RETURN_ERROR_IF(dctx->ddictLocal == NULL, memory_allocation, "NULL pointer!"); + dctx->ddict = dctx->ddictLocal; + dctx->dictUses = ZSTD_use_indefinitely; + } + return 0; +} + +size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + return ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byRef, ZSTD_dct_auto); +} + +size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + return ZSTD_DCtx_loadDictionary_advanced(dctx, dict, dictSize, ZSTD_dlm_byCopy, ZSTD_dct_auto); +} + +size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType) +{ + FORWARD_IF_ERROR(ZSTD_DCtx_loadDictionary_advanced(dctx, prefix, prefixSize, ZSTD_dlm_byRef, dictContentType), ""); + dctx->dictUses = ZSTD_use_once; + return 0; +} + +size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize) +{ + return ZSTD_DCtx_refPrefix_advanced(dctx, prefix, prefixSize, ZSTD_dct_rawContent); +} + + +/* ZSTD_initDStream_usingDict() : + * return : expected size, aka ZSTD_startingInputLength(). + * this function cannot fail */ +size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize) +{ + DEBUGLOG(4, "ZSTD_initDStream_usingDict"); + FORWARD_IF_ERROR( ZSTD_DCtx_reset(zds, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_DCtx_loadDictionary(zds, dict, dictSize) , ""); + return ZSTD_startingInputLength(zds->format); +} + +/* note : this variant can't fail */ +size_t ZSTD_initDStream(ZSTD_DStream* zds) +{ + DEBUGLOG(4, "ZSTD_initDStream"); + FORWARD_IF_ERROR(ZSTD_DCtx_reset(zds, ZSTD_reset_session_only), ""); + FORWARD_IF_ERROR(ZSTD_DCtx_refDDict(zds, NULL), ""); + return ZSTD_startingInputLength(zds->format); +} + +/* ZSTD_initDStream_usingDDict() : + * ddict will just be referenced, and must outlive decompression session + * this function cannot fail */ +size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict) +{ + DEBUGLOG(4, "ZSTD_initDStream_usingDDict"); + FORWARD_IF_ERROR( ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only) , ""); + FORWARD_IF_ERROR( ZSTD_DCtx_refDDict(dctx, ddict) , ""); + return ZSTD_startingInputLength(dctx->format); +} + +/* ZSTD_resetDStream() : + * return : expected size, aka ZSTD_startingInputLength(). + * this function cannot fail */ +size_t ZSTD_resetDStream(ZSTD_DStream* dctx) +{ + DEBUGLOG(4, "ZSTD_resetDStream"); + FORWARD_IF_ERROR(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only), ""); + return ZSTD_startingInputLength(dctx->format); +} + + +size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict) +{ + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); + ZSTD_clearDict(dctx); + if (ddict) { + dctx->ddict = ddict; + dctx->dictUses = ZSTD_use_indefinitely; + if (dctx->refMultipleDDicts == ZSTD_rmd_refMultipleDDicts) { + if (dctx->ddictSet == NULL) { + dctx->ddictSet = ZSTD_createDDictHashSet(dctx->customMem); + if (!dctx->ddictSet) { + RETURN_ERROR(memory_allocation, "Failed to allocate memory for hash set!"); + } + } + assert(!dctx->staticSize); /* Impossible: ddictSet cannot have been allocated if static dctx */ + FORWARD_IF_ERROR(ZSTD_DDictHashSet_addDDict(dctx->ddictSet, ddict, dctx->customMem), ""); + } + } + return 0; +} + +/* ZSTD_DCtx_setMaxWindowSize() : + * note : no direct equivalence in ZSTD_DCtx_setParameter, + * since this version sets windowSize, and the other sets windowLog */ +size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize) +{ + ZSTD_bounds const bounds = ZSTD_dParam_getBounds(ZSTD_d_windowLogMax); + size_t const min = (size_t)1 << bounds.lowerBound; + size_t const max = (size_t)1 << bounds.upperBound; + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); + RETURN_ERROR_IF(maxWindowSize < min, parameter_outOfBound, ""); + RETURN_ERROR_IF(maxWindowSize > max, parameter_outOfBound, ""); + dctx->maxWindowSize = maxWindowSize; + return 0; +} + +size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format) +{ + return ZSTD_DCtx_setParameter(dctx, ZSTD_d_format, (int)format); +} + +ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam) +{ + ZSTD_bounds bounds = { 0, 0, 0 }; + switch(dParam) { + case ZSTD_d_windowLogMax: + bounds.lowerBound = ZSTD_WINDOWLOG_ABSOLUTEMIN; + bounds.upperBound = ZSTD_WINDOWLOG_MAX; + return bounds; + case ZSTD_d_format: + bounds.lowerBound = (int)ZSTD_f_zstd1; + bounds.upperBound = (int)ZSTD_f_zstd1_magicless; + ZSTD_STATIC_ASSERT(ZSTD_f_zstd1 < ZSTD_f_zstd1_magicless); + return bounds; + case ZSTD_d_stableOutBuffer: + bounds.lowerBound = (int)ZSTD_bm_buffered; + bounds.upperBound = (int)ZSTD_bm_stable; + return bounds; + case ZSTD_d_forceIgnoreChecksum: + bounds.lowerBound = (int)ZSTD_d_validateChecksum; + bounds.upperBound = (int)ZSTD_d_ignoreChecksum; + return bounds; + case ZSTD_d_refMultipleDDicts: + bounds.lowerBound = (int)ZSTD_rmd_refSingleDDict; + bounds.upperBound = (int)ZSTD_rmd_refMultipleDDicts; + return bounds; + case ZSTD_d_disableHuffmanAssembly: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + case ZSTD_d_maxBlockSize: + bounds.lowerBound = ZSTD_BLOCKSIZE_MAX_MIN; + bounds.upperBound = ZSTD_BLOCKSIZE_MAX; + return bounds; + + default:; + } + bounds.error = ERROR(parameter_unsupported); + return bounds; +} + +/* ZSTD_dParam_withinBounds: + * @return 1 if value is within dParam bounds, + * 0 otherwise */ +static int ZSTD_dParam_withinBounds(ZSTD_dParameter dParam, int value) +{ + ZSTD_bounds const bounds = ZSTD_dParam_getBounds(dParam); + if (ZSTD_isError(bounds.error)) return 0; + if (value < bounds.lowerBound) return 0; + if (value > bounds.upperBound) return 0; + return 1; +} + +#define CHECK_DBOUNDS(p,v) { \ + RETURN_ERROR_IF(!ZSTD_dParam_withinBounds(p, v), parameter_outOfBound, ""); \ +} + +size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value) +{ + switch (param) { + case ZSTD_d_windowLogMax: + *value = (int)ZSTD_highbit32((U32)dctx->maxWindowSize); + return 0; + case ZSTD_d_format: + *value = (int)dctx->format; + return 0; + case ZSTD_d_stableOutBuffer: + *value = (int)dctx->outBufferMode; + return 0; + case ZSTD_d_forceIgnoreChecksum: + *value = (int)dctx->forceIgnoreChecksum; + return 0; + case ZSTD_d_refMultipleDDicts: + *value = (int)dctx->refMultipleDDicts; + return 0; + case ZSTD_d_disableHuffmanAssembly: + *value = (int)dctx->disableHufAsm; + return 0; + case ZSTD_d_maxBlockSize: + *value = dctx->maxBlockSizeParam; + return 0; + default:; + } + RETURN_ERROR(parameter_unsupported, ""); +} + +size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value) +{ + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); + switch(dParam) { + case ZSTD_d_windowLogMax: + if (value == 0) value = ZSTD_WINDOWLOG_LIMIT_DEFAULT; + CHECK_DBOUNDS(ZSTD_d_windowLogMax, value); + dctx->maxWindowSize = ((size_t)1) << value; + return 0; + case ZSTD_d_format: + CHECK_DBOUNDS(ZSTD_d_format, value); + dctx->format = (ZSTD_format_e)value; + return 0; + case ZSTD_d_stableOutBuffer: + CHECK_DBOUNDS(ZSTD_d_stableOutBuffer, value); + dctx->outBufferMode = (ZSTD_bufferMode_e)value; + return 0; + case ZSTD_d_forceIgnoreChecksum: + CHECK_DBOUNDS(ZSTD_d_forceIgnoreChecksum, value); + dctx->forceIgnoreChecksum = (ZSTD_forceIgnoreChecksum_e)value; + return 0; + case ZSTD_d_refMultipleDDicts: + CHECK_DBOUNDS(ZSTD_d_refMultipleDDicts, value); + if (dctx->staticSize != 0) { + RETURN_ERROR(parameter_unsupported, "Static dctx does not support multiple DDicts!"); + } + dctx->refMultipleDDicts = (ZSTD_refMultipleDDicts_e)value; + return 0; + case ZSTD_d_disableHuffmanAssembly: + CHECK_DBOUNDS(ZSTD_d_disableHuffmanAssembly, value); + dctx->disableHufAsm = value != 0; + return 0; + case ZSTD_d_maxBlockSize: + if (value != 0) CHECK_DBOUNDS(ZSTD_d_maxBlockSize, value); + dctx->maxBlockSizeParam = value; + return 0; + default:; + } + RETURN_ERROR(parameter_unsupported, ""); +} + +size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset) +{ + if ( (reset == ZSTD_reset_session_only) + || (reset == ZSTD_reset_session_and_parameters) ) { + dctx->streamStage = zdss_init; + dctx->noForwardProgress = 0; + dctx->isFrameDecompression = 1; + } + if ( (reset == ZSTD_reset_parameters) + || (reset == ZSTD_reset_session_and_parameters) ) { + RETURN_ERROR_IF(dctx->streamStage != zdss_init, stage_wrong, ""); + ZSTD_clearDict(dctx); + ZSTD_DCtx_resetParameters(dctx); + } + return 0; +} + + +size_t ZSTD_sizeof_DStream(const ZSTD_DStream* dctx) +{ + return ZSTD_sizeof_DCtx(dctx); +} + +static size_t ZSTD_decodingBufferSize_internal(unsigned long long windowSize, unsigned long long frameContentSize, size_t blockSizeMax) +{ + size_t const blockSize = MIN((size_t)MIN(windowSize, ZSTD_BLOCKSIZE_MAX), blockSizeMax); + /* We need blockSize + WILDCOPY_OVERLENGTH worth of buffer so that if a block + * ends at windowSize + WILDCOPY_OVERLENGTH + 1 bytes, we can start writing + * the block at the beginning of the output buffer, and maintain a full window. + * + * We need another blockSize worth of buffer so that we can store split + * literals at the end of the block without overwriting the extDict window. + */ + unsigned long long const neededRBSize = windowSize + (blockSize * 2) + (WILDCOPY_OVERLENGTH * 2); + unsigned long long const neededSize = MIN(frameContentSize, neededRBSize); + size_t const minRBSize = (size_t) neededSize; + RETURN_ERROR_IF((unsigned long long)minRBSize != neededSize, + frameParameter_windowTooLarge, ""); + return minRBSize; +} + +size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize) +{ + return ZSTD_decodingBufferSize_internal(windowSize, frameContentSize, ZSTD_BLOCKSIZE_MAX); +} + +size_t ZSTD_estimateDStreamSize(size_t windowSize) +{ + size_t const blockSize = MIN(windowSize, ZSTD_BLOCKSIZE_MAX); + size_t const inBuffSize = blockSize; /* no block can be larger */ + size_t const outBuffSize = ZSTD_decodingBufferSize_min(windowSize, ZSTD_CONTENTSIZE_UNKNOWN); + return ZSTD_estimateDCtxSize() + inBuffSize + outBuffSize; +} + +size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize) +{ + U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX; /* note : should be user-selectable, but requires an additional parameter (or a dctx) */ + ZSTD_frameHeader zfh; + size_t const err = ZSTD_getFrameHeader(&zfh, src, srcSize); + if (ZSTD_isError(err)) return err; + RETURN_ERROR_IF(err>0, srcSize_wrong, ""); + RETURN_ERROR_IF(zfh.windowSize > windowSizeMax, + frameParameter_windowTooLarge, ""); + return ZSTD_estimateDStreamSize((size_t)zfh.windowSize); +} + + +/* ***** Decompression ***** */ + +static int ZSTD_DCtx_isOverflow(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize) +{ + return (zds->inBuffSize + zds->outBuffSize) >= (neededInBuffSize + neededOutBuffSize) * ZSTD_WORKSPACETOOLARGE_FACTOR; +} + +static void ZSTD_DCtx_updateOversizedDuration(ZSTD_DStream* zds, size_t const neededInBuffSize, size_t const neededOutBuffSize) +{ + if (ZSTD_DCtx_isOverflow(zds, neededInBuffSize, neededOutBuffSize)) + zds->oversizedDuration++; + else + zds->oversizedDuration = 0; +} + +static int ZSTD_DCtx_isOversizedTooLong(ZSTD_DStream* zds) +{ + return zds->oversizedDuration >= ZSTD_WORKSPACETOOLARGE_MAXDURATION; +} + +/* Checks that the output buffer hasn't changed if ZSTD_obm_stable is used. */ +static size_t ZSTD_checkOutBuffer(ZSTD_DStream const* zds, ZSTD_outBuffer const* output) +{ + ZSTD_outBuffer const expect = zds->expectedOutBuffer; + /* No requirement when ZSTD_obm_stable is not enabled. */ + if (zds->outBufferMode != ZSTD_bm_stable) + return 0; + /* Any buffer is allowed in zdss_init, this must be the same for every other call until + * the context is reset. + */ + if (zds->streamStage == zdss_init) + return 0; + /* The buffer must match our expectation exactly. */ + if (expect.dst == output->dst && expect.pos == output->pos && expect.size == output->size) + return 0; + RETURN_ERROR(dstBuffer_wrong, "ZSTD_d_stableOutBuffer enabled but output differs!"); +} + +/* Calls ZSTD_decompressContinue() with the right parameters for ZSTD_decompressStream() + * and updates the stage and the output buffer state. This call is extracted so it can be + * used both when reading directly from the ZSTD_inBuffer, and in buffered input mode. + * NOTE: You must break after calling this function since the streamStage is modified. + */ +static size_t ZSTD_decompressContinueStream( + ZSTD_DStream* zds, char** op, char* oend, + void const* src, size_t srcSize) { + int const isSkipFrame = ZSTD_isSkipFrame(zds); + if (zds->outBufferMode == ZSTD_bm_buffered) { + size_t const dstSize = isSkipFrame ? 0 : zds->outBuffSize - zds->outStart; + size_t const decodedSize = ZSTD_decompressContinue(zds, + zds->outBuff + zds->outStart, dstSize, src, srcSize); + FORWARD_IF_ERROR(decodedSize, ""); + if (!decodedSize && !isSkipFrame) { + zds->streamStage = zdss_read; + } else { + zds->outEnd = zds->outStart + decodedSize; + zds->streamStage = zdss_flush; + } + } else { + /* Write directly into the output buffer */ + size_t const dstSize = isSkipFrame ? 0 : (size_t)(oend - *op); + size_t const decodedSize = ZSTD_decompressContinue(zds, *op, dstSize, src, srcSize); + FORWARD_IF_ERROR(decodedSize, ""); + *op += decodedSize; + /* Flushing is not needed. */ + zds->streamStage = zdss_read; + assert(*op <= oend); + assert(zds->outBufferMode == ZSTD_bm_stable); + } + return 0; +} + +size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input) +{ + const char* const src = (const char*)input->src; + const char* const istart = input->pos != 0 ? src + input->pos : src; + const char* const iend = input->size != 0 ? src + input->size : src; + const char* ip = istart; + char* const dst = (char*)output->dst; + char* const ostart = output->pos != 0 ? dst + output->pos : dst; + char* const oend = output->size != 0 ? dst + output->size : dst; + char* op = ostart; + U32 someMoreWork = 1; + + DEBUGLOG(5, "ZSTD_decompressStream"); + RETURN_ERROR_IF( + input->pos > input->size, + srcSize_wrong, + "forbidden. in: pos: %u vs size: %u", + (U32)input->pos, (U32)input->size); + RETURN_ERROR_IF( + output->pos > output->size, + dstSize_tooSmall, + "forbidden. out: pos: %u vs size: %u", + (U32)output->pos, (U32)output->size); + DEBUGLOG(5, "input size : %u", (U32)(input->size - input->pos)); + FORWARD_IF_ERROR(ZSTD_checkOutBuffer(zds, output), ""); + + while (someMoreWork) { + switch(zds->streamStage) + { + case zdss_init : + DEBUGLOG(5, "stage zdss_init => transparent reset "); + zds->streamStage = zdss_loadHeader; + zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + zds->legacyVersion = 0; +#endif + zds->hostageByte = 0; + zds->expectedOutBuffer = *output; + ZSTD_FALLTHROUGH; + + case zdss_loadHeader : + DEBUGLOG(5, "stage zdss_loadHeader (srcSize : %u)", (U32)(iend - ip)); +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + if (zds->legacyVersion) { + RETURN_ERROR_IF(zds->staticSize, memory_allocation, + "legacy support is incompatible with static dctx"); + { size_t const hint = ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input); + if (hint==0) zds->streamStage = zdss_init; + return hint; + } } +#endif + { size_t const hSize = ZSTD_getFrameHeader_advanced(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format); + if (zds->refMultipleDDicts && zds->ddictSet) { + ZSTD_DCtx_selectFrameDDict(zds); + } + if (ZSTD_isError(hSize)) { +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart); + if (legacyVersion) { + ZSTD_DDict const* const ddict = ZSTD_getDDict(zds); + const void* const dict = ddict ? ZSTD_DDict_dictContent(ddict) : NULL; + size_t const dictSize = ddict ? ZSTD_DDict_dictSize(ddict) : 0; + DEBUGLOG(5, "ZSTD_decompressStream: detected legacy version v0.%u", legacyVersion); + RETURN_ERROR_IF(zds->staticSize, memory_allocation, + "legacy support is incompatible with static dctx"); + FORWARD_IF_ERROR(ZSTD_initLegacyStream(&zds->legacyContext, + zds->previousLegacyVersion, legacyVersion, + dict, dictSize), ""); + zds->legacyVersion = zds->previousLegacyVersion = legacyVersion; + { size_t const hint = ZSTD_decompressLegacyStream(zds->legacyContext, legacyVersion, output, input); + if (hint==0) zds->streamStage = zdss_init; /* or stay in stage zdss_loadHeader */ + return hint; + } } +#endif + return hSize; /* error */ + } + if (hSize != 0) { /* need more input */ + size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */ + size_t const remainingInput = (size_t)(iend-ip); + assert(iend >= ip); + if (toLoad > remainingInput) { /* not enough input to load full header */ + if (remainingInput > 0) { + ZSTD_memcpy(zds->headerBuffer + zds->lhSize, ip, remainingInput); + zds->lhSize += remainingInput; + } + input->pos = input->size; + /* check first few bytes */ + FORWARD_IF_ERROR( + ZSTD_getFrameHeader_advanced(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format), + "First few bytes detected incorrect" ); + /* return hint input size */ + return (MAX((size_t)ZSTD_FRAMEHEADERSIZE_MIN(zds->format), hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ + } + assert(ip != NULL); + ZSTD_memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad; + break; + } } + + /* check for single-pass mode opportunity */ + if (zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN + && zds->fParams.frameType != ZSTD_skippableFrame + && (U64)(size_t)(oend-op) >= zds->fParams.frameContentSize) { + size_t const cSize = ZSTD_findFrameCompressedSize_advanced(istart, (size_t)(iend-istart), zds->format); + if (cSize <= (size_t)(iend-istart)) { + /* shortcut : using single-pass mode */ + size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, (size_t)(oend-op), istart, cSize, ZSTD_getDDict(zds)); + if (ZSTD_isError(decompressedSize)) return decompressedSize; + DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()"); + assert(istart != NULL); + ip = istart + cSize; + op = op ? op + decompressedSize : op; /* can occur if frameContentSize = 0 (empty frame) */ + zds->expected = 0; + zds->streamStage = zdss_init; + someMoreWork = 0; + break; + } } + + /* Check output buffer is large enough for ZSTD_odm_stable. */ + if (zds->outBufferMode == ZSTD_bm_stable + && zds->fParams.frameType != ZSTD_skippableFrame + && zds->fParams.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN + && (U64)(size_t)(oend-op) < zds->fParams.frameContentSize) { + RETURN_ERROR(dstSize_tooSmall, "ZSTD_obm_stable passed but ZSTD_outBuffer is too small"); + } + + /* Consume header (see ZSTDds_decodeFrameHeader) */ + DEBUGLOG(4, "Consume header"); + FORWARD_IF_ERROR(ZSTD_decompressBegin_usingDDict(zds, ZSTD_getDDict(zds)), ""); + + if (zds->format == ZSTD_f_zstd1 + && (MEM_readLE32(zds->headerBuffer) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ + zds->expected = MEM_readLE32(zds->headerBuffer + ZSTD_FRAMEIDSIZE); + zds->stage = ZSTDds_skipFrame; + } else { + FORWARD_IF_ERROR(ZSTD_decodeFrameHeader(zds, zds->headerBuffer, zds->lhSize), ""); + zds->expected = ZSTD_blockHeaderSize; + zds->stage = ZSTDds_decodeBlockHeader; + } + + /* control buffer memory usage */ + DEBUGLOG(4, "Control max memory usage (%u KB <= max %u KB)", + (U32)(zds->fParams.windowSize >>10), + (U32)(zds->maxWindowSize >> 10) ); + zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); + RETURN_ERROR_IF(zds->fParams.windowSize > zds->maxWindowSize, + frameParameter_windowTooLarge, ""); + if (zds->maxBlockSizeParam != 0) + zds->fParams.blockSizeMax = MIN(zds->fParams.blockSizeMax, (unsigned)zds->maxBlockSizeParam); + + /* Adapt buffer sizes to frame header instructions */ + { size_t const neededInBuffSize = MAX(zds->fParams.blockSizeMax, 4 /* frame checksum */); + size_t const neededOutBuffSize = zds->outBufferMode == ZSTD_bm_buffered + ? ZSTD_decodingBufferSize_internal(zds->fParams.windowSize, zds->fParams.frameContentSize, zds->fParams.blockSizeMax) + : 0; + + ZSTD_DCtx_updateOversizedDuration(zds, neededInBuffSize, neededOutBuffSize); + + { int const tooSmall = (zds->inBuffSize < neededInBuffSize) || (zds->outBuffSize < neededOutBuffSize); + int const tooLarge = ZSTD_DCtx_isOversizedTooLong(zds); + + if (tooSmall || tooLarge) { + size_t const bufferSize = neededInBuffSize + neededOutBuffSize; + DEBUGLOG(4, "inBuff : from %u to %u", + (U32)zds->inBuffSize, (U32)neededInBuffSize); + DEBUGLOG(4, "outBuff : from %u to %u", + (U32)zds->outBuffSize, (U32)neededOutBuffSize); + if (zds->staticSize) { /* static DCtx */ + DEBUGLOG(4, "staticSize : %u", (U32)zds->staticSize); + assert(zds->staticSize >= sizeof(ZSTD_DCtx)); /* controlled at init */ + RETURN_ERROR_IF( + bufferSize > zds->staticSize - sizeof(ZSTD_DCtx), + memory_allocation, ""); + } else { + ZSTD_customFree(zds->inBuff, zds->customMem); + zds->inBuffSize = 0; + zds->outBuffSize = 0; + zds->inBuff = (char*)ZSTD_customMalloc(bufferSize, zds->customMem); + RETURN_ERROR_IF(zds->inBuff == NULL, memory_allocation, ""); + } + zds->inBuffSize = neededInBuffSize; + zds->outBuff = zds->inBuff + zds->inBuffSize; + zds->outBuffSize = neededOutBuffSize; + } } } + zds->streamStage = zdss_read; + ZSTD_FALLTHROUGH; + + case zdss_read: + DEBUGLOG(5, "stage zdss_read"); + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompressWithInputSize(zds, (size_t)(iend - ip)); + DEBUGLOG(5, "neededInSize = %u", (U32)neededInSize); + if (neededInSize==0) { /* end of frame */ + zds->streamStage = zdss_init; + someMoreWork = 0; + break; + } + if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ + FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, ip, neededInSize), ""); + assert(ip != NULL); + ip += neededInSize; + /* Function modifies the stage so we must break */ + break; + } } + if (ip==iend) { someMoreWork = 0; break; } /* no more input */ + zds->streamStage = zdss_load; + ZSTD_FALLTHROUGH; + + case zdss_load: + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds); + size_t const toLoad = neededInSize - zds->inPos; + int const isSkipFrame = ZSTD_isSkipFrame(zds); + size_t loadedSize; + /* At this point we shouldn't be decompressing a block that we can stream. */ + assert(neededInSize == ZSTD_nextSrcSizeToDecompressWithInputSize(zds, (size_t)(iend - ip))); + if (isSkipFrame) { + loadedSize = MIN(toLoad, (size_t)(iend-ip)); + } else { + RETURN_ERROR_IF(toLoad > zds->inBuffSize - zds->inPos, + corruption_detected, + "should never happen"); + loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, (size_t)(iend-ip)); + } + if (loadedSize != 0) { + /* ip may be NULL */ + ip += loadedSize; + zds->inPos += loadedSize; + } + if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */ + + /* decode loaded input */ + zds->inPos = 0; /* input is consumed */ + FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, zds->inBuff, neededInSize), ""); + /* Function modifies the stage so we must break */ + break; + } + case zdss_flush: + { + size_t const toFlushSize = zds->outEnd - zds->outStart; + size_t const flushedSize = ZSTD_limitCopy(op, (size_t)(oend-op), zds->outBuff + zds->outStart, toFlushSize); + + op = op ? op + flushedSize : op; + + zds->outStart += flushedSize; + if (flushedSize == toFlushSize) { /* flush completed */ + zds->streamStage = zdss_read; + if ( (zds->outBuffSize < zds->fParams.frameContentSize) + && (zds->outStart + zds->fParams.blockSizeMax > zds->outBuffSize) ) { + DEBUGLOG(5, "restart filling outBuff from beginning (left:%i, needed:%u)", + (int)(zds->outBuffSize - zds->outStart), + (U32)zds->fParams.blockSizeMax); + zds->outStart = zds->outEnd = 0; + } + break; + } } + /* cannot complete flush */ + someMoreWork = 0; + break; + + default: + assert(0); /* impossible */ + RETURN_ERROR(GENERIC, "impossible to reach"); /* some compilers require default to do something */ + } } + + /* result */ + input->pos = (size_t)(ip - (const char*)(input->src)); + output->pos = (size_t)(op - (char*)(output->dst)); + + /* Update the expected output buffer for ZSTD_obm_stable. */ + zds->expectedOutBuffer = *output; + + if ((ip==istart) && (op==ostart)) { /* no forward progress */ + zds->noForwardProgress ++; + if (zds->noForwardProgress >= ZSTD_NO_FORWARD_PROGRESS_MAX) { + RETURN_ERROR_IF(op==oend, noForwardProgress_destFull, ""); + RETURN_ERROR_IF(ip==iend, noForwardProgress_inputEmpty, ""); + assert(0); + } + } else { + zds->noForwardProgress = 0; + } + { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds); + if (!nextSrcSizeHint) { /* frame fully decoded */ + if (zds->outEnd == zds->outStart) { /* output fully flushed */ + if (zds->hostageByte) { + if (input->pos >= input->size) { + /* can't release hostage (not present) */ + zds->streamStage = zdss_read; + return 1; + } + input->pos++; /* release hostage */ + } /* zds->hostageByte */ + return 0; + } /* zds->outEnd == zds->outStart */ + if (!zds->hostageByte) { /* output not fully flushed; keep last byte as hostage; will be released when all output is flushed */ + input->pos--; /* note : pos > 0, otherwise, impossible to finish reading last block */ + zds->hostageByte=1; + } + return 1; + } /* nextSrcSizeHint==0 */ + nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds) == ZSTDnit_block); /* preload header of next block */ + assert(zds->inPos <= nextSrcSizeHint); + nextSrcSizeHint -= zds->inPos; /* part already loaded*/ + return nextSrcSizeHint; + } +} + +size_t ZSTD_decompressStream_simpleArgs ( + ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, size_t* dstPos, + const void* src, size_t srcSize, size_t* srcPos) +{ + ZSTD_outBuffer output; + ZSTD_inBuffer input; + output.dst = dst; + output.size = dstCapacity; + output.pos = *dstPos; + input.src = src; + input.size = srcSize; + input.pos = *srcPos; + { size_t const cErr = ZSTD_decompressStream(dctx, &output, &input); + *dstPos = output.pos; + *srcPos = input.pos; + return cErr; + } +} diff --git a/externals/zstd/lib/decompress/zstd_decompress_block.c b/externals/zstd/lib/decompress/zstd_decompress_block.c new file mode 100644 index 0000000..76d7332 --- /dev/null +++ b/externals/zstd/lib/decompress/zstd_decompress_block.c @@ -0,0 +1,2215 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* zstd_decompress_block : + * this module takes care of decompressing _compressed_ block */ + +/*-******************************************************* +* Dependencies +*********************************************************/ +#include "../common/zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */ +#include "../common/compiler.h" /* prefetch */ +#include "../common/cpu.h" /* bmi2 */ +#include "../common/mem.h" /* low level memory routines */ +#define FSE_STATIC_LINKING_ONLY +#include "../common/fse.h" +#include "../common/huf.h" +#include "../common/zstd_internal.h" +#include "zstd_decompress_internal.h" /* ZSTD_DCtx */ +#include "zstd_ddict.h" /* ZSTD_DDictDictContent */ +#include "zstd_decompress_block.h" +#include "../common/bits.h" /* ZSTD_highbit32 */ + +/*_******************************************************* +* Macros +**********************************************************/ + +/* These two optional macros force the use one way or another of the two + * ZSTD_decompressSequences implementations. You can't force in both directions + * at the same time. + */ +#if defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ + defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) +#error "Cannot force the use of the short and the long ZSTD_decompressSequences variants!" +#endif + + +/*_******************************************************* +* Memory operations +**********************************************************/ +static void ZSTD_copy4(void* dst, const void* src) { ZSTD_memcpy(dst, src, 4); } + + +/*-************************************************************* + * Block decoding + ***************************************************************/ + +static size_t ZSTD_blockSizeMax(ZSTD_DCtx const* dctx) +{ + size_t const blockSizeMax = dctx->isFrameDecompression ? dctx->fParams.blockSizeMax : ZSTD_BLOCKSIZE_MAX; + assert(blockSizeMax <= ZSTD_BLOCKSIZE_MAX); + return blockSizeMax; +} + +/*! ZSTD_getcBlockSize() : + * Provides the size of compressed block from block header `src` */ +size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, + blockProperties_t* bpPtr) +{ + RETURN_ERROR_IF(srcSize < ZSTD_blockHeaderSize, srcSize_wrong, ""); + + { U32 const cBlockHeader = MEM_readLE24(src); + U32 const cSize = cBlockHeader >> 3; + bpPtr->lastBlock = cBlockHeader & 1; + bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3); + bpPtr->origSize = cSize; /* only useful for RLE */ + if (bpPtr->blockType == bt_rle) return 1; + RETURN_ERROR_IF(bpPtr->blockType == bt_reserved, corruption_detected, ""); + return cSize; + } +} + +/* Allocate buffer for literals, either overlapping current dst, or split between dst and litExtraBuffer, or stored entirely within litExtraBuffer */ +static void ZSTD_allocateLiteralsBuffer(ZSTD_DCtx* dctx, void* const dst, const size_t dstCapacity, const size_t litSize, + const streaming_operation streaming, const size_t expectedWriteSize, const unsigned splitImmediately) +{ + size_t const blockSizeMax = ZSTD_blockSizeMax(dctx); + assert(litSize <= blockSizeMax); + assert(dctx->isFrameDecompression || streaming == not_streaming); + assert(expectedWriteSize <= blockSizeMax); + if (streaming == not_streaming && dstCapacity > blockSizeMax + WILDCOPY_OVERLENGTH + litSize + WILDCOPY_OVERLENGTH) { + /* If we aren't streaming, we can just put the literals after the output + * of the current block. We don't need to worry about overwriting the + * extDict of our window, because it doesn't exist. + * So if we have space after the end of the block, just put it there. + */ + dctx->litBuffer = (BYTE*)dst + blockSizeMax + WILDCOPY_OVERLENGTH; + dctx->litBufferEnd = dctx->litBuffer + litSize; + dctx->litBufferLocation = ZSTD_in_dst; + } else if (litSize <= ZSTD_LITBUFFEREXTRASIZE) { + /* Literals fit entirely within the extra buffer, put them there to avoid + * having to split the literals. + */ + dctx->litBuffer = dctx->litExtraBuffer; + dctx->litBufferEnd = dctx->litBuffer + litSize; + dctx->litBufferLocation = ZSTD_not_in_dst; + } else { + assert(blockSizeMax > ZSTD_LITBUFFEREXTRASIZE); + /* Literals must be split between the output block and the extra lit + * buffer. We fill the extra lit buffer with the tail of the literals, + * and put the rest of the literals at the end of the block, with + * WILDCOPY_OVERLENGTH of buffer room to allow for overreads. + * This MUST not write more than our maxBlockSize beyond dst, because in + * streaming mode, that could overwrite part of our extDict window. + */ + if (splitImmediately) { + /* won't fit in litExtraBuffer, so it will be split between end of dst and extra buffer */ + dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH; + dctx->litBufferEnd = dctx->litBuffer + litSize - ZSTD_LITBUFFEREXTRASIZE; + } else { + /* initially this will be stored entirely in dst during huffman decoding, it will partially be shifted to litExtraBuffer after */ + dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize; + dctx->litBufferEnd = (BYTE*)dst + expectedWriteSize; + } + dctx->litBufferLocation = ZSTD_split; + assert(dctx->litBufferEnd <= (BYTE*)dst + expectedWriteSize); + } +} + +/*! ZSTD_decodeLiteralsBlock() : + * Where it is possible to do so without being stomped by the output during decompression, the literals block will be stored + * in the dstBuffer. If there is room to do so, it will be stored in full in the excess dst space after where the current + * block will be output. Otherwise it will be stored at the end of the current dst blockspace, with a small portion being + * stored in dctx->litExtraBuffer to help keep it "ahead" of the current output write. + * + * @return : nb of bytes read from src (< srcSize ) + * note : symbol not declared but exposed for fullbench */ +static size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, + const void* src, size_t srcSize, /* note : srcSize < BLOCKSIZE */ + void* dst, size_t dstCapacity, const streaming_operation streaming) +{ + DEBUGLOG(5, "ZSTD_decodeLiteralsBlock"); + RETURN_ERROR_IF(srcSize < MIN_CBLOCK_SIZE, corruption_detected, ""); + + { const BYTE* const istart = (const BYTE*) src; + symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3); + size_t const blockSizeMax = ZSTD_blockSizeMax(dctx); + + switch(litEncType) + { + case set_repeat: + DEBUGLOG(5, "set_repeat flag : re-using stats from previous compressed literals block"); + RETURN_ERROR_IF(dctx->litEntropy==0, dictionary_corrupted, ""); + ZSTD_FALLTHROUGH; + + case set_compressed: + RETURN_ERROR_IF(srcSize < 5, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need up to 5 for case 3"); + { size_t lhSize, litSize, litCSize; + U32 singleStream=0; + U32 const lhlCode = (istart[0] >> 2) & 3; + U32 const lhc = MEM_readLE32(istart); + size_t hufSuccess; + size_t expectedWriteSize = MIN(blockSizeMax, dstCapacity); + int const flags = 0 + | (ZSTD_DCtx_get_bmi2(dctx) ? HUF_flags_bmi2 : 0) + | (dctx->disableHufAsm ? HUF_flags_disableAsm : 0); + switch(lhlCode) + { + case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */ + /* 2 - 2 - 10 - 10 */ + singleStream = !lhlCode; + lhSize = 3; + litSize = (lhc >> 4) & 0x3FF; + litCSize = (lhc >> 14) & 0x3FF; + break; + case 2: + /* 2 - 2 - 14 - 14 */ + lhSize = 4; + litSize = (lhc >> 4) & 0x3FFF; + litCSize = lhc >> 18; + break; + case 3: + /* 2 - 2 - 18 - 18 */ + lhSize = 5; + litSize = (lhc >> 4) & 0x3FFFF; + litCSize = (lhc >> 22) + ((size_t)istart[4] << 10); + break; + } + RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(litSize > blockSizeMax, corruption_detected, ""); + if (!singleStream) + RETURN_ERROR_IF(litSize < MIN_LITERALS_FOR_4_STREAMS, literals_headerWrong, + "Not enough literals (%zu) for the 4-streams mode (min %u)", + litSize, MIN_LITERALS_FOR_4_STREAMS); + RETURN_ERROR_IF(litCSize + lhSize > srcSize, corruption_detected, ""); + RETURN_ERROR_IF(expectedWriteSize < litSize , dstSize_tooSmall, ""); + ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 0); + + /* prefetch huffman table if cold */ + if (dctx->ddictIsCold && (litSize > 768 /* heuristic */)) { + PREFETCH_AREA(dctx->HUFptr, sizeof(dctx->entropy.hufTable)); + } + + if (litEncType==set_repeat) { + if (singleStream) { + hufSuccess = HUF_decompress1X_usingDTable( + dctx->litBuffer, litSize, istart+lhSize, litCSize, + dctx->HUFptr, flags); + } else { + assert(litSize >= MIN_LITERALS_FOR_4_STREAMS); + hufSuccess = HUF_decompress4X_usingDTable( + dctx->litBuffer, litSize, istart+lhSize, litCSize, + dctx->HUFptr, flags); + } + } else { + if (singleStream) { +#if defined(HUF_FORCE_DECOMPRESS_X2) + hufSuccess = HUF_decompress1X_DCtx_wksp( + dctx->entropy.hufTable, dctx->litBuffer, litSize, + istart+lhSize, litCSize, dctx->workspace, + sizeof(dctx->workspace), flags); +#else + hufSuccess = HUF_decompress1X1_DCtx_wksp( + dctx->entropy.hufTable, dctx->litBuffer, litSize, + istart+lhSize, litCSize, dctx->workspace, + sizeof(dctx->workspace), flags); +#endif + } else { + hufSuccess = HUF_decompress4X_hufOnly_wksp( + dctx->entropy.hufTable, dctx->litBuffer, litSize, + istart+lhSize, litCSize, dctx->workspace, + sizeof(dctx->workspace), flags); + } + } + if (dctx->litBufferLocation == ZSTD_split) + { + assert(litSize > ZSTD_LITBUFFEREXTRASIZE); + ZSTD_memcpy(dctx->litExtraBuffer, dctx->litBufferEnd - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTRASIZE); + ZSTD_memmove(dctx->litBuffer + ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH, dctx->litBuffer, litSize - ZSTD_LITBUFFEREXTRASIZE); + dctx->litBuffer += ZSTD_LITBUFFEREXTRASIZE - WILDCOPY_OVERLENGTH; + dctx->litBufferEnd -= WILDCOPY_OVERLENGTH; + assert(dctx->litBufferEnd <= (BYTE*)dst + blockSizeMax); + } + + RETURN_ERROR_IF(HUF_isError(hufSuccess), corruption_detected, ""); + + dctx->litPtr = dctx->litBuffer; + dctx->litSize = litSize; + dctx->litEntropy = 1; + if (litEncType==set_compressed) dctx->HUFptr = dctx->entropy.hufTable; + return litCSize + lhSize; + } + + case set_basic: + { size_t litSize, lhSize; + U32 const lhlCode = ((istart[0]) >> 2) & 3; + size_t expectedWriteSize = MIN(blockSizeMax, dstCapacity); + switch(lhlCode) + { + case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ + lhSize = 1; + litSize = istart[0] >> 3; + break; + case 1: + lhSize = 2; + litSize = MEM_readLE16(istart) >> 4; + break; + case 3: + lhSize = 3; + RETURN_ERROR_IF(srcSize<3, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize = 3"); + litSize = MEM_readLE24(istart) >> 4; + break; + } + + RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(litSize > blockSizeMax, corruption_detected, ""); + RETURN_ERROR_IF(expectedWriteSize < litSize, dstSize_tooSmall, ""); + ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 1); + if (lhSize+litSize+WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */ + RETURN_ERROR_IF(litSize+lhSize > srcSize, corruption_detected, ""); + if (dctx->litBufferLocation == ZSTD_split) + { + ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize - ZSTD_LITBUFFEREXTRASIZE); + ZSTD_memcpy(dctx->litExtraBuffer, istart + lhSize + litSize - ZSTD_LITBUFFEREXTRASIZE, ZSTD_LITBUFFEREXTRASIZE); + } + else + { + ZSTD_memcpy(dctx->litBuffer, istart + lhSize, litSize); + } + dctx->litPtr = dctx->litBuffer; + dctx->litSize = litSize; + return lhSize+litSize; + } + /* direct reference into compressed stream */ + dctx->litPtr = istart+lhSize; + dctx->litSize = litSize; + dctx->litBufferEnd = dctx->litPtr + litSize; + dctx->litBufferLocation = ZSTD_not_in_dst; + return lhSize+litSize; + } + + case set_rle: + { U32 const lhlCode = ((istart[0]) >> 2) & 3; + size_t litSize, lhSize; + size_t expectedWriteSize = MIN(blockSizeMax, dstCapacity); + switch(lhlCode) + { + case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ + lhSize = 1; + litSize = istart[0] >> 3; + break; + case 1: + lhSize = 2; + RETURN_ERROR_IF(srcSize<3, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize+1 = 3"); + litSize = MEM_readLE16(istart) >> 4; + break; + case 3: + lhSize = 3; + RETURN_ERROR_IF(srcSize<4, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize+1 = 4"); + litSize = MEM_readLE24(istart) >> 4; + break; + } + RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(litSize > blockSizeMax, corruption_detected, ""); + RETURN_ERROR_IF(expectedWriteSize < litSize, dstSize_tooSmall, ""); + ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 1); + if (dctx->litBufferLocation == ZSTD_split) + { + ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize - ZSTD_LITBUFFEREXTRASIZE); + ZSTD_memset(dctx->litExtraBuffer, istart[lhSize], ZSTD_LITBUFFEREXTRASIZE); + } + else + { + ZSTD_memset(dctx->litBuffer, istart[lhSize], litSize); + } + dctx->litPtr = dctx->litBuffer; + dctx->litSize = litSize; + return lhSize+1; + } + default: + RETURN_ERROR(corruption_detected, "impossible"); + } + } +} + +/* Hidden declaration for fullbench */ +size_t ZSTD_decodeLiteralsBlock_wrapper(ZSTD_DCtx* dctx, + const void* src, size_t srcSize, + void* dst, size_t dstCapacity); +size_t ZSTD_decodeLiteralsBlock_wrapper(ZSTD_DCtx* dctx, + const void* src, size_t srcSize, + void* dst, size_t dstCapacity) +{ + dctx->isFrameDecompression = 0; + return ZSTD_decodeLiteralsBlock(dctx, src, srcSize, dst, dstCapacity, not_streaming); +} + +/* Default FSE distribution tables. + * These are pre-calculated FSE decoding tables using default distributions as defined in specification : + * https://github.com/facebook/zstd/blob/release/doc/zstd_compression_format.md#default-distributions + * They were generated programmatically with following method : + * - start from default distributions, present in /lib/common/zstd_internal.h + * - generate tables normally, using ZSTD_buildFSETable() + * - printout the content of tables + * - pretify output, report below, test with fuzzer to ensure it's correct */ + +/* Default FSE distribution table for Literal Lengths */ +static const ZSTD_seqSymbol LL_defaultDTable[(1<tableLog = 0; + DTableH->fastMode = 0; + + cell->nbBits = 0; + cell->nextState = 0; + assert(nbAddBits < 255); + cell->nbAdditionalBits = nbAddBits; + cell->baseValue = baseValue; +} + + +/* ZSTD_buildFSETable() : + * generate FSE decoding table for one symbol (ll, ml or off) + * cannot fail if input is valid => + * all inputs are presumed validated at this stage */ +FORCE_INLINE_TEMPLATE +void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt, + const short* normalizedCounter, unsigned maxSymbolValue, + const U32* baseValue, const U8* nbAdditionalBits, + unsigned tableLog, void* wksp, size_t wkspSize) +{ + ZSTD_seqSymbol* const tableDecode = dt+1; + U32 const maxSV1 = maxSymbolValue + 1; + U32 const tableSize = 1 << tableLog; + + U16* symbolNext = (U16*)wksp; + BYTE* spread = (BYTE*)(symbolNext + MaxSeq + 1); + U32 highThreshold = tableSize - 1; + + + /* Sanity Checks */ + assert(maxSymbolValue <= MaxSeq); + assert(tableLog <= MaxFSELog); + assert(wkspSize >= ZSTD_BUILD_FSE_TABLE_WKSP_SIZE); + (void)wkspSize; + /* Init, lay down lowprob symbols */ + { ZSTD_seqSymbol_header DTableH; + DTableH.tableLog = tableLog; + DTableH.fastMode = 1; + { S16 const largeLimit= (S16)(1 << (tableLog-1)); + U32 s; + for (s=0; s= largeLimit) DTableH.fastMode=0; + assert(normalizedCounter[s]>=0); + symbolNext[s] = (U16)normalizedCounter[s]; + } } } + ZSTD_memcpy(dt, &DTableH, sizeof(DTableH)); + } + + /* Spread symbols */ + assert(tableSize <= 512); + /* Specialized symbol spreading for the case when there are + * no low probability (-1 count) symbols. When compressing + * small blocks we avoid low probability symbols to hit this + * case, since header decoding speed matters more. + */ + if (highThreshold == tableSize - 1) { + size_t const tableMask = tableSize-1; + size_t const step = FSE_TABLESTEP(tableSize); + /* First lay down the symbols in order. + * We use a uint64_t to lay down 8 bytes at a time. This reduces branch + * misses since small blocks generally have small table logs, so nearly + * all symbols have counts <= 8. We ensure we have 8 bytes at the end of + * our buffer to handle the over-write. + */ + { + U64 const add = 0x0101010101010101ull; + size_t pos = 0; + U64 sv = 0; + U32 s; + for (s=0; s=0); + pos += (size_t)n; + } + } + /* Now we spread those positions across the table. + * The benefit of doing it in two stages is that we avoid the + * variable size inner loop, which caused lots of branch misses. + * Now we can run through all the positions without any branch misses. + * We unroll the loop twice, since that is what empirically worked best. + */ + { + size_t position = 0; + size_t s; + size_t const unroll = 2; + assert(tableSize % unroll == 0); /* FSE_MIN_TABLELOG is 5 */ + for (s = 0; s < (size_t)tableSize; s += unroll) { + size_t u; + for (u = 0; u < unroll; ++u) { + size_t const uPosition = (position + (u * step)) & tableMask; + tableDecode[uPosition].baseValue = spread[s + u]; + } + position = (position + (unroll * step)) & tableMask; + } + assert(position == 0); + } + } else { + U32 const tableMask = tableSize-1; + U32 const step = FSE_TABLESTEP(tableSize); + U32 s, position = 0; + for (s=0; s highThreshold)) position = (position + step) & tableMask; /* lowprob area */ + } } + assert(position == 0); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ + } + + /* Build Decoding table */ + { + U32 u; + for (u=0; u max, corruption_detected, ""); + { U32 const symbol = *(const BYTE*)src; + U32 const baseline = baseValue[symbol]; + U8 const nbBits = nbAdditionalBits[symbol]; + ZSTD_buildSeqTable_rle(DTableSpace, baseline, nbBits); + } + *DTablePtr = DTableSpace; + return 1; + case set_basic : + *DTablePtr = defaultTable; + return 0; + case set_repeat: + RETURN_ERROR_IF(!flagRepeatTable, corruption_detected, ""); + /* prefetch FSE table if used */ + if (ddictIsCold && (nbSeq > 24 /* heuristic */)) { + const void* const pStart = *DTablePtr; + size_t const pSize = sizeof(ZSTD_seqSymbol) * (SEQSYMBOL_TABLE_SIZE(maxLog)); + PREFETCH_AREA(pStart, pSize); + } + return 0; + case set_compressed : + { unsigned tableLog; + S16 norm[MaxSeq+1]; + size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize); + RETURN_ERROR_IF(FSE_isError(headerSize), corruption_detected, ""); + RETURN_ERROR_IF(tableLog > maxLog, corruption_detected, ""); + ZSTD_buildFSETable(DTableSpace, norm, max, baseValue, nbAdditionalBits, tableLog, wksp, wkspSize, bmi2); + *DTablePtr = DTableSpace; + return headerSize; + } + default : + assert(0); + RETURN_ERROR(GENERIC, "impossible"); + } +} + +size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, + const void* src, size_t srcSize) +{ + const BYTE* const istart = (const BYTE*)src; + const BYTE* const iend = istart + srcSize; + const BYTE* ip = istart; + int nbSeq; + DEBUGLOG(5, "ZSTD_decodeSeqHeaders"); + + /* check */ + RETURN_ERROR_IF(srcSize < MIN_SEQUENCES_SIZE, srcSize_wrong, ""); + + /* SeqHead */ + nbSeq = *ip++; + if (nbSeq > 0x7F) { + if (nbSeq == 0xFF) { + RETURN_ERROR_IF(ip+2 > iend, srcSize_wrong, ""); + nbSeq = MEM_readLE16(ip) + LONGNBSEQ; + ip+=2; + } else { + RETURN_ERROR_IF(ip >= iend, srcSize_wrong, ""); + nbSeq = ((nbSeq-0x80)<<8) + *ip++; + } + } + *nbSeqPtr = nbSeq; + + if (nbSeq == 0) { + /* No sequence : section ends immediately */ + RETURN_ERROR_IF(ip != iend, corruption_detected, + "extraneous data present in the Sequences section"); + return (size_t)(ip - istart); + } + + /* FSE table descriptors */ + RETURN_ERROR_IF(ip+1 > iend, srcSize_wrong, ""); /* minimum possible size: 1 byte for symbol encoding types */ + RETURN_ERROR_IF(*ip & 3, corruption_detected, ""); /* The last field, Reserved, must be all-zeroes. */ + { symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6); + symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3); + symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3); + ip++; + + /* Build DTables */ + { size_t const llhSize = ZSTD_buildSeqTable(dctx->entropy.LLTable, &dctx->LLTptr, + LLtype, MaxLL, LLFSELog, + ip, iend-ip, + LL_base, LL_bits, + LL_defaultDTable, dctx->fseEntropy, + dctx->ddictIsCold, nbSeq, + dctx->workspace, sizeof(dctx->workspace), + ZSTD_DCtx_get_bmi2(dctx)); + RETURN_ERROR_IF(ZSTD_isError(llhSize), corruption_detected, "ZSTD_buildSeqTable failed"); + ip += llhSize; + } + + { size_t const ofhSize = ZSTD_buildSeqTable(dctx->entropy.OFTable, &dctx->OFTptr, + OFtype, MaxOff, OffFSELog, + ip, iend-ip, + OF_base, OF_bits, + OF_defaultDTable, dctx->fseEntropy, + dctx->ddictIsCold, nbSeq, + dctx->workspace, sizeof(dctx->workspace), + ZSTD_DCtx_get_bmi2(dctx)); + RETURN_ERROR_IF(ZSTD_isError(ofhSize), corruption_detected, "ZSTD_buildSeqTable failed"); + ip += ofhSize; + } + + { size_t const mlhSize = ZSTD_buildSeqTable(dctx->entropy.MLTable, &dctx->MLTptr, + MLtype, MaxML, MLFSELog, + ip, iend-ip, + ML_base, ML_bits, + ML_defaultDTable, dctx->fseEntropy, + dctx->ddictIsCold, nbSeq, + dctx->workspace, sizeof(dctx->workspace), + ZSTD_DCtx_get_bmi2(dctx)); + RETURN_ERROR_IF(ZSTD_isError(mlhSize), corruption_detected, "ZSTD_buildSeqTable failed"); + ip += mlhSize; + } + } + + return ip-istart; +} + + +typedef struct { + size_t litLength; + size_t matchLength; + size_t offset; +} seq_t; + +typedef struct { + size_t state; + const ZSTD_seqSymbol* table; +} ZSTD_fseState; + +typedef struct { + BIT_DStream_t DStream; + ZSTD_fseState stateLL; + ZSTD_fseState stateOffb; + ZSTD_fseState stateML; + size_t prevOffset[ZSTD_REP_NUM]; +} seqState_t; + +/*! ZSTD_overlapCopy8() : + * Copies 8 bytes from ip to op and updates op and ip where ip <= op. + * If the offset is < 8 then the offset is spread to at least 8 bytes. + * + * Precondition: *ip <= *op + * Postcondition: *op - *op >= 8 + */ +HINT_INLINE void ZSTD_overlapCopy8(BYTE** op, BYTE const** ip, size_t offset) { + assert(*ip <= *op); + if (offset < 8) { + /* close range match, overlap */ + static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */ + static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* subtracted */ + int const sub2 = dec64table[offset]; + (*op)[0] = (*ip)[0]; + (*op)[1] = (*ip)[1]; + (*op)[2] = (*ip)[2]; + (*op)[3] = (*ip)[3]; + *ip += dec32table[offset]; + ZSTD_copy4(*op+4, *ip); + *ip -= sub2; + } else { + ZSTD_copy8(*op, *ip); + } + *ip += 8; + *op += 8; + assert(*op - *ip >= 8); +} + +/*! ZSTD_safecopy() : + * Specialized version of memcpy() that is allowed to READ up to WILDCOPY_OVERLENGTH past the input buffer + * and write up to 16 bytes past oend_w (op >= oend_w is allowed). + * This function is only called in the uncommon case where the sequence is near the end of the block. It + * should be fast for a single long sequence, but can be slow for several short sequences. + * + * @param ovtype controls the overlap detection + * - ZSTD_no_overlap: The source and destination are guaranteed to be at least WILDCOPY_VECLEN bytes apart. + * - ZSTD_overlap_src_before_dst: The src and dst may overlap and may be any distance apart. + * The src buffer must be before the dst buffer. + */ +static void ZSTD_safecopy(BYTE* op, const BYTE* const oend_w, BYTE const* ip, ptrdiff_t length, ZSTD_overlap_e ovtype) { + ptrdiff_t const diff = op - ip; + BYTE* const oend = op + length; + + assert((ovtype == ZSTD_no_overlap && (diff <= -8 || diff >= 8 || op >= oend_w)) || + (ovtype == ZSTD_overlap_src_before_dst && diff >= 0)); + + if (length < 8) { + /* Handle short lengths. */ + while (op < oend) *op++ = *ip++; + return; + } + if (ovtype == ZSTD_overlap_src_before_dst) { + /* Copy 8 bytes and ensure the offset >= 8 when there can be overlap. */ + assert(length >= 8); + ZSTD_overlapCopy8(&op, &ip, diff); + length -= 8; + assert(op - ip >= 8); + assert(op <= oend); + } + + if (oend <= oend_w) { + /* No risk of overwrite. */ + ZSTD_wildcopy(op, ip, length, ovtype); + return; + } + if (op <= oend_w) { + /* Wildcopy until we get close to the end. */ + assert(oend > oend_w); + ZSTD_wildcopy(op, ip, oend_w - op, ovtype); + ip += oend_w - op; + op += oend_w - op; + } + /* Handle the leftovers. */ + while (op < oend) *op++ = *ip++; +} + +/* ZSTD_safecopyDstBeforeSrc(): + * This version allows overlap with dst before src, or handles the non-overlap case with dst after src + * Kept separate from more common ZSTD_safecopy case to avoid performance impact to the safecopy common case */ +static void ZSTD_safecopyDstBeforeSrc(BYTE* op, const BYTE* ip, ptrdiff_t length) { + ptrdiff_t const diff = op - ip; + BYTE* const oend = op + length; + + if (length < 8 || diff > -8) { + /* Handle short lengths, close overlaps, and dst not before src. */ + while (op < oend) *op++ = *ip++; + return; + } + + if (op <= oend - WILDCOPY_OVERLENGTH && diff < -WILDCOPY_VECLEN) { + ZSTD_wildcopy(op, ip, oend - WILDCOPY_OVERLENGTH - op, ZSTD_no_overlap); + ip += oend - WILDCOPY_OVERLENGTH - op; + op += oend - WILDCOPY_OVERLENGTH - op; + } + + /* Handle the leftovers. */ + while (op < oend) *op++ = *ip++; +} + +/* ZSTD_execSequenceEnd(): + * This version handles cases that are near the end of the output buffer. It requires + * more careful checks to make sure there is no overflow. By separating out these hard + * and unlikely cases, we can speed up the common cases. + * + * NOTE: This function needs to be fast for a single long sequence, but doesn't need + * to be optimized for many small sequences, since those fall into ZSTD_execSequence(). + */ +FORCE_NOINLINE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_execSequenceEnd(BYTE* op, + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; + + /* bounds checks : careful of address space overflow in 32-bit mode */ + RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit within dstBuffer"); + RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to read beyond literal buffer"); + assert(op < op + sequenceLength); + assert(oLitEnd < op + sequenceLength); + + /* copy literals */ + ZSTD_safecopy(op, oend_w, *litPtr, sequence.litLength, ZSTD_no_overlap); + op = oLitEnd; + *litPtr = iLitEnd; + + /* copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix */ + RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, ""); + match = dictEnd - (prefixStart - match); + if (match + sequence.matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } + } + ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst); + return sequenceLength; +} + +/* ZSTD_execSequenceEndSplitLitBuffer(): + * This version is intended to be used during instances where the litBuffer is still split. It is kept separate to avoid performance impact for the good case. + */ +FORCE_NOINLINE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_execSequenceEndSplitLitBuffer(BYTE* op, + BYTE* const oend, const BYTE* const oend_w, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + + /* bounds checks : careful of address space overflow in 32-bit mode */ + RETURN_ERROR_IF(sequenceLength > (size_t)(oend - op), dstSize_tooSmall, "last match must fit within dstBuffer"); + RETURN_ERROR_IF(sequence.litLength > (size_t)(litLimit - *litPtr), corruption_detected, "try to read beyond literal buffer"); + assert(op < op + sequenceLength); + assert(oLitEnd < op + sequenceLength); + + /* copy literals */ + RETURN_ERROR_IF(op > *litPtr && op < *litPtr + sequence.litLength, dstSize_tooSmall, "output should not catch up to and overwrite literal buffer"); + ZSTD_safecopyDstBeforeSrc(op, *litPtr, sequence.litLength); + op = oLitEnd; + *litPtr = iLitEnd; + + /* copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix */ + RETURN_ERROR_IF(sequence.offset > (size_t)(oLitEnd - virtualStart), corruption_detected, ""); + match = dictEnd - (prefixStart - match); + if (match + sequence.matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } + } + ZSTD_safecopy(op, oend_w, match, sequence.matchLength, ZSTD_overlap_src_before_dst); + return sequenceLength; +} + +HINT_INLINE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_execSequence(BYTE* op, + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ + BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; /* risk : address space underflow on oend=NULL */ + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + assert(op != NULL /* Precondition */); + assert(oend_w < oend /* No underflow */); + +#if defined(__aarch64__) + /* prefetch sequence starting from match that will be used for copy later */ + PREFETCH_L1(match); +#endif + /* Handle edge cases in a slow path: + * - Read beyond end of literals + * - Match end is within WILDCOPY_OVERLIMIT of oend + * - 32-bit mode and the match length overflows + */ + if (UNLIKELY( + iLitEnd > litLimit || + oMatchEnd > oend_w || + (MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH))) + return ZSTD_execSequenceEnd(op, oend, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); + + /* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */ + assert(op <= oLitEnd /* No overflow */); + assert(oLitEnd < oMatchEnd /* Non-zero match & no overflow */); + assert(oMatchEnd <= oend /* No underflow */); + assert(iLitEnd <= litLimit /* Literal length is in bounds */); + assert(oLitEnd <= oend_w /* Can wildcopy literals */); + assert(oMatchEnd <= oend_w /* Can wildcopy matches */); + + /* Copy Literals: + * Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9. + * We likely don't need the full 32-byte wildcopy. + */ + assert(WILDCOPY_OVERLENGTH >= 16); + ZSTD_copy16(op, (*litPtr)); + if (UNLIKELY(sequence.litLength > 16)) { + ZSTD_wildcopy(op + 16, (*litPtr) + 16, sequence.litLength - 16, ZSTD_no_overlap); + } + op = oLitEnd; + *litPtr = iLitEnd; /* update for next sequence */ + + /* Copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix -> go into extDict */ + RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, ""); + match = dictEnd + (match - prefixStart); + if (match + sequence.matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } + } + /* Match within prefix of 1 or more bytes */ + assert(op <= oMatchEnd); + assert(oMatchEnd <= oend_w); + assert(match >= prefixStart); + assert(sequence.matchLength >= 1); + + /* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy + * without overlap checking. + */ + if (LIKELY(sequence.offset >= WILDCOPY_VECLEN)) { + /* We bet on a full wildcopy for matches, since we expect matches to be + * longer than literals (in general). In silesia, ~10% of matches are longer + * than 16 bytes. + */ + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap); + return sequenceLength; + } + assert(sequence.offset < WILDCOPY_VECLEN); + + /* Copy 8 bytes and spread the offset to be >= 8. */ + ZSTD_overlapCopy8(&op, &match, sequence.offset); + + /* If the match length is > 8 bytes, then continue with the wildcopy. */ + if (sequence.matchLength > 8) { + assert(op < oMatchEnd); + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength - 8, ZSTD_overlap_src_before_dst); + } + return sequenceLength; +} + +HINT_INLINE +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +size_t ZSTD_execSequenceSplitLitBuffer(BYTE* op, + BYTE* const oend, const BYTE* const oend_w, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const prefixStart, const BYTE* const virtualStart, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + assert(op != NULL /* Precondition */); + assert(oend_w < oend /* No underflow */); + /* Handle edge cases in a slow path: + * - Read beyond end of literals + * - Match end is within WILDCOPY_OVERLIMIT of oend + * - 32-bit mode and the match length overflows + */ + if (UNLIKELY( + iLitEnd > litLimit || + oMatchEnd > oend_w || + (MEM_32bits() && (size_t)(oend - op) < sequenceLength + WILDCOPY_OVERLENGTH))) + return ZSTD_execSequenceEndSplitLitBuffer(op, oend, oend_w, sequence, litPtr, litLimit, prefixStart, virtualStart, dictEnd); + + /* Assumptions (everything else goes into ZSTD_execSequenceEnd()) */ + assert(op <= oLitEnd /* No overflow */); + assert(oLitEnd < oMatchEnd /* Non-zero match & no overflow */); + assert(oMatchEnd <= oend /* No underflow */); + assert(iLitEnd <= litLimit /* Literal length is in bounds */); + assert(oLitEnd <= oend_w /* Can wildcopy literals */); + assert(oMatchEnd <= oend_w /* Can wildcopy matches */); + + /* Copy Literals: + * Split out litLength <= 16 since it is nearly always true. +1.6% on gcc-9. + * We likely don't need the full 32-byte wildcopy. + */ + assert(WILDCOPY_OVERLENGTH >= 16); + ZSTD_copy16(op, (*litPtr)); + if (UNLIKELY(sequence.litLength > 16)) { + ZSTD_wildcopy(op+16, (*litPtr)+16, sequence.litLength-16, ZSTD_no_overlap); + } + op = oLitEnd; + *litPtr = iLitEnd; /* update for next sequence */ + + /* Copy Match */ + if (sequence.offset > (size_t)(oLitEnd - prefixStart)) { + /* offset beyond prefix -> go into extDict */ + RETURN_ERROR_IF(UNLIKELY(sequence.offset > (size_t)(oLitEnd - virtualStart)), corruption_detected, ""); + match = dictEnd + (match - prefixStart); + if (match + sequence.matchLength <= dictEnd) { + ZSTD_memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + ZSTD_memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = prefixStart; + } } + /* Match within prefix of 1 or more bytes */ + assert(op <= oMatchEnd); + assert(oMatchEnd <= oend_w); + assert(match >= prefixStart); + assert(sequence.matchLength >= 1); + + /* Nearly all offsets are >= WILDCOPY_VECLEN bytes, which means we can use wildcopy + * without overlap checking. + */ + if (LIKELY(sequence.offset >= WILDCOPY_VECLEN)) { + /* We bet on a full wildcopy for matches, since we expect matches to be + * longer than literals (in general). In silesia, ~10% of matches are longer + * than 16 bytes. + */ + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength, ZSTD_no_overlap); + return sequenceLength; + } + assert(sequence.offset < WILDCOPY_VECLEN); + + /* Copy 8 bytes and spread the offset to be >= 8. */ + ZSTD_overlapCopy8(&op, &match, sequence.offset); + + /* If the match length is > 8 bytes, then continue with the wildcopy. */ + if (sequence.matchLength > 8) { + assert(op < oMatchEnd); + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8, ZSTD_overlap_src_before_dst); + } + return sequenceLength; +} + + +static void +ZSTD_initFseState(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, const ZSTD_seqSymbol* dt) +{ + const void* ptr = dt; + const ZSTD_seqSymbol_header* const DTableH = (const ZSTD_seqSymbol_header*)ptr; + DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); + DEBUGLOG(6, "ZSTD_initFseState : val=%u using %u bits", + (U32)DStatePtr->state, DTableH->tableLog); + BIT_reloadDStream(bitD); + DStatePtr->table = dt + 1; +} + +FORCE_INLINE_TEMPLATE void +ZSTD_updateFseStateWithDInfo(ZSTD_fseState* DStatePtr, BIT_DStream_t* bitD, U16 nextState, U32 nbBits) +{ + size_t const lowBits = BIT_readBits(bitD, nbBits); + DStatePtr->state = nextState + lowBits; +} + +/* We need to add at most (ZSTD_WINDOWLOG_MAX_32 - 1) bits to read the maximum + * offset bits. But we can only read at most STREAM_ACCUMULATOR_MIN_32 + * bits before reloading. This value is the maximum number of bytes we read + * after reloading when we are decoding long offsets. + */ +#define LONG_OFFSETS_MAX_EXTRA_BITS_32 \ + (ZSTD_WINDOWLOG_MAX_32 > STREAM_ACCUMULATOR_MIN_32 \ + ? ZSTD_WINDOWLOG_MAX_32 - STREAM_ACCUMULATOR_MIN_32 \ + : 0) + +typedef enum { ZSTD_lo_isRegularOffset, ZSTD_lo_isLongOffset=1 } ZSTD_longOffset_e; + +/** + * ZSTD_decodeSequence(): + * @p longOffsets : tells the decoder to reload more bit while decoding large offsets + * only used in 32-bit mode + * @return : Sequence (litL + matchL + offset) + */ +FORCE_INLINE_TEMPLATE seq_t +ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets, const int isLastSeq) +{ + seq_t seq; + /* + * ZSTD_seqSymbol is a 64 bits wide structure. + * It can be loaded in one operation + * and its fields extracted by simply shifting or bit-extracting on aarch64. + * GCC doesn't recognize this and generates more unnecessary ldr/ldrb/ldrh + * operations that cause performance drop. This can be avoided by using this + * ZSTD_memcpy hack. + */ +#if defined(__aarch64__) && (defined(__GNUC__) && !defined(__clang__)) + ZSTD_seqSymbol llDInfoS, mlDInfoS, ofDInfoS; + ZSTD_seqSymbol* const llDInfo = &llDInfoS; + ZSTD_seqSymbol* const mlDInfo = &mlDInfoS; + ZSTD_seqSymbol* const ofDInfo = &ofDInfoS; + ZSTD_memcpy(llDInfo, seqState->stateLL.table + seqState->stateLL.state, sizeof(ZSTD_seqSymbol)); + ZSTD_memcpy(mlDInfo, seqState->stateML.table + seqState->stateML.state, sizeof(ZSTD_seqSymbol)); + ZSTD_memcpy(ofDInfo, seqState->stateOffb.table + seqState->stateOffb.state, sizeof(ZSTD_seqSymbol)); +#else + const ZSTD_seqSymbol* const llDInfo = seqState->stateLL.table + seqState->stateLL.state; + const ZSTD_seqSymbol* const mlDInfo = seqState->stateML.table + seqState->stateML.state; + const ZSTD_seqSymbol* const ofDInfo = seqState->stateOffb.table + seqState->stateOffb.state; +#endif + seq.matchLength = mlDInfo->baseValue; + seq.litLength = llDInfo->baseValue; + { U32 const ofBase = ofDInfo->baseValue; + BYTE const llBits = llDInfo->nbAdditionalBits; + BYTE const mlBits = mlDInfo->nbAdditionalBits; + BYTE const ofBits = ofDInfo->nbAdditionalBits; + BYTE const totalBits = llBits+mlBits+ofBits; + + U16 const llNext = llDInfo->nextState; + U16 const mlNext = mlDInfo->nextState; + U16 const ofNext = ofDInfo->nextState; + U32 const llnbBits = llDInfo->nbBits; + U32 const mlnbBits = mlDInfo->nbBits; + U32 const ofnbBits = ofDInfo->nbBits; + + assert(llBits <= MaxLLBits); + assert(mlBits <= MaxMLBits); + assert(ofBits <= MaxOff); + /* + * As gcc has better branch and block analyzers, sometimes it is only + * valuable to mark likeliness for clang, it gives around 3-4% of + * performance. + */ + + /* sequence */ + { size_t offset; + if (ofBits > 1) { + ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1); + ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5); + ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 > LONG_OFFSETS_MAX_EXTRA_BITS_32); + ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 - LONG_OFFSETS_MAX_EXTRA_BITS_32 >= MaxMLBits); + if (MEM_32bits() && longOffsets && (ofBits >= STREAM_ACCUMULATOR_MIN_32)) { + /* Always read extra bits, this keeps the logic simple, + * avoids branches, and avoids accidentally reading 0 bits. + */ + U32 const extraBits = LONG_OFFSETS_MAX_EXTRA_BITS_32; + offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits); + BIT_reloadDStream(&seqState->DStream); + offset += BIT_readBitsFast(&seqState->DStream, extraBits); + } else { + offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits/*>0*/); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); + } + seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset; + } else { + U32 const ll0 = (llDInfo->baseValue == 0); + if (LIKELY((ofBits == 0))) { + offset = seqState->prevOffset[ll0]; + seqState->prevOffset[1] = seqState->prevOffset[!ll0]; + seqState->prevOffset[0] = offset; + } else { + offset = ofBase + ll0 + BIT_readBitsFast(&seqState->DStream, 1); + { size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; + temp -= !temp; /* 0 is not valid: input corrupted => force offset to -1 => corruption detected at execSequence */ + if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset = temp; + } } } + seq.offset = offset; + } + + if (mlBits > 0) + seq.matchLength += BIT_readBitsFast(&seqState->DStream, mlBits/*>0*/); + + if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32)) + BIT_reloadDStream(&seqState->DStream); + if (MEM_64bits() && UNLIKELY(totalBits >= STREAM_ACCUMULATOR_MIN_64-(LLFSELog+MLFSELog+OffFSELog))) + BIT_reloadDStream(&seqState->DStream); + /* Ensure there are enough bits to read the rest of data in 64-bit mode. */ + ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64); + + if (llBits > 0) + seq.litLength += BIT_readBitsFast(&seqState->DStream, llBits/*>0*/); + + if (MEM_32bits()) + BIT_reloadDStream(&seqState->DStream); + + DEBUGLOG(6, "seq: litL=%u, matchL=%u, offset=%u", + (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); + + if (!isLastSeq) { + /* don't update FSE state for last Sequence */ + ZSTD_updateFseStateWithDInfo(&seqState->stateLL, &seqState->DStream, llNext, llnbBits); /* <= 9 bits */ + ZSTD_updateFseStateWithDInfo(&seqState->stateML, &seqState->DStream, mlNext, mlnbBits); /* <= 9 bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ + ZSTD_updateFseStateWithDInfo(&seqState->stateOffb, &seqState->DStream, ofNext, ofnbBits); /* <= 8 bits */ + BIT_reloadDStream(&seqState->DStream); + } + } + + return seq; +} + +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) +#if DEBUGLEVEL >= 1 +static int ZSTD_dictionaryIsActive(ZSTD_DCtx const* dctx, BYTE const* prefixStart, BYTE const* oLitEnd) +{ + size_t const windowSize = dctx->fParams.windowSize; + /* No dictionary used. */ + if (dctx->dictContentEndForFuzzing == NULL) return 0; + /* Dictionary is our prefix. */ + if (prefixStart == dctx->dictContentBeginForFuzzing) return 1; + /* Dictionary is not our ext-dict. */ + if (dctx->dictEnd != dctx->dictContentEndForFuzzing) return 0; + /* Dictionary is not within our window size. */ + if ((size_t)(oLitEnd - prefixStart) >= windowSize) return 0; + /* Dictionary is active. */ + return 1; +} +#endif + +static void ZSTD_assertValidSequence( + ZSTD_DCtx const* dctx, + BYTE const* op, BYTE const* oend, + seq_t const seq, + BYTE const* prefixStart, BYTE const* virtualStart) +{ +#if DEBUGLEVEL >= 1 + if (dctx->isFrameDecompression) { + size_t const windowSize = dctx->fParams.windowSize; + size_t const sequenceSize = seq.litLength + seq.matchLength; + BYTE const* const oLitEnd = op + seq.litLength; + DEBUGLOG(6, "Checking sequence: litL=%u matchL=%u offset=%u", + (U32)seq.litLength, (U32)seq.matchLength, (U32)seq.offset); + assert(op <= oend); + assert((size_t)(oend - op) >= sequenceSize); + assert(sequenceSize <= ZSTD_blockSizeMax(dctx)); + if (ZSTD_dictionaryIsActive(dctx, prefixStart, oLitEnd)) { + size_t const dictSize = (size_t)((char const*)dctx->dictContentEndForFuzzing - (char const*)dctx->dictContentBeginForFuzzing); + /* Offset must be within the dictionary. */ + assert(seq.offset <= (size_t)(oLitEnd - virtualStart)); + assert(seq.offset <= windowSize + dictSize); + } else { + /* Offset must be within our window. */ + assert(seq.offset <= windowSize); + } + } +#else + (void)dctx, (void)op, (void)oend, (void)seq, (void)prefixStart, (void)virtualStart; +#endif +} +#endif + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG + + +FORCE_INLINE_TEMPLATE size_t +DONT_VECTORIZE +ZSTD_decompressSequences_bodySplitLitBuffer( ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + const BYTE* ip = (const BYTE*)seqStart; + const BYTE* const iend = ip + seqSize; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ZSTD_maybeNullPtrAdd(ostart, maxDstSize); + BYTE* op = ostart; + const BYTE* litPtr = dctx->litPtr; + const BYTE* litBufferEnd = dctx->litBufferEnd; + const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart); + const BYTE* const vBase = (const BYTE*) (dctx->virtualStart); + const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); + DEBUGLOG(5, "ZSTD_decompressSequences_bodySplitLitBuffer (%i seqs)", nbSeq); + + /* Literals are split between internal buffer & output buffer */ + if (nbSeq) { + seqState_t seqState; + dctx->fseEntropy = 1; + { U32 i; for (i=0; ientropy.rep[i]; } + RETURN_ERROR_IF( + ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)), + corruption_detected, ""); + ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); + ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); + ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + assert(dst != NULL); + + ZSTD_STATIC_ASSERT( + BIT_DStream_unfinished < BIT_DStream_completed && + BIT_DStream_endOfBuffer < BIT_DStream_completed && + BIT_DStream_completed < BIT_DStream_overflow); + + /* decompress without overrunning litPtr begins */ + { seq_t sequence = {0,0,0}; /* some static analyzer believe that @sequence is not initialized (it necessarily is, since for(;;) loop as at least one iteration) */ + /* Align the decompression loop to 32 + 16 bytes. + * + * zstd compiled with gcc-9 on an Intel i9-9900k shows 10% decompression + * speed swings based on the alignment of the decompression loop. This + * performance swing is caused by parts of the decompression loop falling + * out of the DSB. The entire decompression loop should fit in the DSB, + * when it can't we get much worse performance. You can measure if you've + * hit the good case or the bad case with this perf command for some + * compressed file test.zst: + * + * perf stat -e cycles -e instructions -e idq.all_dsb_cycles_any_uops \ + * -e idq.all_mite_cycles_any_uops -- ./zstd -tq test.zst + * + * If you see most cycles served out of the MITE you've hit the bad case. + * If you see most cycles served out of the DSB you've hit the good case. + * If it is pretty even then you may be in an okay case. + * + * This issue has been reproduced on the following CPUs: + * - Kabylake: Macbook Pro (15-inch, 2019) 2.4 GHz Intel Core i9 + * Use Instruments->Counters to get DSB/MITE cycles. + * I never got performance swings, but I was able to + * go from the good case of mostly DSB to half of the + * cycles served from MITE. + * - Coffeelake: Intel i9-9900k + * - Coffeelake: Intel i7-9700k + * + * I haven't been able to reproduce the instability or DSB misses on any + * of the following CPUS: + * - Haswell + * - Broadwell: Intel(R) Xeon(R) CPU E5-2680 v4 @ 2.40GH + * - Skylake + * + * Alignment is done for each of the three major decompression loops: + * - ZSTD_decompressSequences_bodySplitLitBuffer - presplit section of the literal buffer + * - ZSTD_decompressSequences_bodySplitLitBuffer - postsplit section of the literal buffer + * - ZSTD_decompressSequences_body + * Alignment choices are made to minimize large swings on bad cases and influence on performance + * from changes external to this code, rather than to overoptimize on the current commit. + * + * If you are seeing performance stability this script can help test. + * It tests on 4 commits in zstd where I saw performance change. + * + * https://gist.github.com/terrelln/9889fc06a423fd5ca6e99351564473f4 + */ +#if defined(__GNUC__) && defined(__x86_64__) + __asm__(".p2align 6"); +# if __GNUC__ >= 7 + /* good for gcc-7, gcc-9, and gcc-11 */ + __asm__("nop"); + __asm__(".p2align 5"); + __asm__("nop"); + __asm__(".p2align 4"); +# if __GNUC__ == 8 || __GNUC__ == 10 + /* good for gcc-8 and gcc-10 */ + __asm__("nop"); + __asm__(".p2align 3"); +# endif +# endif +#endif + + /* Handle the initial state where litBuffer is currently split between dst and litExtraBuffer */ + for ( ; nbSeq; nbSeq--) { + sequence = ZSTD_decodeSequence(&seqState, isLongOffset, nbSeq==1); + if (litPtr + sequence.litLength > dctx->litBufferEnd) break; + { size_t const oneSeqSize = ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence.litLength - WILDCOPY_OVERLENGTH, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; + } } + DEBUGLOG(6, "reached: (litPtr + sequence.litLength > dctx->litBufferEnd)"); + + /* If there are more sequences, they will need to read literals from litExtraBuffer; copy over the remainder from dst and update litPtr and litEnd */ + if (nbSeq > 0) { + const size_t leftoverLit = dctx->litBufferEnd - litPtr; + DEBUGLOG(6, "There are %i sequences left, and %zu/%zu literals left in buffer", nbSeq, leftoverLit, sequence.litLength); + if (leftoverLit) { + RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer"); + ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit); + sequence.litLength -= leftoverLit; + op += leftoverLit; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + { size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; + } + nbSeq--; + } + } + + if (nbSeq > 0) { + /* there is remaining lit from extra buffer */ + +#if defined(__GNUC__) && defined(__x86_64__) + __asm__(".p2align 6"); + __asm__("nop"); +# if __GNUC__ != 7 + /* worse for gcc-7 better for gcc-8, gcc-9, and gcc-10 and clang */ + __asm__(".p2align 4"); + __asm__("nop"); + __asm__(".p2align 3"); +# elif __GNUC__ >= 11 + __asm__(".p2align 3"); +# else + __asm__(".p2align 5"); + __asm__("nop"); + __asm__(".p2align 3"); +# endif +#endif + + for ( ; nbSeq ; nbSeq--) { + seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset, nbSeq==1); + size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litBufferEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; + } + } + + /* check if reached exact end */ + DEBUGLOG(5, "ZSTD_decompressSequences_bodySplitLitBuffer: after decode loop, remaining nbSeq : %i", nbSeq); + RETURN_ERROR_IF(nbSeq, corruption_detected, ""); + DEBUGLOG(5, "bitStream : start=%p, ptr=%p, bitsConsumed=%u", seqState.DStream.start, seqState.DStream.ptr, seqState.DStream.bitsConsumed); + RETURN_ERROR_IF(!BIT_endOfDStream(&seqState.DStream), corruption_detected, ""); + /* save reps for next block */ + { U32 i; for (i=0; ientropy.rep[i] = (U32)(seqState.prevOffset[i]); } + } + + /* last literal segment */ + if (dctx->litBufferLocation == ZSTD_split) { + /* split hasn't been reached yet, first get dst then copy litExtraBuffer */ + size_t const lastLLSize = (size_t)(litBufferEnd - litPtr); + DEBUGLOG(6, "copy last literals from segment : %u", (U32)lastLLSize); + RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memmove(op, litPtr, lastLLSize); + op += lastLLSize; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + } + /* copy last literals from internal buffer */ + { size_t const lastLLSize = (size_t)(litBufferEnd - litPtr); + DEBUGLOG(6, "copy last literals from internal buffer : %u", (U32)lastLLSize); + RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memcpy(op, litPtr, lastLLSize); + op += lastLLSize; + } } + + DEBUGLOG(6, "decoded block of size %u bytes", (U32)(op - ostart)); + return (size_t)(op - ostart); +} + +FORCE_INLINE_TEMPLATE size_t +DONT_VECTORIZE +ZSTD_decompressSequences_body(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + const BYTE* ip = (const BYTE*)seqStart; + const BYTE* const iend = ip + seqSize; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = dctx->litBufferLocation == ZSTD_not_in_dst ? ZSTD_maybeNullPtrAdd(ostart, maxDstSize) : dctx->litBuffer; + BYTE* op = ostart; + const BYTE* litPtr = dctx->litPtr; + const BYTE* const litEnd = litPtr + dctx->litSize; + const BYTE* const prefixStart = (const BYTE*)(dctx->prefixStart); + const BYTE* const vBase = (const BYTE*)(dctx->virtualStart); + const BYTE* const dictEnd = (const BYTE*)(dctx->dictEnd); + DEBUGLOG(5, "ZSTD_decompressSequences_body: nbSeq = %d", nbSeq); + + /* Regen sequences */ + if (nbSeq) { + seqState_t seqState; + dctx->fseEntropy = 1; + { U32 i; for (i = 0; i < ZSTD_REP_NUM; i++) seqState.prevOffset[i] = dctx->entropy.rep[i]; } + RETURN_ERROR_IF( + ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend - ip)), + corruption_detected, ""); + ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); + ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); + ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + assert(dst != NULL); + +#if defined(__GNUC__) && defined(__x86_64__) + __asm__(".p2align 6"); + __asm__("nop"); +# if __GNUC__ >= 7 + __asm__(".p2align 5"); + __asm__("nop"); + __asm__(".p2align 3"); +# else + __asm__(".p2align 4"); + __asm__("nop"); + __asm__(".p2align 3"); +# endif +#endif + + for ( ; nbSeq ; nbSeq--) { + seq_t const sequence = ZSTD_decodeSequence(&seqState, isLongOffset, nbSeq==1); + size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, prefixStart, vBase, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequence, prefixStart, vBase); +#endif + if (UNLIKELY(ZSTD_isError(oneSeqSize))) + return oneSeqSize; + DEBUGLOG(6, "regenerated sequence size : %u", (U32)oneSeqSize); + op += oneSeqSize; + } + + /* check if reached exact end */ + assert(nbSeq == 0); + RETURN_ERROR_IF(!BIT_endOfDStream(&seqState.DStream), corruption_detected, ""); + /* save reps for next block */ + { U32 i; for (i=0; ientropy.rep[i] = (U32)(seqState.prevOffset[i]); } + } + + /* last literal segment */ + { size_t const lastLLSize = (size_t)(litEnd - litPtr); + DEBUGLOG(6, "copy last literals : %u", (U32)lastLLSize); + RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memcpy(op, litPtr, lastLLSize); + op += lastLLSize; + } } + + DEBUGLOG(6, "decoded block of size %u bytes", (U32)(op - ostart)); + return (size_t)(op - ostart); +} + +static size_t +ZSTD_decompressSequences_default(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} + +static size_t +ZSTD_decompressSequencesSplitLitBuffer_default(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT + +FORCE_INLINE_TEMPLATE + +size_t ZSTD_prefetchMatch(size_t prefetchPos, seq_t const sequence, + const BYTE* const prefixStart, const BYTE* const dictEnd) +{ + prefetchPos += sequence.litLength; + { const BYTE* const matchBase = (sequence.offset > prefetchPos) ? dictEnd : prefixStart; + /* note : this operation can overflow when seq.offset is really too large, which can only happen when input is corrupted. + * No consequence though : memory address is only used for prefetching, not for dereferencing */ + const BYTE* const match = ZSTD_wrappedPtrSub(ZSTD_wrappedPtrAdd(matchBase, prefetchPos), sequence.offset); + PREFETCH_L1(match); PREFETCH_L1(match+CACHELINE_SIZE); /* note : it's safe to invoke PREFETCH() on any memory address, including invalid ones */ + } + return prefetchPos + sequence.matchLength; +} + +/* This decoding function employs prefetching + * to reduce latency impact of cache misses. + * It's generally employed when block contains a significant portion of long-distance matches + * or when coupled with a "cold" dictionary */ +FORCE_INLINE_TEMPLATE size_t +ZSTD_decompressSequencesLong_body( + ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + const BYTE* ip = (const BYTE*)seqStart; + const BYTE* const iend = ip + seqSize; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = dctx->litBufferLocation == ZSTD_in_dst ? dctx->litBuffer : ZSTD_maybeNullPtrAdd(ostart, maxDstSize); + BYTE* op = ostart; + const BYTE* litPtr = dctx->litPtr; + const BYTE* litBufferEnd = dctx->litBufferEnd; + const BYTE* const prefixStart = (const BYTE*) (dctx->prefixStart); + const BYTE* const dictStart = (const BYTE*) (dctx->virtualStart); + const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); + + /* Regen sequences */ + if (nbSeq) { +#define STORED_SEQS 8 +#define STORED_SEQS_MASK (STORED_SEQS-1) +#define ADVANCED_SEQS STORED_SEQS + seq_t sequences[STORED_SEQS]; + int const seqAdvance = MIN(nbSeq, ADVANCED_SEQS); + seqState_t seqState; + int seqNb; + size_t prefetchPos = (size_t)(op-prefixStart); /* track position relative to prefixStart */ + + dctx->fseEntropy = 1; + { int i; for (i=0; ientropy.rep[i]; } + assert(dst != NULL); + assert(iend >= ip); + RETURN_ERROR_IF( + ERR_isError(BIT_initDStream(&seqState.DStream, ip, iend-ip)), + corruption_detected, ""); + ZSTD_initFseState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); + ZSTD_initFseState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); + ZSTD_initFseState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + + /* prepare in advance */ + for (seqNb=0; seqNblitBufferLocation == ZSTD_split && litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength > dctx->litBufferEnd) { + /* lit buffer is reaching split point, empty out the first buffer and transition to litExtraBuffer */ + const size_t leftoverLit = dctx->litBufferEnd - litPtr; + if (leftoverLit) + { + RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer"); + ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit); + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength -= leftoverLit; + op += leftoverLit; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + { size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart); +#endif + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + + prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd); + sequences[seqNb & STORED_SEQS_MASK] = sequence; + op += oneSeqSize; + } } + else + { + /* lit buffer is either wholly contained in first or second split, or not split at all*/ + size_t const oneSeqSize = dctx->litBufferLocation == ZSTD_split ? + ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK].litLength - WILDCOPY_OVERLENGTH, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) : + ZSTD_execSequence(op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequences[(seqNb - ADVANCED_SEQS) & STORED_SEQS_MASK], prefixStart, dictStart); +#endif + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + + prefetchPos = ZSTD_prefetchMatch(prefetchPos, sequence, prefixStart, dictEnd); + sequences[seqNb & STORED_SEQS_MASK] = sequence; + op += oneSeqSize; + } + } + RETURN_ERROR_IF(!BIT_endOfDStream(&seqState.DStream), corruption_detected, ""); + + /* finish queue */ + seqNb -= seqAdvance; + for ( ; seqNblitBufferLocation == ZSTD_split && litPtr + sequence->litLength > dctx->litBufferEnd) { + const size_t leftoverLit = dctx->litBufferEnd - litPtr; + if (leftoverLit) { + RETURN_ERROR_IF(leftoverLit > (size_t)(oend - op), dstSize_tooSmall, "remaining lit must fit within dstBuffer"); + ZSTD_safecopyDstBeforeSrc(op, litPtr, leftoverLit); + sequence->litLength -= leftoverLit; + op += leftoverLit; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + dctx->litBufferLocation = ZSTD_not_in_dst; + { size_t const oneSeqSize = ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart); +#endif + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + op += oneSeqSize; + } + } + else + { + size_t const oneSeqSize = dctx->litBufferLocation == ZSTD_split ? + ZSTD_execSequenceSplitLitBuffer(op, oend, litPtr + sequence->litLength - WILDCOPY_OVERLENGTH, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd) : + ZSTD_execSequence(op, oend, *sequence, &litPtr, litBufferEnd, prefixStart, dictStart, dictEnd); +#if defined(FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION) && defined(FUZZING_ASSERT_VALID_SEQUENCE) + assert(!ZSTD_isError(oneSeqSize)); + ZSTD_assertValidSequence(dctx, op, oend, sequences[seqNb&STORED_SEQS_MASK], prefixStart, dictStart); +#endif + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + op += oneSeqSize; + } + } + + /* save reps for next block */ + { U32 i; for (i=0; ientropy.rep[i] = (U32)(seqState.prevOffset[i]); } + } + + /* last literal segment */ + if (dctx->litBufferLocation == ZSTD_split) { /* first deplete literal buffer in dst, then copy litExtraBuffer */ + size_t const lastLLSize = litBufferEnd - litPtr; + RETURN_ERROR_IF(lastLLSize > (size_t)(oend - op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memmove(op, litPtr, lastLLSize); + op += lastLLSize; + } + litPtr = dctx->litExtraBuffer; + litBufferEnd = dctx->litExtraBuffer + ZSTD_LITBUFFEREXTRASIZE; + } + { size_t const lastLLSize = litBufferEnd - litPtr; + RETURN_ERROR_IF(lastLLSize > (size_t)(oend-op), dstSize_tooSmall, ""); + if (op != NULL) { + ZSTD_memmove(op, litPtr, lastLLSize); + op += lastLLSize; + } + } + + return (size_t)(op - ostart); +} + +static size_t +ZSTD_decompressSequencesLong_default(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ + + + +#if DYNAMIC_BMI2 + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG +static BMI2_TARGET_ATTRIBUTE size_t +DONT_VECTORIZE +ZSTD_decompressSequences_bmi2(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + return ZSTD_decompressSequences_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} +static BMI2_TARGET_ATTRIBUTE size_t +DONT_VECTORIZE +ZSTD_decompressSequencesSplitLitBuffer_bmi2(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + return ZSTD_decompressSequences_bodySplitLitBuffer(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT +static BMI2_TARGET_ATTRIBUTE size_t +ZSTD_decompressSequencesLong_bmi2(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + return ZSTD_decompressSequencesLong_body(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ + +#endif /* DYNAMIC_BMI2 */ + +typedef size_t (*ZSTD_decompressSequences_t)( + ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset); + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG +static size_t +ZSTD_decompressSequences(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + DEBUGLOG(5, "ZSTD_decompressSequences"); +#if DYNAMIC_BMI2 + if (ZSTD_DCtx_get_bmi2(dctx)) { + return ZSTD_decompressSequences_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); + } +#endif + return ZSTD_decompressSequences_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} +static size_t +ZSTD_decompressSequencesSplitLitBuffer(ZSTD_DCtx* dctx, void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + DEBUGLOG(5, "ZSTD_decompressSequencesSplitLitBuffer"); +#if DYNAMIC_BMI2 + if (ZSTD_DCtx_get_bmi2(dctx)) { + return ZSTD_decompressSequencesSplitLitBuffer_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); + } +#endif + return ZSTD_decompressSequencesSplitLitBuffer_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG */ + + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT +/* ZSTD_decompressSequencesLong() : + * decompression function triggered when a minimum share of offsets is considered "long", + * aka out of cache. + * note : "long" definition seems overloaded here, sometimes meaning "wider than bitstream register", and sometimes meaning "farther than memory cache distance". + * This function will try to mitigate main memory latency through the use of prefetching */ +static size_t +ZSTD_decompressSequencesLong(ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize, int nbSeq, + const ZSTD_longOffset_e isLongOffset) +{ + DEBUGLOG(5, "ZSTD_decompressSequencesLong"); +#if DYNAMIC_BMI2 + if (ZSTD_DCtx_get_bmi2(dctx)) { + return ZSTD_decompressSequencesLong_bmi2(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); + } +#endif + return ZSTD_decompressSequencesLong_default(dctx, dst, maxDstSize, seqStart, seqSize, nbSeq, isLongOffset); +} +#endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ + + +/** + * @returns The total size of the history referenceable by zstd, including + * both the prefix and the extDict. At @p op any offset larger than this + * is invalid. + */ +static size_t ZSTD_totalHistorySize(BYTE* op, BYTE const* virtualStart) +{ + return (size_t)(op - virtualStart); +} + +typedef struct { + unsigned longOffsetShare; + unsigned maxNbAdditionalBits; +} ZSTD_OffsetInfo; + +/* ZSTD_getOffsetInfo() : + * condition : offTable must be valid + * @return : "share" of long offsets (arbitrarily defined as > (1<<23)) + * compared to maximum possible of (1< 22) info.longOffsetShare += 1; + } + + assert(tableLog <= OffFSELog); + info.longOffsetShare <<= (OffFSELog - tableLog); /* scale to OffFSELog */ + } + + return info; +} + +/** + * @returns The maximum offset we can decode in one read of our bitstream, without + * reloading more bits in the middle of the offset bits read. Any offsets larger + * than this must use the long offset decoder. + */ +static size_t ZSTD_maxShortOffset(void) +{ + if (MEM_64bits()) { + /* We can decode any offset without reloading bits. + * This might change if the max window size grows. + */ + ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31); + return (size_t)-1; + } else { + /* The maximum offBase is (1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1. + * This offBase would require STREAM_ACCUMULATOR_MIN extra bits. + * Then we have to subtract ZSTD_REP_NUM to get the maximum possible offset. + */ + size_t const maxOffbase = ((size_t)1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1; + size_t const maxOffset = maxOffbase - ZSTD_REP_NUM; + assert(ZSTD_highbit32((U32)maxOffbase) == STREAM_ACCUMULATOR_MIN); + return maxOffset; + } +} + +size_t +ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, const streaming_operation streaming) +{ /* blockType == blockCompressed */ + const BYTE* ip = (const BYTE*)src; + DEBUGLOG(5, "ZSTD_decompressBlock_internal (cSize : %u)", (unsigned)srcSize); + + /* Note : the wording of the specification + * allows compressed block to be sized exactly ZSTD_blockSizeMax(dctx). + * This generally does not happen, as it makes little sense, + * since an uncompressed block would feature same size and have no decompression cost. + * Also, note that decoder from reference libzstd before < v1.5.4 + * would consider this edge case as an error. + * As a consequence, avoid generating compressed blocks of size ZSTD_blockSizeMax(dctx) + * for broader compatibility with the deployed ecosystem of zstd decoders */ + RETURN_ERROR_IF(srcSize > ZSTD_blockSizeMax(dctx), srcSize_wrong, ""); + + /* Decode literals section */ + { size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize, dst, dstCapacity, streaming); + DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : cSize=%u, nbLiterals=%zu", (U32)litCSize, dctx->litSize); + if (ZSTD_isError(litCSize)) return litCSize; + ip += litCSize; + srcSize -= litCSize; + } + + /* Build Decoding Tables */ + { + /* Compute the maximum block size, which must also work when !frame and fParams are unset. + * Additionally, take the min with dstCapacity to ensure that the totalHistorySize fits in a size_t. + */ + size_t const blockSizeMax = MIN(dstCapacity, ZSTD_blockSizeMax(dctx)); + size_t const totalHistorySize = ZSTD_totalHistorySize(ZSTD_maybeNullPtrAdd((BYTE*)dst, blockSizeMax), (BYTE const*)dctx->virtualStart); + /* isLongOffset must be true if there are long offsets. + * Offsets are long if they are larger than ZSTD_maxShortOffset(). + * We don't expect that to be the case in 64-bit mode. + * + * We check here to see if our history is large enough to allow long offsets. + * If it isn't, then we can't possible have (valid) long offsets. If the offset + * is invalid, then it is okay to read it incorrectly. + * + * If isLongOffsets is true, then we will later check our decoding table to see + * if it is even possible to generate long offsets. + */ + ZSTD_longOffset_e isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (totalHistorySize > ZSTD_maxShortOffset())); + /* These macros control at build-time which decompressor implementation + * we use. If neither is defined, we do some inspection and dispatch at + * runtime. + */ +#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ + !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) + int usePrefetchDecoder = dctx->ddictIsCold; +#else + /* Set to 1 to avoid computing offset info if we don't need to. + * Otherwise this value is ignored. + */ + int usePrefetchDecoder = 1; +#endif + int nbSeq; + size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, srcSize); + if (ZSTD_isError(seqHSize)) return seqHSize; + ip += seqHSize; + srcSize -= seqHSize; + + RETURN_ERROR_IF((dst == NULL || dstCapacity == 0) && nbSeq > 0, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(MEM_64bits() && sizeof(size_t) == sizeof(void*) && (size_t)(-1) - (size_t)dst < (size_t)(1 << 20), dstSize_tooSmall, + "invalid dst"); + + /* If we could potentially have long offsets, or we might want to use the prefetch decoder, + * compute information about the share of long offsets, and the maximum nbAdditionalBits. + * NOTE: could probably use a larger nbSeq limit + */ + if (isLongOffset || (!usePrefetchDecoder && (totalHistorySize > (1u << 24)) && (nbSeq > 8))) { + ZSTD_OffsetInfo const info = ZSTD_getOffsetInfo(dctx->OFTptr, nbSeq); + if (isLongOffset && info.maxNbAdditionalBits <= STREAM_ACCUMULATOR_MIN) { + /* If isLongOffset, but the maximum number of additional bits that we see in our table is small + * enough, then we know it is impossible to have too long an offset in this block, so we can + * use the regular offset decoder. + */ + isLongOffset = ZSTD_lo_isRegularOffset; + } + if (!usePrefetchDecoder) { + U32 const minShare = MEM_64bits() ? 7 : 20; /* heuristic values, correspond to 2.73% and 7.81% */ + usePrefetchDecoder = (info.longOffsetShare >= minShare); + } + } + + dctx->ddictIsCold = 0; + +#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ + !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) + if (usePrefetchDecoder) { +#else + (void)usePrefetchDecoder; + { +#endif +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT + return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset); +#endif + } + +#ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG + /* else */ + if (dctx->litBufferLocation == ZSTD_split) + return ZSTD_decompressSequencesSplitLitBuffer(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset); + else + return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset); +#endif + } +} + + +ZSTD_ALLOW_POINTER_OVERFLOW_ATTR +void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize) +{ + if (dst != dctx->previousDstEnd && dstSize > 0) { /* not contiguous */ + dctx->dictEnd = dctx->previousDstEnd; + dctx->virtualStart = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->prefixStart)); + dctx->prefixStart = dst; + dctx->previousDstEnd = dst; + } +} + + +size_t ZSTD_decompressBlock_deprecated(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + size_t dSize; + dctx->isFrameDecompression = 0; + ZSTD_checkContinuity(dctx, dst, dstCapacity); + dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize, not_streaming); + FORWARD_IF_ERROR(dSize, ""); + dctx->previousDstEnd = (char*)dst + dSize; + return dSize; +} + + +/* NOTE: Must just wrap ZSTD_decompressBlock_deprecated() */ +size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + return ZSTD_decompressBlock_deprecated(dctx, dst, dstCapacity, src, srcSize); +} diff --git a/externals/zstd/lib/decompress/zstd_decompress_block.h b/externals/zstd/lib/decompress/zstd_decompress_block.h new file mode 100644 index 0000000..ab15240 --- /dev/null +++ b/externals/zstd/lib/decompress/zstd_decompress_block.h @@ -0,0 +1,73 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +#ifndef ZSTD_DEC_BLOCK_H +#define ZSTD_DEC_BLOCK_H + +/*-******************************************************* + * Dependencies + *********************************************************/ +#include "../common/zstd_deps.h" /* size_t */ +#include "../zstd.h" /* DCtx, and some public functions */ +#include "../common/zstd_internal.h" /* blockProperties_t, and some public functions */ +#include "zstd_decompress_internal.h" /* ZSTD_seqSymbol */ + + +/* === Prototypes === */ + +/* note: prototypes already published within `zstd.h` : + * ZSTD_decompressBlock() + */ + +/* note: prototypes already published within `zstd_internal.h` : + * ZSTD_getcBlockSize() + * ZSTD_decodeSeqHeaders() + */ + + + /* Streaming state is used to inform allocation of the literal buffer */ +typedef enum { + not_streaming = 0, + is_streaming = 1 +} streaming_operation; + +/* ZSTD_decompressBlock_internal() : + * decompress block, starting at `src`, + * into destination buffer `dst`. + * @return : decompressed block size, + * or an error code (which can be tested using ZSTD_isError()) + */ +size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, const streaming_operation streaming); + +/* ZSTD_buildFSETable() : + * generate FSE decoding table for one symbol (ll, ml or off) + * this function must be called with valid parameters only + * (dt is large enough, normalizedCounter distribution total is a power of 2, max is within range, etc.) + * in which case it cannot fail. + * The workspace must be 4-byte aligned and at least ZSTD_BUILD_FSE_TABLE_WKSP_SIZE bytes, which is + * defined in zstd_decompress_internal.h. + * Internal use only. + */ +void ZSTD_buildFSETable(ZSTD_seqSymbol* dt, + const short* normalizedCounter, unsigned maxSymbolValue, + const U32* baseValue, const U8* nbAdditionalBits, + unsigned tableLog, void* wksp, size_t wkspSize, + int bmi2); + +/* Internal definition of ZSTD_decompressBlock() to avoid deprecation warnings. */ +size_t ZSTD_decompressBlock_deprecated(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + + +#endif /* ZSTD_DEC_BLOCK_H */ diff --git a/externals/zstd/lib/decompress/zstd_decompress_internal.h b/externals/zstd/lib/decompress/zstd_decompress_internal.h new file mode 100644 index 0000000..83a7a01 --- /dev/null +++ b/externals/zstd/lib/decompress/zstd_decompress_internal.h @@ -0,0 +1,240 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/* zstd_decompress_internal: + * objects and definitions shared within lib/decompress modules */ + + #ifndef ZSTD_DECOMPRESS_INTERNAL_H + #define ZSTD_DECOMPRESS_INTERNAL_H + + +/*-******************************************************* + * Dependencies + *********************************************************/ +#include "../common/mem.h" /* BYTE, U16, U32 */ +#include "../common/zstd_internal.h" /* constants : MaxLL, MaxML, MaxOff, LLFSELog, etc. */ + + + +/*-******************************************************* + * Constants + *********************************************************/ +static UNUSED_ATTR const U32 LL_base[MaxLL+1] = { + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 18, 20, 22, 24, 28, 32, 40, + 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, + 0x2000, 0x4000, 0x8000, 0x10000 }; + +static UNUSED_ATTR const U32 OF_base[MaxOff+1] = { + 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, + 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, + 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, + 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD, 0x1FFFFFFD, 0x3FFFFFFD, 0x7FFFFFFD }; + +static UNUSED_ATTR const U8 OF_bits[MaxOff+1] = { + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31 }; + +static UNUSED_ATTR const U32 ML_base[MaxML+1] = { + 3, 4, 5, 6, 7, 8, 9, 10, + 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, + 27, 28, 29, 30, 31, 32, 33, 34, + 35, 37, 39, 41, 43, 47, 51, 59, + 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, + 0x1003, 0x2003, 0x4003, 0x8003, 0x10003 }; + + +/*-******************************************************* + * Decompression types + *********************************************************/ + typedef struct { + U32 fastMode; + U32 tableLog; + } ZSTD_seqSymbol_header; + + typedef struct { + U16 nextState; + BYTE nbAdditionalBits; + BYTE nbBits; + U32 baseValue; + } ZSTD_seqSymbol; + + #define SEQSYMBOL_TABLE_SIZE(log) (1 + (1 << (log))) + +#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE (sizeof(S16) * (MaxSeq + 1) + (1u << MaxFSELog) + sizeof(U64)) +#define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32 ((ZSTD_BUILD_FSE_TABLE_WKSP_SIZE + sizeof(U32) - 1) / sizeof(U32)) +#define ZSTD_HUFFDTABLE_CAPACITY_LOG 12 + +typedef struct { + ZSTD_seqSymbol LLTable[SEQSYMBOL_TABLE_SIZE(LLFSELog)]; /* Note : Space reserved for FSE Tables */ + ZSTD_seqSymbol OFTable[SEQSYMBOL_TABLE_SIZE(OffFSELog)]; /* is also used as temporary workspace while building hufTable during DDict creation */ + ZSTD_seqSymbol MLTable[SEQSYMBOL_TABLE_SIZE(MLFSELog)]; /* and therefore must be at least HUF_DECOMPRESS_WORKSPACE_SIZE large */ + HUF_DTable hufTable[HUF_DTABLE_SIZE(ZSTD_HUFFDTABLE_CAPACITY_LOG)]; /* can accommodate HUF_decompress4X */ + U32 rep[ZSTD_REP_NUM]; + U32 workspace[ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32]; +} ZSTD_entropyDTables_t; + +typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader, + ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock, + ZSTDds_decompressLastBlock, ZSTDds_checkChecksum, + ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage; + +typedef enum { zdss_init=0, zdss_loadHeader, + zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage; + +typedef enum { + ZSTD_use_indefinitely = -1, /* Use the dictionary indefinitely */ + ZSTD_dont_use = 0, /* Do not use the dictionary (if one exists free it) */ + ZSTD_use_once = 1 /* Use the dictionary once and set to ZSTD_dont_use */ +} ZSTD_dictUses_e; + +/* Hashset for storing references to multiple ZSTD_DDict within ZSTD_DCtx */ +typedef struct { + const ZSTD_DDict** ddictPtrTable; + size_t ddictPtrTableSize; + size_t ddictPtrCount; +} ZSTD_DDictHashSet; + +#ifndef ZSTD_DECODER_INTERNAL_BUFFER +# define ZSTD_DECODER_INTERNAL_BUFFER (1 << 16) +#endif + +#define ZSTD_LBMIN 64 +#define ZSTD_LBMAX (128 << 10) + +/* extra buffer, compensates when dst is not large enough to store litBuffer */ +#define ZSTD_LITBUFFEREXTRASIZE BOUNDED(ZSTD_LBMIN, ZSTD_DECODER_INTERNAL_BUFFER, ZSTD_LBMAX) + +typedef enum { + ZSTD_not_in_dst = 0, /* Stored entirely within litExtraBuffer */ + ZSTD_in_dst = 1, /* Stored entirely within dst (in memory after current output write) */ + ZSTD_split = 2 /* Split between litExtraBuffer and dst */ +} ZSTD_litLocation_e; + +struct ZSTD_DCtx_s +{ + const ZSTD_seqSymbol* LLTptr; + const ZSTD_seqSymbol* MLTptr; + const ZSTD_seqSymbol* OFTptr; + const HUF_DTable* HUFptr; + ZSTD_entropyDTables_t entropy; + U32 workspace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; /* space needed when building huffman tables */ + const void* previousDstEnd; /* detect continuity */ + const void* prefixStart; /* start of current segment */ + const void* virtualStart; /* virtual start of previous segment if it was just before current one */ + const void* dictEnd; /* end of previous segment */ + size_t expected; + ZSTD_frameHeader fParams; + U64 processedCSize; + U64 decodedSize; + blockType_e bType; /* used in ZSTD_decompressContinue(), store blockType between block header decoding and block decompression stages */ + ZSTD_dStage stage; + U32 litEntropy; + U32 fseEntropy; + XXH64_state_t xxhState; + size_t headerSize; + ZSTD_format_e format; + ZSTD_forceIgnoreChecksum_e forceIgnoreChecksum; /* User specified: if == 1, will ignore checksums in compressed frame. Default == 0 */ + U32 validateChecksum; /* if == 1, will validate checksum. Is == 1 if (fParams.checksumFlag == 1) and (forceIgnoreChecksum == 0). */ + const BYTE* litPtr; + ZSTD_customMem customMem; + size_t litSize; + size_t rleSize; + size_t staticSize; + int isFrameDecompression; +#if DYNAMIC_BMI2 != 0 + int bmi2; /* == 1 if the CPU supports BMI2 and 0 otherwise. CPU support is determined dynamically once per context lifetime. */ +#endif + + /* dictionary */ + ZSTD_DDict* ddictLocal; + const ZSTD_DDict* ddict; /* set by ZSTD_initDStream_usingDDict(), or ZSTD_DCtx_refDDict() */ + U32 dictID; + int ddictIsCold; /* if == 1 : dictionary is "new" for working context, and presumed "cold" (not in cpu cache) */ + ZSTD_dictUses_e dictUses; + ZSTD_DDictHashSet* ddictSet; /* Hash set for multiple ddicts */ + ZSTD_refMultipleDDicts_e refMultipleDDicts; /* User specified: if == 1, will allow references to multiple DDicts. Default == 0 (disabled) */ + int disableHufAsm; + int maxBlockSizeParam; + + /* streaming */ + ZSTD_dStreamStage streamStage; + char* inBuff; + size_t inBuffSize; + size_t inPos; + size_t maxWindowSize; + char* outBuff; + size_t outBuffSize; + size_t outStart; + size_t outEnd; + size_t lhSize; +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + void* legacyContext; + U32 previousLegacyVersion; + U32 legacyVersion; +#endif + U32 hostageByte; + int noForwardProgress; + ZSTD_bufferMode_e outBufferMode; + ZSTD_outBuffer expectedOutBuffer; + + /* workspace */ + BYTE* litBuffer; + const BYTE* litBufferEnd; + ZSTD_litLocation_e litBufferLocation; + BYTE litExtraBuffer[ZSTD_LITBUFFEREXTRASIZE + WILDCOPY_OVERLENGTH]; /* literal buffer can be split between storage within dst and within this scratch buffer */ + BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; + + size_t oversizedDuration; + +#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION + void const* dictContentBeginForFuzzing; + void const* dictContentEndForFuzzing; +#endif + + /* Tracing */ +#if ZSTD_TRACE + ZSTD_TraceCtx traceCtx; +#endif +}; /* typedef'd to ZSTD_DCtx within "zstd.h" */ + +MEM_STATIC int ZSTD_DCtx_get_bmi2(const struct ZSTD_DCtx_s *dctx) { +#if DYNAMIC_BMI2 != 0 + return dctx->bmi2; +#else + (void)dctx; + return 0; +#endif +} + +/*-******************************************************* + * Shared internal functions + *********************************************************/ + +/*! ZSTD_loadDEntropy() : + * dict : must point at beginning of a valid zstd dictionary. + * @return : size of dictionary header (size of magic number + dict ID + entropy tables) */ +size_t ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, + const void* const dict, size_t const dictSize); + +/*! ZSTD_checkContinuity() : + * check if next `dst` follows previous position, where decompression ended. + * If yes, do nothing (continue on current segment). + * If not, classify previous segment as "external dictionary", and start a new segment. + * This function cannot fail. */ +void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize); + + +#endif /* ZSTD_DECOMPRESS_INTERNAL_H */ diff --git a/externals/zstd/lib/zstd.h b/externals/zstd/lib/zstd.h new file mode 100644 index 0000000..5d1fef8 --- /dev/null +++ b/externals/zstd/lib/zstd.h @@ -0,0 +1,3089 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ +#if defined (__cplusplus) +extern "C" { +#endif + +#ifndef ZSTD_H_235446 +#define ZSTD_H_235446 + +/* ====== Dependencies ======*/ +#include /* INT_MAX */ +#include /* size_t */ + + +/* ===== ZSTDLIB_API : control library symbols visibility ===== */ +#ifndef ZSTDLIB_VISIBLE + /* Backwards compatibility with old macro name */ +# ifdef ZSTDLIB_VISIBILITY +# define ZSTDLIB_VISIBLE ZSTDLIB_VISIBILITY +# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDLIB_VISIBLE __attribute__ ((visibility ("default"))) +# else +# define ZSTDLIB_VISIBLE +# endif +#endif + +#ifndef ZSTDLIB_HIDDEN +# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDLIB_HIDDEN __attribute__ ((visibility ("hidden"))) +# else +# define ZSTDLIB_HIDDEN +# endif +#endif + +#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBLE +#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZSTDLIB_API __declspec(dllimport) ZSTDLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define ZSTDLIB_API ZSTDLIB_VISIBLE +#endif + +/* Deprecation warnings : + * Should these warnings be a problem, it is generally possible to disable them, + * typically with -Wno-deprecated-declarations for gcc or _CRT_SECURE_NO_WARNINGS in Visual. + * Otherwise, it's also possible to define ZSTD_DISABLE_DEPRECATE_WARNINGS. + */ +#ifdef ZSTD_DISABLE_DEPRECATE_WARNINGS +# define ZSTD_DEPRECATED(message) /* disable deprecation warnings */ +#else +# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ +# define ZSTD_DEPRECATED(message) [[deprecated(message)]] +# elif (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__) +# define ZSTD_DEPRECATED(message) __attribute__((deprecated(message))) +# elif defined(__GNUC__) && (__GNUC__ >= 3) +# define ZSTD_DEPRECATED(message) __attribute__((deprecated)) +# elif defined(_MSC_VER) +# define ZSTD_DEPRECATED(message) __declspec(deprecated(message)) +# else +# pragma message("WARNING: You need to implement ZSTD_DEPRECATED for this compiler") +# define ZSTD_DEPRECATED(message) +# endif +#endif /* ZSTD_DISABLE_DEPRECATE_WARNINGS */ + + +/******************************************************************************* + Introduction + + zstd, short for Zstandard, is a fast lossless compression algorithm, targeting + real-time compression scenarios at zlib-level and better compression ratios. + The zstd compression library provides in-memory compression and decompression + functions. + + The library supports regular compression levels from 1 up to ZSTD_maxCLevel(), + which is currently 22. Levels >= 20, labeled `--ultra`, should be used with + caution, as they require more memory. The library also offers negative + compression levels, which extend the range of speed vs. ratio preferences. + The lower the level, the faster the speed (at the cost of compression). + + Compression can be done in: + - a single step (described as Simple API) + - a single step, reusing a context (described as Explicit context) + - unbounded multiple steps (described as Streaming compression) + + The compression ratio achievable on small data can be highly improved using + a dictionary. Dictionary compression can be performed in: + - a single step (described as Simple dictionary API) + - a single step, reusing a dictionary (described as Bulk-processing + dictionary API) + + Advanced experimental functions can be accessed using + `#define ZSTD_STATIC_LINKING_ONLY` before including zstd.h. + + Advanced experimental APIs should never be used with a dynamically-linked + library. They are not "stable"; their definitions or signatures may change in + the future. Only static linking is allowed. +*******************************************************************************/ + +/*------ Version ------*/ +#define ZSTD_VERSION_MAJOR 1 +#define ZSTD_VERSION_MINOR 5 +#define ZSTD_VERSION_RELEASE 6 +#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) + +/*! ZSTD_versionNumber() : + * Return runtime library version, the value is (MAJOR*100*100 + MINOR*100 + RELEASE). */ +ZSTDLIB_API unsigned ZSTD_versionNumber(void); + +#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE +#define ZSTD_QUOTE(str) #str +#define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str) +#define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION) + +/*! ZSTD_versionString() : + * Return runtime library version, like "1.4.5". Requires v1.3.0+. */ +ZSTDLIB_API const char* ZSTD_versionString(void); + +/* ************************************* + * Default constant + ***************************************/ +#ifndef ZSTD_CLEVEL_DEFAULT +# define ZSTD_CLEVEL_DEFAULT 3 +#endif + +/* ************************************* + * Constants + ***************************************/ + +/* All magic numbers are supposed read/written to/from files/memory using little-endian convention */ +#define ZSTD_MAGICNUMBER 0xFD2FB528 /* valid since v0.8.0 */ +#define ZSTD_MAGIC_DICTIONARY 0xEC30A437 /* valid since v0.7.0 */ +#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50 /* all 16 values, from 0x184D2A50 to 0x184D2A5F, signal the beginning of a skippable frame */ +#define ZSTD_MAGIC_SKIPPABLE_MASK 0xFFFFFFF0 + +#define ZSTD_BLOCKSIZELOG_MAX 17 +#define ZSTD_BLOCKSIZE_MAX (1<= ZSTD_compressBound(srcSize)` guarantees that zstd will have + * enough space to successfully compress the data. + * @return : compressed size written into `dst` (<= `dstCapacity), + * or an error code if it fails (which can be tested using ZSTD_isError()). */ +ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + int compressionLevel); + +/*! ZSTD_decompress() : + * `compressedSize` : must be the _exact_ size of some number of compressed and/or skippable frames. + * `dstCapacity` is an upper bound of originalSize to regenerate. + * If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data. + * @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), + * or an errorCode if it fails (which can be tested using ZSTD_isError()). */ +ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity, + const void* src, size_t compressedSize); + +/*! ZSTD_getFrameContentSize() : requires v1.3.0+ + * `src` should point to the start of a ZSTD encoded frame. + * `srcSize` must be at least as large as the frame header. + * hint : any size >= `ZSTD_frameHeaderSize_max` is large enough. + * @return : - decompressed size of `src` frame content, if known + * - ZSTD_CONTENTSIZE_UNKNOWN if the size cannot be determined + * - ZSTD_CONTENTSIZE_ERROR if an error occurred (e.g. invalid magic number, srcSize too small) + * note 1 : a 0 return value means the frame is valid but "empty". + * note 2 : decompressed size is an optional field, it may not be present, typically in streaming mode. + * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. + * In which case, it's necessary to use streaming mode to decompress data. + * Optionally, application can rely on some implicit limit, + * as ZSTD_decompress() only needs an upper bound of decompressed size. + * (For example, data could be necessarily cut into blocks <= 16 KB). + * note 3 : decompressed size is always present when compression is completed using single-pass functions, + * such as ZSTD_compress(), ZSTD_compressCCtx() ZSTD_compress_usingDict() or ZSTD_compress_usingCDict(). + * note 4 : decompressed size can be very large (64-bits value), + * potentially larger than what local system can handle as a single memory segment. + * In which case, it's necessary to use streaming mode to decompress data. + * note 5 : If source is untrusted, decompressed size could be wrong or intentionally modified. + * Always ensure return value fits within application's authorized limits. + * Each application can set its own limits. + * note 6 : This function replaces ZSTD_getDecompressedSize() */ +#define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1) +#define ZSTD_CONTENTSIZE_ERROR (0ULL - 2) +ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t srcSize); + +/*! ZSTD_getDecompressedSize() : + * NOTE: This function is now obsolete, in favor of ZSTD_getFrameContentSize(). + * Both functions work the same way, but ZSTD_getDecompressedSize() blends + * "empty", "unknown" and "error" results to the same return value (0), + * while ZSTD_getFrameContentSize() gives them separate return values. + * @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */ +ZSTD_DEPRECATED("Replaced by ZSTD_getFrameContentSize") +ZSTDLIB_API +unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); + +/*! ZSTD_findFrameCompressedSize() : Requires v1.4.0+ + * `src` should point to the start of a ZSTD frame or skippable frame. + * `srcSize` must be >= first frame size + * @return : the compressed size of the first frame starting at `src`, + * suitable to pass as `srcSize` to `ZSTD_decompress` or similar, + * or an error code if input is invalid */ +ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize); + + +/*====== Helper functions ======*/ +/* ZSTD_compressBound() : + * maximum compressed size in worst case single-pass scenario. + * When invoking `ZSTD_compress()` or any other one-pass compression function, + * it's recommended to provide @dstCapacity >= ZSTD_compressBound(srcSize) + * as it eliminates one potential failure scenario, + * aka not enough room in dst buffer to write the compressed frame. + * Note : ZSTD_compressBound() itself can fail, if @srcSize > ZSTD_MAX_INPUT_SIZE . + * In which case, ZSTD_compressBound() will return an error code + * which can be tested using ZSTD_isError(). + * + * ZSTD_COMPRESSBOUND() : + * same as ZSTD_compressBound(), but as a macro. + * It can be used to produce constants, which can be useful for static allocation, + * for example to size a static array on stack. + * Will produce constant value 0 if srcSize too large. + */ +#define ZSTD_MAX_INPUT_SIZE ((sizeof(size_t)==8) ? 0xFF00FF00FF00FF00ULL : 0xFF00FF00U) +#define ZSTD_COMPRESSBOUND(srcSize) (((size_t)(srcSize) >= ZSTD_MAX_INPUT_SIZE) ? 0 : (srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */ +ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */ +/* ZSTD_isError() : + * Most ZSTD_* functions returning a size_t value can be tested for error, + * using ZSTD_isError(). + * @return 1 if error, 0 otherwise + */ +ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ +ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */ +ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed, requires v1.4.0+ */ +ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */ +ZSTDLIB_API int ZSTD_defaultCLevel(void); /*!< default compression level, specified by ZSTD_CLEVEL_DEFAULT, requires v1.5.0+ */ + + +/*************************************** +* Explicit context +***************************************/ +/*= Compression context + * When compressing many times, + * it is recommended to allocate a context just once, + * and reuse it for each successive compression operation. + * This will make workload friendlier for system's memory. + * Note : re-using context is just a speed / resource optimization. + * It doesn't change the compression ratio, which remains identical. + * Note 2 : In multi-threaded environments, + * use one different context per thread for parallel execution. + */ +typedef struct ZSTD_CCtx_s ZSTD_CCtx; +ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void); +ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); /* accept NULL pointer */ + +/*! ZSTD_compressCCtx() : + * Same as ZSTD_compress(), using an explicit ZSTD_CCtx. + * Important : in order to mirror `ZSTD_compress()` behavior, + * this function compresses at the requested compression level, + * __ignoring any other advanced parameter__ . + * If any advanced parameter was set using the advanced API, + * they will all be reset. Only `compressionLevel` remains. + */ +ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + int compressionLevel); + +/*= Decompression context + * When decompressing many times, + * it is recommended to allocate a context only once, + * and reuse it for each successive compression operation. + * This will make workload friendlier for system's memory. + * Use one context per thread for parallel execution. */ +typedef struct ZSTD_DCtx_s ZSTD_DCtx; +ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void); +ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); /* accept NULL pointer */ + +/*! ZSTD_decompressDCtx() : + * Same as ZSTD_decompress(), + * requires an allocated ZSTD_DCtx. + * Compatible with sticky parameters (see below). + */ +ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + + +/********************************************* +* Advanced compression API (Requires v1.4.0+) +**********************************************/ + +/* API design : + * Parameters are pushed one by one into an existing context, + * using ZSTD_CCtx_set*() functions. + * Pushed parameters are sticky : they are valid for next compressed frame, and any subsequent frame. + * "sticky" parameters are applicable to `ZSTD_compress2()` and `ZSTD_compressStream*()` ! + * __They do not apply to one-shot variants such as ZSTD_compressCCtx()__ . + * + * It's possible to reset all parameters to "default" using ZSTD_CCtx_reset(). + * + * This API supersedes all other "advanced" API entry points in the experimental section. + * In the future, we expect to remove API entry points from experimental which are redundant with this API. + */ + + +/* Compression strategies, listed from fastest to strongest */ +typedef enum { ZSTD_fast=1, + ZSTD_dfast=2, + ZSTD_greedy=3, + ZSTD_lazy=4, + ZSTD_lazy2=5, + ZSTD_btlazy2=6, + ZSTD_btopt=7, + ZSTD_btultra=8, + ZSTD_btultra2=9 + /* note : new strategies _might_ be added in the future. + Only the order (from fast to strong) is guaranteed */ +} ZSTD_strategy; + +typedef enum { + + /* compression parameters + * Note: When compressing with a ZSTD_CDict these parameters are superseded + * by the parameters used to construct the ZSTD_CDict. + * See ZSTD_CCtx_refCDict() for more info (superseded-by-cdict). */ + ZSTD_c_compressionLevel=100, /* Set compression parameters according to pre-defined cLevel table. + * Note that exact compression parameters are dynamically determined, + * depending on both compression level and srcSize (when known). + * Default level is ZSTD_CLEVEL_DEFAULT==3. + * Special: value 0 means default, which is controlled by ZSTD_CLEVEL_DEFAULT. + * Note 1 : it's possible to pass a negative compression level. + * Note 2 : setting a level does not automatically set all other compression parameters + * to default. Setting this will however eventually dynamically impact the compression + * parameters which have not been manually set. The manually set + * ones will 'stick'. */ + /* Advanced compression parameters : + * It's possible to pin down compression parameters to some specific values. + * In which case, these values are no longer dynamically selected by the compressor */ + ZSTD_c_windowLog=101, /* Maximum allowed back-reference distance, expressed as power of 2. + * This will set a memory budget for streaming decompression, + * with larger values requiring more memory + * and typically compressing more. + * Must be clamped between ZSTD_WINDOWLOG_MIN and ZSTD_WINDOWLOG_MAX. + * Special: value 0 means "use default windowLog". + * Note: Using a windowLog greater than ZSTD_WINDOWLOG_LIMIT_DEFAULT + * requires explicitly allowing such size at streaming decompression stage. */ + ZSTD_c_hashLog=102, /* Size of the initial probe table, as a power of 2. + * Resulting memory usage is (1 << (hashLog+2)). + * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX. + * Larger tables improve compression ratio of strategies <= dFast, + * and improve speed of strategies > dFast. + * Special: value 0 means "use default hashLog". */ + ZSTD_c_chainLog=103, /* Size of the multi-probe search table, as a power of 2. + * Resulting memory usage is (1 << (chainLog+2)). + * Must be clamped between ZSTD_CHAINLOG_MIN and ZSTD_CHAINLOG_MAX. + * Larger tables result in better and slower compression. + * This parameter is useless for "fast" strategy. + * It's still useful when using "dfast" strategy, + * in which case it defines a secondary probe table. + * Special: value 0 means "use default chainLog". */ + ZSTD_c_searchLog=104, /* Number of search attempts, as a power of 2. + * More attempts result in better and slower compression. + * This parameter is useless for "fast" and "dFast" strategies. + * Special: value 0 means "use default searchLog". */ + ZSTD_c_minMatch=105, /* Minimum size of searched matches. + * Note that Zstandard can still find matches of smaller size, + * it just tweaks its search algorithm to look for this size and larger. + * Larger values increase compression and decompression speed, but decrease ratio. + * Must be clamped between ZSTD_MINMATCH_MIN and ZSTD_MINMATCH_MAX. + * Note that currently, for all strategies < btopt, effective minimum is 4. + * , for all strategies > fast, effective maximum is 6. + * Special: value 0 means "use default minMatchLength". */ + ZSTD_c_targetLength=106, /* Impact of this field depends on strategy. + * For strategies btopt, btultra & btultra2: + * Length of Match considered "good enough" to stop search. + * Larger values make compression stronger, and slower. + * For strategy fast: + * Distance between match sampling. + * Larger values make compression faster, and weaker. + * Special: value 0 means "use default targetLength". */ + ZSTD_c_strategy=107, /* See ZSTD_strategy enum definition. + * The higher the value of selected strategy, the more complex it is, + * resulting in stronger and slower compression. + * Special: value 0 means "use default strategy". */ + + ZSTD_c_targetCBlockSize=130, /* v1.5.6+ + * Attempts to fit compressed block size into approximatively targetCBlockSize. + * Bound by ZSTD_TARGETCBLOCKSIZE_MIN and ZSTD_TARGETCBLOCKSIZE_MAX. + * Note that it's not a guarantee, just a convergence target (default:0). + * No target when targetCBlockSize == 0. + * This is helpful in low bandwidth streaming environments to improve end-to-end latency, + * when a client can make use of partial documents (a prominent example being Chrome). + * Note: this parameter is stable since v1.5.6. + * It was present as an experimental parameter in earlier versions, + * but it's not recommended using it with earlier library versions + * due to massive performance regressions. + */ + /* LDM mode parameters */ + ZSTD_c_enableLongDistanceMatching=160, /* Enable long distance matching. + * This parameter is designed to improve compression ratio + * for large inputs, by finding large matches at long distance. + * It increases memory usage and window size. + * Note: enabling this parameter increases default ZSTD_c_windowLog to 128 MB + * except when expressly set to a different value. + * Note: will be enabled by default if ZSTD_c_windowLog >= 128 MB and + * compression strategy >= ZSTD_btopt (== compression level 16+) */ + ZSTD_c_ldmHashLog=161, /* Size of the table for long distance matching, as a power of 2. + * Larger values increase memory usage and compression ratio, + * but decrease compression speed. + * Must be clamped between ZSTD_HASHLOG_MIN and ZSTD_HASHLOG_MAX + * default: windowlog - 7. + * Special: value 0 means "automatically determine hashlog". */ + ZSTD_c_ldmMinMatch=162, /* Minimum match size for long distance matcher. + * Larger/too small values usually decrease compression ratio. + * Must be clamped between ZSTD_LDM_MINMATCH_MIN and ZSTD_LDM_MINMATCH_MAX. + * Special: value 0 means "use default value" (default: 64). */ + ZSTD_c_ldmBucketSizeLog=163, /* Log size of each bucket in the LDM hash table for collision resolution. + * Larger values improve collision resolution but decrease compression speed. + * The maximum value is ZSTD_LDM_BUCKETSIZELOG_MAX. + * Special: value 0 means "use default value" (default: 3). */ + ZSTD_c_ldmHashRateLog=164, /* Frequency of inserting/looking up entries into the LDM hash table. + * Must be clamped between 0 and (ZSTD_WINDOWLOG_MAX - ZSTD_HASHLOG_MIN). + * Default is MAX(0, (windowLog - ldmHashLog)), optimizing hash table usage. + * Larger values improve compression speed. + * Deviating far from default value will likely result in a compression ratio decrease. + * Special: value 0 means "automatically determine hashRateLog". */ + + /* frame parameters */ + ZSTD_c_contentSizeFlag=200, /* Content size will be written into frame header _whenever known_ (default:1) + * Content size must be known at the beginning of compression. + * This is automatically the case when using ZSTD_compress2(), + * For streaming scenarios, content size must be provided with ZSTD_CCtx_setPledgedSrcSize() */ + ZSTD_c_checksumFlag=201, /* A 32-bits checksum of content is written at end of frame (default:0) */ + ZSTD_c_dictIDFlag=202, /* When applicable, dictionary's ID is written into frame header (default:1) */ + + /* multi-threading parameters */ + /* These parameters are only active if multi-threading is enabled (compiled with build macro ZSTD_MULTITHREAD). + * Otherwise, trying to set any other value than default (0) will be a no-op and return an error. + * In a situation where it's unknown if the linked library supports multi-threading or not, + * setting ZSTD_c_nbWorkers to any value >= 1 and consulting the return value provides a quick way to check this property. + */ + ZSTD_c_nbWorkers=400, /* Select how many threads will be spawned to compress in parallel. + * When nbWorkers >= 1, triggers asynchronous mode when invoking ZSTD_compressStream*() : + * ZSTD_compressStream*() consumes input and flush output if possible, but immediately gives back control to caller, + * while compression is performed in parallel, within worker thread(s). + * (note : a strong exception to this rule is when first invocation of ZSTD_compressStream2() sets ZSTD_e_end : + * in which case, ZSTD_compressStream2() delegates to ZSTD_compress2(), which is always a blocking call). + * More workers improve speed, but also increase memory usage. + * Default value is `0`, aka "single-threaded mode" : no worker is spawned, + * compression is performed inside Caller's thread, and all invocations are blocking */ + ZSTD_c_jobSize=401, /* Size of a compression job. This value is enforced only when nbWorkers >= 1. + * Each compression job is completed in parallel, so this value can indirectly impact the nb of active threads. + * 0 means default, which is dynamically determined based on compression parameters. + * Job size must be a minimum of overlap size, or ZSTDMT_JOBSIZE_MIN (= 512 KB), whichever is largest. + * The minimum size is automatically and transparently enforced. */ + ZSTD_c_overlapLog=402, /* Control the overlap size, as a fraction of window size. + * The overlap size is an amount of data reloaded from previous job at the beginning of a new job. + * It helps preserve compression ratio, while each job is compressed in parallel. + * This value is enforced only when nbWorkers >= 1. + * Larger values increase compression ratio, but decrease speed. + * Possible values range from 0 to 9 : + * - 0 means "default" : value will be determined by the library, depending on strategy + * - 1 means "no overlap" + * - 9 means "full overlap", using a full window size. + * Each intermediate rank increases/decreases load size by a factor 2 : + * 9: full window; 8: w/2; 7: w/4; 6: w/8; 5:w/16; 4: w/32; 3:w/64; 2:w/128; 1:no overlap; 0:default + * default value varies between 6 and 9, depending on strategy */ + + /* note : additional experimental parameters are also available + * within the experimental section of the API. + * At the time of this writing, they include : + * ZSTD_c_rsyncable + * ZSTD_c_format + * ZSTD_c_forceMaxWindow + * ZSTD_c_forceAttachDict + * ZSTD_c_literalCompressionMode + * ZSTD_c_srcSizeHint + * ZSTD_c_enableDedicatedDictSearch + * ZSTD_c_stableInBuffer + * ZSTD_c_stableOutBuffer + * ZSTD_c_blockDelimiters + * ZSTD_c_validateSequences + * ZSTD_c_useBlockSplitter + * ZSTD_c_useRowMatchFinder + * ZSTD_c_prefetchCDictTables + * ZSTD_c_enableSeqProducerFallback + * ZSTD_c_maxBlockSize + * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. + * note : never ever use experimentalParam? names directly; + * also, the enums values themselves are unstable and can still change. + */ + ZSTD_c_experimentalParam1=500, + ZSTD_c_experimentalParam2=10, + ZSTD_c_experimentalParam3=1000, + ZSTD_c_experimentalParam4=1001, + ZSTD_c_experimentalParam5=1002, + /* was ZSTD_c_experimentalParam6=1003; is now ZSTD_c_targetCBlockSize */ + ZSTD_c_experimentalParam7=1004, + ZSTD_c_experimentalParam8=1005, + ZSTD_c_experimentalParam9=1006, + ZSTD_c_experimentalParam10=1007, + ZSTD_c_experimentalParam11=1008, + ZSTD_c_experimentalParam12=1009, + ZSTD_c_experimentalParam13=1010, + ZSTD_c_experimentalParam14=1011, + ZSTD_c_experimentalParam15=1012, + ZSTD_c_experimentalParam16=1013, + ZSTD_c_experimentalParam17=1014, + ZSTD_c_experimentalParam18=1015, + ZSTD_c_experimentalParam19=1016 +} ZSTD_cParameter; + +typedef struct { + size_t error; + int lowerBound; + int upperBound; +} ZSTD_bounds; + +/*! ZSTD_cParam_getBounds() : + * All parameters must belong to an interval with lower and upper bounds, + * otherwise they will either trigger an error or be automatically clamped. + * @return : a structure, ZSTD_bounds, which contains + * - an error status field, which must be tested using ZSTD_isError() + * - lower and upper bounds, both inclusive + */ +ZSTDLIB_API ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter cParam); + +/*! ZSTD_CCtx_setParameter() : + * Set one compression parameter, selected by enum ZSTD_cParameter. + * All parameters have valid bounds. Bounds can be queried using ZSTD_cParam_getBounds(). + * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). + * Setting a parameter is generally only possible during frame initialization (before starting compression). + * Exception : when using multi-threading mode (nbWorkers >= 1), + * the following parameters can be updated _during_ compression (within same frame): + * => compressionLevel, hashLog, chainLog, searchLog, minMatch, targetLength and strategy. + * new parameters will be active for next job only (after a flush()). + * @return : an error code (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value); + +/*! ZSTD_CCtx_setPledgedSrcSize() : + * Total input data size to be compressed as a single frame. + * Value will be written in frame header, unless if explicitly forbidden using ZSTD_c_contentSizeFlag. + * This value will also be controlled at end of frame, and trigger an error if not respected. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Note 1 : pledgedSrcSize==0 actually means zero, aka an empty frame. + * In order to mean "unknown content size", pass constant ZSTD_CONTENTSIZE_UNKNOWN. + * ZSTD_CONTENTSIZE_UNKNOWN is default value for any new frame. + * Note 2 : pledgedSrcSize is only valid once, for the next frame. + * It's discarded at the end of the frame, and replaced by ZSTD_CONTENTSIZE_UNKNOWN. + * Note 3 : Whenever all input data is provided and consumed in a single round, + * for example with ZSTD_compress2(), + * or invoking immediately ZSTD_compressStream2(,,,ZSTD_e_end), + * this value is automatically overridden by srcSize instead. + */ +ZSTDLIB_API size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize); + +typedef enum { + ZSTD_reset_session_only = 1, + ZSTD_reset_parameters = 2, + ZSTD_reset_session_and_parameters = 3 +} ZSTD_ResetDirective; + +/*! ZSTD_CCtx_reset() : + * There are 2 different things that can be reset, independently or jointly : + * - The session : will stop compressing current frame, and make CCtx ready to start a new one. + * Useful after an error, or to interrupt any ongoing compression. + * Any internal data not yet flushed is cancelled. + * Compression parameters and dictionary remain unchanged. + * They will be used to compress next frame. + * Resetting session never fails. + * - The parameters : changes all parameters back to "default". + * This also removes any reference to any dictionary or external sequence producer. + * Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing) + * otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError()) + * - Both : similar to resetting the session, followed by resetting parameters. + */ +ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset); + +/*! ZSTD_compress2() : + * Behave the same as ZSTD_compressCCtx(), but compression parameters are set using the advanced API. + * (note that this entry point doesn't even expose a compression level parameter). + * ZSTD_compress2() always starts a new frame. + * Should cctx hold data from a previously unfinished frame, everything about it is forgotten. + * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() + * - The function is always blocking, returns when compression is completed. + * NOTE: Providing `dstCapacity >= ZSTD_compressBound(srcSize)` guarantees that zstd will have + * enough space to successfully compress the data, though it is possible it fails for other reasons. + * @return : compressed size written into `dst` (<= `dstCapacity), + * or an error code if it fails (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_compress2( ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + + +/*********************************************** +* Advanced decompression API (Requires v1.4.0+) +************************************************/ + +/* The advanced API pushes parameters one by one into an existing DCtx context. + * Parameters are sticky, and remain valid for all following frames + * using the same DCtx context. + * It's possible to reset parameters to default values using ZSTD_DCtx_reset(). + * Note : This API is compatible with existing ZSTD_decompressDCtx() and ZSTD_decompressStream(). + * Therefore, no new decompression function is necessary. + */ + +typedef enum { + + ZSTD_d_windowLogMax=100, /* Select a size limit (in power of 2) beyond which + * the streaming API will refuse to allocate memory buffer + * in order to protect the host from unreasonable memory requirements. + * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode. + * By default, a decompression context accepts window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT). + * Special: value 0 means "use default maximum windowLog". */ + + /* note : additional experimental parameters are also available + * within the experimental section of the API. + * At the time of this writing, they include : + * ZSTD_d_format + * ZSTD_d_stableOutBuffer + * ZSTD_d_forceIgnoreChecksum + * ZSTD_d_refMultipleDDicts + * ZSTD_d_disableHuffmanAssembly + * ZSTD_d_maxBlockSize + * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. + * note : never ever use experimentalParam? names directly + */ + ZSTD_d_experimentalParam1=1000, + ZSTD_d_experimentalParam2=1001, + ZSTD_d_experimentalParam3=1002, + ZSTD_d_experimentalParam4=1003, + ZSTD_d_experimentalParam5=1004, + ZSTD_d_experimentalParam6=1005 + +} ZSTD_dParameter; + +/*! ZSTD_dParam_getBounds() : + * All parameters must belong to an interval with lower and upper bounds, + * otherwise they will either trigger an error or be automatically clamped. + * @return : a structure, ZSTD_bounds, which contains + * - an error status field, which must be tested using ZSTD_isError() + * - both lower and upper bounds, inclusive + */ +ZSTDLIB_API ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam); + +/*! ZSTD_DCtx_setParameter() : + * Set one compression parameter, selected by enum ZSTD_dParameter. + * All parameters have valid bounds. Bounds can be queried using ZSTD_dParam_getBounds(). + * Providing a value beyond bound will either clamp it, or trigger an error (depending on parameter). + * Setting a parameter is only possible during frame initialization (before starting decompression). + * @return : 0, or an error code (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_API size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int value); + +/*! ZSTD_DCtx_reset() : + * Return a DCtx to clean state. + * Session and parameters can be reset jointly or separately. + * Parameters can only be reset when no active frame is being decompressed. + * @return : 0, or an error code, which can be tested with ZSTD_isError() + */ +ZSTDLIB_API size_t ZSTD_DCtx_reset(ZSTD_DCtx* dctx, ZSTD_ResetDirective reset); + + +/**************************** +* Streaming +****************************/ + +typedef struct ZSTD_inBuffer_s { + const void* src; /**< start of input buffer */ + size_t size; /**< size of input buffer */ + size_t pos; /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */ +} ZSTD_inBuffer; + +typedef struct ZSTD_outBuffer_s { + void* dst; /**< start of output buffer */ + size_t size; /**< size of output buffer */ + size_t pos; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */ +} ZSTD_outBuffer; + + + +/*-*********************************************************************** +* Streaming compression - HowTo +* +* A ZSTD_CStream object is required to track streaming operation. +* Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources. +* ZSTD_CStream objects can be reused multiple times on consecutive compression operations. +* It is recommended to reuse ZSTD_CStream since it will play nicer with system's memory, by re-using already allocated memory. +* +* For parallel execution, use one separate ZSTD_CStream per thread. +* +* note : since v1.3.0, ZSTD_CStream and ZSTD_CCtx are the same thing. +* +* Parameters are sticky : when starting a new compression on the same context, +* it will reuse the same sticky parameters as previous compression session. +* When in doubt, it's recommended to fully initialize the context before usage. +* Use ZSTD_CCtx_reset() to reset the context and ZSTD_CCtx_setParameter(), +* ZSTD_CCtx_setPledgedSrcSize(), or ZSTD_CCtx_loadDictionary() and friends to +* set more specific parameters, the pledged source size, or load a dictionary. +* +* Use ZSTD_compressStream2() with ZSTD_e_continue as many times as necessary to +* consume input stream. The function will automatically update both `pos` +* fields within `input` and `output`. +* Note that the function may not consume the entire input, for example, because +* the output buffer is already full, in which case `input.pos < input.size`. +* The caller must check if input has been entirely consumed. +* If not, the caller must make some room to receive more compressed data, +* and then present again remaining input data. +* note: ZSTD_e_continue is guaranteed to make some forward progress when called, +* but doesn't guarantee maximal forward progress. This is especially relevant +* when compressing with multiple threads. The call won't block if it can +* consume some input, but if it can't it will wait for some, but not all, +* output to be flushed. +* @return : provides a minimum amount of data remaining to be flushed from internal buffers +* or an error code, which can be tested using ZSTD_isError(). +* +* At any moment, it's possible to flush whatever data might remain stuck within internal buffer, +* using ZSTD_compressStream2() with ZSTD_e_flush. `output->pos` will be updated. +* Note that, if `output->size` is too small, a single invocation with ZSTD_e_flush might not be enough (return code > 0). +* In which case, make some room to receive more compressed data, and call again ZSTD_compressStream2() with ZSTD_e_flush. +* You must continue calling ZSTD_compressStream2() with ZSTD_e_flush until it returns 0, at which point you can change the +* operation. +* note: ZSTD_e_flush will flush as much output as possible, meaning when compressing with multiple threads, it will +* block until the flush is complete or the output buffer is full. +* @return : 0 if internal buffers are entirely flushed, +* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), +* or an error code, which can be tested using ZSTD_isError(). +* +* Calling ZSTD_compressStream2() with ZSTD_e_end instructs to finish a frame. +* It will perform a flush and write frame epilogue. +* The epilogue is required for decoders to consider a frame completed. +* flush operation is the same, and follows same rules as calling ZSTD_compressStream2() with ZSTD_e_flush. +* You must continue calling ZSTD_compressStream2() with ZSTD_e_end until it returns 0, at which point you are free to +* start a new frame. +* note: ZSTD_e_end will flush as much output as possible, meaning when compressing with multiple threads, it will +* block until the flush is complete or the output buffer is full. +* @return : 0 if frame fully completed and fully flushed, +* >0 if some data still present within internal buffer (the value is minimal estimation of remaining size), +* or an error code, which can be tested using ZSTD_isError(). +* +* *******************************************************************/ + +typedef ZSTD_CCtx ZSTD_CStream; /**< CCtx and CStream are now effectively same object (>= v1.3.0) */ + /* Continue to distinguish them for compatibility with older versions <= v1.2.0 */ +/*===== ZSTD_CStream management functions =====*/ +ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void); +ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); /* accept NULL pointer */ + +/*===== Streaming compression functions =====*/ +typedef enum { + ZSTD_e_continue=0, /* collect more data, encoder decides when to output compressed result, for optimal compression ratio */ + ZSTD_e_flush=1, /* flush any data provided so far, + * it creates (at least) one new block, that can be decoded immediately on reception; + * frame will continue: any future data can still reference previously compressed data, improving compression. + * note : multithreaded compression will block to flush as much output as possible. */ + ZSTD_e_end=2 /* flush any remaining data _and_ close current frame. + * note that frame is only closed after compressed data is fully flushed (return value == 0). + * After that point, any additional data starts a new frame. + * note : each frame is independent (does not reference any content from previous frame). + : note : multithreaded compression will block to flush as much output as possible. */ +} ZSTD_EndDirective; + +/*! ZSTD_compressStream2() : Requires v1.4.0+ + * Behaves about the same as ZSTD_compressStream, with additional control on end directive. + * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() + * - Compression parameters cannot be changed once compression is started (save a list of exceptions in multi-threading mode) + * - output->pos must be <= dstCapacity, input->pos must be <= srcSize + * - output->pos and input->pos will be updated. They are guaranteed to remain below their respective limit. + * - endOp must be a valid directive + * - When nbWorkers==0 (default), function is blocking : it completes its job before returning to caller. + * - When nbWorkers>=1, function is non-blocking : it copies a portion of input, distributes jobs to internal worker threads, flush to output whatever is available, + * and then immediately returns, just indicating that there is some data remaining to be flushed. + * The function nonetheless guarantees forward progress : it will return only after it reads or write at least 1+ byte. + * - Exception : if the first call requests a ZSTD_e_end directive and provides enough dstCapacity, the function delegates to ZSTD_compress2() which is always blocking. + * - @return provides a minimum amount of data remaining to be flushed from internal buffers + * or an error code, which can be tested using ZSTD_isError(). + * if @return != 0, flush is not fully completed, there is still some data left within internal buffers. + * This is useful for ZSTD_e_flush, since in this case more flushes are necessary to empty all buffers. + * For ZSTD_e_end, @return == 0 when internal buffers are fully flushed and frame is completed. + * - after a ZSTD_e_end directive, if internal buffer is not fully flushed (@return != 0), + * only ZSTD_e_end or ZSTD_e_flush operations are allowed. + * Before starting a new compression job, or changing compression parameters, + * it is required to fully flush internal buffers. + * - note: if an operation ends with an error, it may leave @cctx in an undefined state. + * Therefore, it's UB to invoke ZSTD_compressStream2() of ZSTD_compressStream() on such a state. + * In order to be re-employed after an error, a state must be reset, + * which can be done explicitly (ZSTD_CCtx_reset()), + * or is sometimes implied by methods starting a new compression job (ZSTD_initCStream(), ZSTD_compressCCtx()) + */ +ZSTDLIB_API size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, + ZSTD_outBuffer* output, + ZSTD_inBuffer* input, + ZSTD_EndDirective endOp); + + +/* These buffer sizes are softly recommended. + * They are not required : ZSTD_compressStream*() happily accepts any buffer size, for both input and output. + * Respecting the recommended size just makes it a bit easier for ZSTD_compressStream*(), + * reducing the amount of memory shuffling and buffering, resulting in minor performance savings. + * + * However, note that these recommendations are from the perspective of a C caller program. + * If the streaming interface is invoked from some other language, + * especially managed ones such as Java or Go, through a foreign function interface such as jni or cgo, + * a major performance rule is to reduce crossing such interface to an absolute minimum. + * It's not rare that performance ends being spent more into the interface, rather than compression itself. + * In which cases, prefer using large buffers, as large as practical, + * for both input and output, to reduce the nb of roundtrips. + */ +ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ +ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block. */ + + +/* ***************************************************************************** + * This following is a legacy streaming API, available since v1.0+ . + * It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2(). + * It is redundant, but remains fully supported. + ******************************************************************************/ + +/*! + * Equivalent to: + * + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) + * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + * + * Note that ZSTD_initCStream() clears any previously set dictionary. Use the new API + * to compress with a dictionary. + */ +ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); +/*! + * Alternative for ZSTD_compressStream2(zcs, output, input, ZSTD_e_continue). + * NOTE: The return value is different. ZSTD_compressStream() returns a hint for + * the next read size (if non-zero and not an error). ZSTD_compressStream2() + * returns the minimum nb of bytes left to flush (if non-zero and not an error). + */ +ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); +/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_flush). */ +ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); +/*! Equivalent to ZSTD_compressStream2(zcs, output, &emptyInput, ZSTD_e_end). */ +ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); + + +/*-*************************************************************************** +* Streaming decompression - HowTo +* +* A ZSTD_DStream object is required to track streaming operations. +* Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. +* ZSTD_DStream objects can be reused multiple times. +* +* Use ZSTD_initDStream() to start a new decompression operation. +* @return : recommended first input size +* Alternatively, use advanced API to set specific properties. +* +* Use ZSTD_decompressStream() repetitively to consume your input. +* The function will update both `pos` fields. +* If `input.pos < input.size`, some input has not been consumed. +* It's up to the caller to present again remaining data. +* The function tries to flush all data decoded immediately, respecting output buffer size. +* If `output.pos < output.size`, decoder has flushed everything it could. +* But if `output.pos == output.size`, there might be some data left within internal buffers., +* In which case, call ZSTD_decompressStream() again to flush whatever remains in the buffer. +* Note : with no additional input provided, amount of data flushed is necessarily <= ZSTD_BLOCKSIZE_MAX. +* @return : 0 when a frame is completely decoded and fully flushed, +* or an error code, which can be tested using ZSTD_isError(), +* or any other value > 0, which means there is still some decoding or flushing to do to complete current frame : +* the return value is a suggested next input size (just a hint for better latency) +* that will never request more than the remaining frame size. +* *******************************************************************************/ + +typedef ZSTD_DCtx ZSTD_DStream; /**< DCtx and DStream are now effectively same object (>= v1.3.0) */ + /* For compatibility with versions <= v1.2.0, prefer differentiating them. */ +/*===== ZSTD_DStream management functions =====*/ +ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void); +ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); /* accept NULL pointer */ + +/*===== Streaming decompression functions =====*/ + +/*! ZSTD_initDStream() : + * Initialize/reset DStream state for new decompression operation. + * Call before new decompression operation using same DStream. + * + * Note : This function is redundant with the advanced API and equivalent to: + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * ZSTD_DCtx_refDDict(zds, NULL); + */ +ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds); + +/*! ZSTD_decompressStream() : + * Streaming decompression function. + * Call repetitively to consume full input updating it as necessary. + * Function will update both input and output `pos` fields exposing current state via these fields: + * - `input.pos < input.size`, some input remaining and caller should provide remaining input + * on the next call. + * - `output.pos < output.size`, decoder finished and flushed all remaining buffers. + * - `output.pos == output.size`, potentially uncflushed data present in the internal buffers, + * call ZSTD_decompressStream() again to flush remaining data to output. + * Note : with no additional input, amount of data flushed <= ZSTD_BLOCKSIZE_MAX. + * + * @return : 0 when a frame is completely decoded and fully flushed, + * or an error code, which can be tested using ZSTD_isError(), + * or any other value > 0, which means there is some decoding or flushing to do to complete current frame. + * + * Note: when an operation returns with an error code, the @zds state may be left in undefined state. + * It's UB to invoke `ZSTD_decompressStream()` on such a state. + * In order to re-use such a state, it must be first reset, + * which can be done explicitly (`ZSTD_DCtx_reset()`), + * or is implied for operations starting some new decompression job (`ZSTD_initDStream`, `ZSTD_decompressDCtx()`, `ZSTD_decompress_usingDict()`) + */ +ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); + +ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ +ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */ + + +/************************** +* Simple dictionary API +***************************/ +/*! ZSTD_compress_usingDict() : + * Compression at an explicit compression level using a Dictionary. + * A dictionary can be any arbitrary data segment (also called a prefix), + * or a buffer with specified information (see zdict.h). + * Note : This function loads the dictionary, resulting in significant startup delay. + * It's intended for a dictionary used only once. + * Note 2 : When `dict == NULL || dictSize < 8` no dictionary is used. */ +ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + int compressionLevel); + +/*! ZSTD_decompress_usingDict() : + * Decompression using a known Dictionary. + * Dictionary must be identical to the one used during compression. + * Note : This function loads the dictionary, resulting in significant startup delay. + * It's intended for a dictionary used only once. + * Note : When `dict == NULL || dictSize < 8` no dictionary is used. */ +ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize); + + +/*********************************** + * Bulk processing dictionary API + **********************************/ +typedef struct ZSTD_CDict_s ZSTD_CDict; + +/*! ZSTD_createCDict() : + * When compressing multiple messages or blocks using the same dictionary, + * it's recommended to digest the dictionary only once, since it's a costly operation. + * ZSTD_createCDict() will create a state from digesting a dictionary. + * The resulting state can be used for future compression operations with very limited startup cost. + * ZSTD_CDict can be created once and shared by multiple threads concurrently, since its usage is read-only. + * @dictBuffer can be released after ZSTD_CDict creation, because its content is copied within CDict. + * Note 1 : Consider experimental function `ZSTD_createCDict_byReference()` if you prefer to not duplicate @dictBuffer content. + * Note 2 : A ZSTD_CDict can be created from an empty @dictBuffer, + * in which case the only thing that it transports is the @compressionLevel. + * This can be useful in a pipeline featuring ZSTD_compress_usingCDict() exclusively, + * expecting a ZSTD_CDict parameter with any data, including those without a known dictionary. */ +ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dictBuffer, size_t dictSize, + int compressionLevel); + +/*! ZSTD_freeCDict() : + * Function frees memory allocated by ZSTD_createCDict(). + * If a NULL pointer is passed, no operation is performed. */ +ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict); + +/*! ZSTD_compress_usingCDict() : + * Compression using a digested Dictionary. + * Recommended when same dictionary is used multiple times. + * Note : compression level is _decided at dictionary creation time_, + * and frame parameters are hardcoded (dictID=yes, contentSize=yes, checksum=no) */ +ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict); + + +typedef struct ZSTD_DDict_s ZSTD_DDict; + +/*! ZSTD_createDDict() : + * Create a digested dictionary, ready to start decompression operation without startup delay. + * dictBuffer can be released after DDict creation, as its content is copied inside DDict. */ +ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dictBuffer, size_t dictSize); + +/*! ZSTD_freeDDict() : + * Function frees memory allocated with ZSTD_createDDict() + * If a NULL pointer is passed, no operation is performed. */ +ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict); + +/*! ZSTD_decompress_usingDDict() : + * Decompression using a digested Dictionary. + * Recommended when same dictionary is used multiple times. */ +ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_DDict* ddict); + + +/******************************** + * Dictionary helper functions + *******************************/ + +/*! ZSTD_getDictID_fromDict() : Requires v1.4.0+ + * Provides the dictID stored within dictionary. + * if @return == 0, the dictionary is not conformant with Zstandard specification. + * It can still be loaded, but as a content-only dictionary. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize); + +/*! ZSTD_getDictID_fromCDict() : Requires v1.5.0+ + * Provides the dictID of the dictionary loaded into `cdict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromCDict(const ZSTD_CDict* cdict); + +/*! ZSTD_getDictID_fromDDict() : Requires v1.4.0+ + * Provides the dictID of the dictionary loaded into `ddict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict); + +/*! ZSTD_getDictID_fromFrame() : Requires v1.4.0+ + * Provides the dictID required to decompressed the frame stored within `src`. + * If @return == 0, the dictID could not be decoded. + * This could for one of the following reasons : + * - The frame does not require a dictionary to be decoded (most common case). + * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden piece of information. + * Note : this use case also happens when using a non-conformant dictionary. + * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`). + * - This is not a Zstandard frame. + * When identifying the exact failure cause, it's possible to use ZSTD_getFrameHeader(), which will provide a more precise error code. */ +ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); + + +/******************************************************************************* + * Advanced dictionary and prefix API (Requires v1.4.0+) + * + * This API allows dictionaries to be used with ZSTD_compress2(), + * ZSTD_compressStream2(), and ZSTD_decompressDCtx(). + * Dictionaries are sticky, they remain valid when same context is reused, + * they only reset when the context is reset + * with ZSTD_reset_parameters or ZSTD_reset_session_and_parameters. + * In contrast, Prefixes are single-use. + ******************************************************************************/ + + +/*! ZSTD_CCtx_loadDictionary() : Requires v1.4.0+ + * Create an internal CDict from `dict` buffer. + * Decompression will have to use same dictionary. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary, + * meaning "return to no-dictionary mode". + * Note 1 : Dictionary is sticky, it will be used for all future compressed frames, + * until parameters are reset, a new dictionary is loaded, or the dictionary + * is explicitly invalidated by loading a NULL dictionary. + * Note 2 : Loading a dictionary involves building tables. + * It's also a CPU consuming operation, with non-negligible impact on latency. + * Tables are dependent on compression parameters, and for this reason, + * compression parameters can no longer be changed after loading a dictionary. + * Note 3 :`dict` content will be copied internally. + * Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead. + * In such a case, dictionary buffer must outlive its users. + * Note 4 : Use ZSTD_CCtx_loadDictionary_advanced() + * to precisely select how dictionary content must be interpreted. + * Note 5 : This method does not benefit from LDM (long distance mode). + * If you want to employ LDM on some large dictionary content, + * prefer employing ZSTD_CCtx_refPrefix() described below. + */ +ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); + +/*! ZSTD_CCtx_refCDict() : Requires v1.4.0+ + * Reference a prepared dictionary, to be used for all future compressed frames. + * Note that compression parameters are enforced from within CDict, + * and supersede any compression parameter previously set within CCtx. + * The parameters ignored are labelled as "superseded-by-cdict" in the ZSTD_cParameter enum docs. + * The ignored parameters will be used again if the CCtx is returned to no-dictionary mode. + * The dictionary will remain valid for future compressed frames using same CCtx. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special : Referencing a NULL CDict means "return to no-dictionary mode". + * Note 1 : Currently, only one dictionary can be managed. + * Referencing a new dictionary effectively "discards" any previous one. + * Note 2 : CDict is just referenced, its lifetime must outlive its usage within CCtx. */ +ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); + +/*! ZSTD_CCtx_refPrefix() : Requires v1.4.0+ + * Reference a prefix (single-usage dictionary) for next compressed frame. + * A prefix is **only used once**. Tables are discarded at end of frame (ZSTD_e_end). + * Decompression will need same prefix to properly regenerate data. + * Compressing with a prefix is similar in outcome as performing a diff and compressing it, + * but performs much faster, especially during decompression (compression speed is tunable with compression level). + * This method is compatible with LDM (long distance mode). + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary + * Note 1 : Prefix buffer is referenced. It **must** outlive compression. + * Its content must remain unmodified during compression. + * Note 2 : If the intention is to diff some large src data blob with some prior version of itself, + * ensure that the window size is large enough to contain the entire source. + * See ZSTD_c_windowLog. + * Note 3 : Referencing a prefix involves building tables, which are dependent on compression parameters. + * It's a CPU consuming operation, with non-negligible impact on latency. + * If there is a need to use the same prefix multiple times, consider loadDictionary instead. + * Note 4 : By default, the prefix is interpreted as raw content (ZSTD_dct_rawContent). + * Use experimental ZSTD_CCtx_refPrefix_advanced() to alter dictionary interpretation. */ +ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, + const void* prefix, size_t prefixSize); + +/*! ZSTD_DCtx_loadDictionary() : Requires v1.4.0+ + * Create an internal DDict from dict buffer, to be used to decompress all future frames. + * The dictionary remains valid for all future frames, until explicitly invalidated, or + * a new dictionary is loaded. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary, + * meaning "return to no-dictionary mode". + * Note 1 : Loading a dictionary involves building tables, + * which has a non-negligible impact on CPU usage and latency. + * It's recommended to "load once, use many times", to amortize the cost + * Note 2 :`dict` content will be copied internally, so `dict` can be released after loading. + * Use ZSTD_DCtx_loadDictionary_byReference() to reference dictionary content instead. + * Note 3 : Use ZSTD_DCtx_loadDictionary_advanced() to take control of + * how dictionary content is loaded and interpreted. + */ +ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); + +/*! ZSTD_DCtx_refDDict() : Requires v1.4.0+ + * Reference a prepared dictionary, to be used to decompress next frames. + * The dictionary remains active for decompression of future frames using same DCtx. + * + * If called with ZSTD_d_refMultipleDDicts enabled, repeated calls of this function + * will store the DDict references in a table, and the DDict used for decompression + * will be determined at decompression time, as per the dict ID in the frame. + * The memory for the table is allocated on the first call to refDDict, and can be + * freed with ZSTD_freeDCtx(). + * + * If called with ZSTD_d_refMultipleDDicts disabled (the default), only one dictionary + * will be managed, and referencing a dictionary effectively "discards" any previous one. + * + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Special: referencing a NULL DDict means "return to no-dictionary mode". + * Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx. + */ +ZSTDLIB_API size_t ZSTD_DCtx_refDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); + +/*! ZSTD_DCtx_refPrefix() : Requires v1.4.0+ + * Reference a prefix (single-usage dictionary) to decompress next frame. + * This is the reverse operation of ZSTD_CCtx_refPrefix(), + * and must use the same prefix as the one used during compression. + * Prefix is **only used once**. Reference is discarded at end of frame. + * End of frame is reached when ZSTD_decompressStream() returns 0. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + * Note 1 : Adding any prefix (including NULL) invalidates any previously set prefix or dictionary + * Note 2 : Prefix buffer is referenced. It **must** outlive decompression. + * Prefix buffer must remain unmodified up to the end of frame, + * reached when ZSTD_decompressStream() returns 0. + * Note 3 : By default, the prefix is treated as raw content (ZSTD_dct_rawContent). + * Use ZSTD_CCtx_refPrefix_advanced() to alter dictMode (Experimental section) + * Note 4 : Referencing a raw content prefix has almost no cpu nor memory cost. + * A full dictionary is more costly, as it requires building tables. + */ +ZSTDLIB_API size_t ZSTD_DCtx_refPrefix(ZSTD_DCtx* dctx, + const void* prefix, size_t prefixSize); + +/* === Memory management === */ + +/*! ZSTD_sizeof_*() : Requires v1.4.0+ + * These functions give the _current_ memory usage of selected object. + * Note that object memory usage can evolve (increase or decrease) over time. */ +ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); +ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); +ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); +ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); +ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict); +ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); + +#endif /* ZSTD_H_235446 */ + + +/* ************************************************************************************** + * ADVANCED AND EXPERIMENTAL FUNCTIONS + **************************************************************************************** + * The definitions in the following section are considered experimental. + * They are provided for advanced scenarios. + * They should never be used with a dynamic library, as prototypes may change in the future. + * Use them only in association with static linking. + * ***************************************************************************************/ + +#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY) +#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY + +/* This can be overridden externally to hide static symbols. */ +#ifndef ZSTDLIB_STATIC_API +# if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDLIB_STATIC_API __declspec(dllexport) ZSTDLIB_VISIBLE +# elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZSTDLIB_STATIC_API __declspec(dllimport) ZSTDLIB_VISIBLE +# else +# define ZSTDLIB_STATIC_API ZSTDLIB_VISIBLE +# endif +#endif + +/**************************************************************************************** + * experimental API (static linking only) + **************************************************************************************** + * The following symbols and constants + * are not planned to join "stable API" status in the near future. + * They can still change in future versions. + * Some of them are planned to remain in the static_only section indefinitely. + * Some of them might be removed in the future (especially when redundant with existing stable functions) + * ***************************************************************************************/ + +#define ZSTD_FRAMEHEADERSIZE_PREFIX(format) ((format) == ZSTD_f_zstd1 ? 5 : 1) /* minimum input size required to query frame header size */ +#define ZSTD_FRAMEHEADERSIZE_MIN(format) ((format) == ZSTD_f_zstd1 ? 6 : 2) +#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* can be useful for static allocation */ +#define ZSTD_SKIPPABLEHEADERSIZE 8 + +/* compression parameter bounds */ +#define ZSTD_WINDOWLOG_MAX_32 30 +#define ZSTD_WINDOWLOG_MAX_64 31 +#define ZSTD_WINDOWLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64)) +#define ZSTD_WINDOWLOG_MIN 10 +#define ZSTD_HASHLOG_MAX ((ZSTD_WINDOWLOG_MAX < 30) ? ZSTD_WINDOWLOG_MAX : 30) +#define ZSTD_HASHLOG_MIN 6 +#define ZSTD_CHAINLOG_MAX_32 29 +#define ZSTD_CHAINLOG_MAX_64 30 +#define ZSTD_CHAINLOG_MAX ((int)(sizeof(size_t) == 4 ? ZSTD_CHAINLOG_MAX_32 : ZSTD_CHAINLOG_MAX_64)) +#define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN +#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1) +#define ZSTD_SEARCHLOG_MIN 1 +#define ZSTD_MINMATCH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */ +#define ZSTD_MINMATCH_MIN 3 /* only for ZSTD_btopt+, faster strategies are limited to 4 */ +#define ZSTD_TARGETLENGTH_MAX ZSTD_BLOCKSIZE_MAX +#define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */ +#define ZSTD_STRATEGY_MIN ZSTD_fast +#define ZSTD_STRATEGY_MAX ZSTD_btultra2 +#define ZSTD_BLOCKSIZE_MAX_MIN (1 << 10) /* The minimum valid max blocksize. Maximum blocksizes smaller than this make compressBound() inaccurate. */ + + +#define ZSTD_OVERLAPLOG_MIN 0 +#define ZSTD_OVERLAPLOG_MAX 9 + +#define ZSTD_WINDOWLOG_LIMIT_DEFAULT 27 /* by default, the streaming decoder will refuse any frame + * requiring larger than (1< 0: + * If litLength != 0: + * rep == 1 --> offset == repeat_offset_1 + * rep == 2 --> offset == repeat_offset_2 + * rep == 3 --> offset == repeat_offset_3 + * If litLength == 0: + * rep == 1 --> offset == repeat_offset_2 + * rep == 2 --> offset == repeat_offset_3 + * rep == 3 --> offset == repeat_offset_1 - 1 + * + * Note: This field is optional. ZSTD_generateSequences() will calculate the value of + * 'rep', but repeat offsets do not necessarily need to be calculated from an external + * sequence provider's perspective. For example, ZSTD_compressSequences() does not + * use this 'rep' field at all (as of now). + */ +} ZSTD_Sequence; + +typedef struct { + unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */ + unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */ + unsigned hashLog; /**< dispatch table : larger == faster, more memory */ + unsigned searchLog; /**< nb of searches : larger == more compression, slower */ + unsigned minMatch; /**< match length searched : larger == faster decompression, sometimes less compression */ + unsigned targetLength; /**< acceptable match size for optimal parser (only) : larger == more compression, slower */ + ZSTD_strategy strategy; /**< see ZSTD_strategy definition above */ +} ZSTD_compressionParameters; + +typedef struct { + int contentSizeFlag; /**< 1: content size will be in frame header (when known) */ + int checksumFlag; /**< 1: generate a 32-bits checksum using XXH64 algorithm at end of frame, for error detection */ + int noDictIDFlag; /**< 1: no dictID will be saved into frame header (dictID is only useful for dictionary compression) */ +} ZSTD_frameParameters; + +typedef struct { + ZSTD_compressionParameters cParams; + ZSTD_frameParameters fParams; +} ZSTD_parameters; + +typedef enum { + ZSTD_dct_auto = 0, /* dictionary is "full" when starting with ZSTD_MAGIC_DICTIONARY, otherwise it is "rawContent" */ + ZSTD_dct_rawContent = 1, /* ensures dictionary is always loaded as rawContent, even if it starts with ZSTD_MAGIC_DICTIONARY */ + ZSTD_dct_fullDict = 2 /* refuses to load a dictionary if it does not respect Zstandard's specification, starting with ZSTD_MAGIC_DICTIONARY */ +} ZSTD_dictContentType_e; + +typedef enum { + ZSTD_dlm_byCopy = 0, /**< Copy dictionary content internally */ + ZSTD_dlm_byRef = 1 /**< Reference dictionary content -- the dictionary buffer must outlive its users. */ +} ZSTD_dictLoadMethod_e; + +typedef enum { + ZSTD_f_zstd1 = 0, /* zstd frame format, specified in zstd_compression_format.md (default) */ + ZSTD_f_zstd1_magicless = 1 /* Variant of zstd frame format, without initial 4-bytes magic number. + * Useful to save 4 bytes per generated frame. + * Decoder cannot recognise automatically this format, requiring this instruction. */ +} ZSTD_format_e; + +typedef enum { + /* Note: this enum controls ZSTD_d_forceIgnoreChecksum */ + ZSTD_d_validateChecksum = 0, + ZSTD_d_ignoreChecksum = 1 +} ZSTD_forceIgnoreChecksum_e; + +typedef enum { + /* Note: this enum controls ZSTD_d_refMultipleDDicts */ + ZSTD_rmd_refSingleDDict = 0, + ZSTD_rmd_refMultipleDDicts = 1 +} ZSTD_refMultipleDDicts_e; + +typedef enum { + /* Note: this enum and the behavior it controls are effectively internal + * implementation details of the compressor. They are expected to continue + * to evolve and should be considered only in the context of extremely + * advanced performance tuning. + * + * Zstd currently supports the use of a CDict in three ways: + * + * - The contents of the CDict can be copied into the working context. This + * means that the compression can search both the dictionary and input + * while operating on a single set of internal tables. This makes + * the compression faster per-byte of input. However, the initial copy of + * the CDict's tables incurs a fixed cost at the beginning of the + * compression. For small compressions (< 8 KB), that copy can dominate + * the cost of the compression. + * + * - The CDict's tables can be used in-place. In this model, compression is + * slower per input byte, because the compressor has to search two sets of + * tables. However, this model incurs no start-up cost (as long as the + * working context's tables can be reused). For small inputs, this can be + * faster than copying the CDict's tables. + * + * - The CDict's tables are not used at all, and instead we use the working + * context alone to reload the dictionary and use params based on the source + * size. See ZSTD_compress_insertDictionary() and ZSTD_compress_usingDict(). + * This method is effective when the dictionary sizes are very small relative + * to the input size, and the input size is fairly large to begin with. + * + * Zstd has a simple internal heuristic that selects which strategy to use + * at the beginning of a compression. However, if experimentation shows that + * Zstd is making poor choices, it is possible to override that choice with + * this enum. + */ + ZSTD_dictDefaultAttach = 0, /* Use the default heuristic. */ + ZSTD_dictForceAttach = 1, /* Never copy the dictionary. */ + ZSTD_dictForceCopy = 2, /* Always copy the dictionary. */ + ZSTD_dictForceLoad = 3 /* Always reload the dictionary */ +} ZSTD_dictAttachPref_e; + +typedef enum { + ZSTD_lcm_auto = 0, /**< Automatically determine the compression mode based on the compression level. + * Negative compression levels will be uncompressed, and positive compression + * levels will be compressed. */ + ZSTD_lcm_huffman = 1, /**< Always attempt Huffman compression. Uncompressed literals will still be + * emitted if Huffman compression is not profitable. */ + ZSTD_lcm_uncompressed = 2 /**< Always emit uncompressed literals. */ +} ZSTD_literalCompressionMode_e; + +typedef enum { + /* Note: This enum controls features which are conditionally beneficial. Zstd typically will make a final + * decision on whether or not to enable the feature (ZSTD_ps_auto), but setting the switch to ZSTD_ps_enable + * or ZSTD_ps_disable allow for a force enable/disable the feature. + */ + ZSTD_ps_auto = 0, /* Let the library automatically determine whether the feature shall be enabled */ + ZSTD_ps_enable = 1, /* Force-enable the feature */ + ZSTD_ps_disable = 2 /* Do not use the feature */ +} ZSTD_paramSwitch_e; + +/*************************************** +* Frame header and size functions +***************************************/ + +/*! ZSTD_findDecompressedSize() : + * `src` should point to the start of a series of ZSTD encoded and/or skippable frames + * `srcSize` must be the _exact_ size of this series + * (i.e. there should be a frame boundary at `src + srcSize`) + * @return : - decompressed size of all data in all successive frames + * - if the decompressed size cannot be determined: ZSTD_CONTENTSIZE_UNKNOWN + * - if an error occurred: ZSTD_CONTENTSIZE_ERROR + * + * note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode. + * When `return==ZSTD_CONTENTSIZE_UNKNOWN`, data to decompress could be any size. + * In which case, it's necessary to use streaming mode to decompress data. + * note 2 : decompressed size is always present when compression is done with ZSTD_compress() + * note 3 : decompressed size can be very large (64-bits value), + * potentially larger than what local system can handle as a single memory segment. + * In which case, it's necessary to use streaming mode to decompress data. + * note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified. + * Always ensure result fits within application's authorized limits. + * Each application can set its own limits. + * note 5 : ZSTD_findDecompressedSize handles multiple frames, and so it must traverse the input to + * read each contained frame header. This is fast as most of the data is skipped, + * however it does mean that all frame data must be present and valid. */ +ZSTDLIB_STATIC_API unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize); + +/*! ZSTD_decompressBound() : + * `src` should point to the start of a series of ZSTD encoded and/or skippable frames + * `srcSize` must be the _exact_ size of this series + * (i.e. there should be a frame boundary at `src + srcSize`) + * @return : - upper-bound for the decompressed size of all data in all successive frames + * - if an error occurred: ZSTD_CONTENTSIZE_ERROR + * + * note 1 : an error can occur if `src` contains an invalid or incorrectly formatted frame. + * note 2 : the upper-bound is exact when the decompressed size field is available in every ZSTD encoded frame of `src`. + * in this case, `ZSTD_findDecompressedSize` and `ZSTD_decompressBound` return the same value. + * note 3 : when the decompressed size field isn't available, the upper-bound for that frame is calculated by: + * upper-bound = # blocks * min(128 KB, Window_Size) + */ +ZSTDLIB_STATIC_API unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize); + +/*! ZSTD_frameHeaderSize() : + * srcSize must be >= ZSTD_FRAMEHEADERSIZE_PREFIX. + * @return : size of the Frame Header, + * or an error code (if srcSize is too small) */ +ZSTDLIB_STATIC_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); + +typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e; +typedef struct { + unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */ + unsigned long long windowSize; /* can be very large, up to <= frameContentSize */ + unsigned blockSizeMax; + ZSTD_frameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */ + unsigned headerSize; + unsigned dictID; + unsigned checksumFlag; + unsigned _reserved1; + unsigned _reserved2; +} ZSTD_frameHeader; + +/*! ZSTD_getFrameHeader() : + * decode Frame Header, or requires larger `srcSize`. + * @return : 0, `zfhPtr` is correctly filled, + * >0, `srcSize` is too small, value is wanted `srcSize` amount, + * or an error code, which can be tested using ZSTD_isError() */ +ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */ +/*! ZSTD_getFrameHeader_advanced() : + * same as ZSTD_getFrameHeader(), + * with added capability to select a format (like ZSTD_f_zstd1_magicless) */ +ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format); + +/*! ZSTD_decompressionMargin() : + * Zstd supports in-place decompression, where the input and output buffers overlap. + * In this case, the output buffer must be at least (Margin + Output_Size) bytes large, + * and the input buffer must be at the end of the output buffer. + * + * _______________________ Output Buffer ________________________ + * | | + * | ____ Input Buffer ____| + * | | | + * v v v + * |---------------------------------------|-----------|----------| + * ^ ^ ^ + * |___________________ Output_Size ___________________|_ Margin _| + * + * NOTE: See also ZSTD_DECOMPRESSION_MARGIN(). + * NOTE: This applies only to single-pass decompression through ZSTD_decompress() or + * ZSTD_decompressDCtx(). + * NOTE: This function supports multi-frame input. + * + * @param src The compressed frame(s) + * @param srcSize The size of the compressed frame(s) + * @returns The decompression margin or an error that can be checked with ZSTD_isError(). + */ +ZSTDLIB_STATIC_API size_t ZSTD_decompressionMargin(const void* src, size_t srcSize); + +/*! ZSTD_DECOMPRESS_MARGIN() : + * Similar to ZSTD_decompressionMargin(), but instead of computing the margin from + * the compressed frame, compute it from the original size and the blockSizeLog. + * See ZSTD_decompressionMargin() for details. + * + * WARNING: This macro does not support multi-frame input, the input must be a single + * zstd frame. If you need that support use the function, or implement it yourself. + * + * @param originalSize The original uncompressed size of the data. + * @param blockSize The block size == MIN(windowSize, ZSTD_BLOCKSIZE_MAX). + * Unless you explicitly set the windowLog smaller than + * ZSTD_BLOCKSIZELOG_MAX you can just use ZSTD_BLOCKSIZE_MAX. + */ +#define ZSTD_DECOMPRESSION_MARGIN(originalSize, blockSize) ((size_t)( \ + ZSTD_FRAMEHEADERSIZE_MAX /* Frame header */ + \ + 4 /* checksum */ + \ + ((originalSize) == 0 ? 0 : 3 * (((originalSize) + (blockSize) - 1) / blockSize)) /* 3 bytes per block */ + \ + (blockSize) /* One block of margin */ \ + )) + +typedef enum { + ZSTD_sf_noBlockDelimiters = 0, /* Representation of ZSTD_Sequence has no block delimiters, sequences only */ + ZSTD_sf_explicitBlockDelimiters = 1 /* Representation of ZSTD_Sequence contains explicit block delimiters */ +} ZSTD_sequenceFormat_e; + +/*! ZSTD_sequenceBound() : + * `srcSize` : size of the input buffer + * @return : upper-bound for the number of sequences that can be generated + * from a buffer of srcSize bytes + * + * note : returns number of sequences - to get bytes, multiply by sizeof(ZSTD_Sequence). + */ +ZSTDLIB_STATIC_API size_t ZSTD_sequenceBound(size_t srcSize); + +/*! ZSTD_generateSequences() : + * WARNING: This function is meant for debugging and informational purposes ONLY! + * Its implementation is flawed, and it will be deleted in a future version. + * It is not guaranteed to succeed, as there are several cases where it will give + * up and fail. You should NOT use this function in production code. + * + * This function is deprecated, and will be removed in a future version. + * + * Generate sequences using ZSTD_compress2(), given a source buffer. + * + * @param zc The compression context to be used for ZSTD_compress2(). Set any + * compression parameters you need on this context. + * @param outSeqs The output sequences buffer of size @p outSeqsSize + * @param outSeqsSize The size of the output sequences buffer. + * ZSTD_sequenceBound(srcSize) is an upper bound on the number + * of sequences that can be generated. + * @param src The source buffer to generate sequences from of size @p srcSize. + * @param srcSize The size of the source buffer. + * + * Each block will end with a dummy sequence + * with offset == 0, matchLength == 0, and litLength == length of last literals. + * litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0) + * simply acts as a block delimiter. + * + * @returns The number of sequences generated, necessarily less than + * ZSTD_sequenceBound(srcSize), or an error code that can be checked + * with ZSTD_isError(). + */ +ZSTD_DEPRECATED("For debugging only, will be replaced by ZSTD_extractSequences()") +ZSTDLIB_STATIC_API size_t +ZSTD_generateSequences(ZSTD_CCtx* zc, + ZSTD_Sequence* outSeqs, size_t outSeqsSize, + const void* src, size_t srcSize); + +/*! ZSTD_mergeBlockDelimiters() : + * Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals + * by merging them into the literals of the next sequence. + * + * As such, the final generated result has no explicit representation of block boundaries, + * and the final last literals segment is not represented in the sequences. + * + * The output of this function can be fed into ZSTD_compressSequences() with CCtx + * setting of ZSTD_c_blockDelimiters as ZSTD_sf_noBlockDelimiters + * @return : number of sequences left after merging + */ +ZSTDLIB_STATIC_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize); + +/*! ZSTD_compressSequences() : + * Compress an array of ZSTD_Sequence, associated with @src buffer, into dst. + * @src contains the entire input (not just the literals). + * If @srcSize > sum(sequence.length), the remaining bytes are considered all literals + * If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.) + * The entire source is compressed into a single frame. + * + * The compression behavior changes based on cctx params. In particular: + * If ZSTD_c_blockDelimiters == ZSTD_sf_noBlockDelimiters, the array of ZSTD_Sequence is expected to contain + * no block delimiters (defined in ZSTD_Sequence). Block boundaries are roughly determined based on + * the block size derived from the cctx, and sequences may be split. This is the default setting. + * + * If ZSTD_c_blockDelimiters == ZSTD_sf_explicitBlockDelimiters, the array of ZSTD_Sequence is expected to contain + * block delimiters (defined in ZSTD_Sequence). Behavior is undefined if no block delimiters are provided. + * + * If ZSTD_c_validateSequences == 0, this function will blindly accept the sequences provided. Invalid sequences cause undefined + * behavior. If ZSTD_c_validateSequences == 1, then if sequence is invalid (see doc/zstd_compression_format.md for + * specifics regarding offset/matchlength requirements) then the function will bail out and return an error. + * + * In addition to the two adjustable experimental params, there are other important cctx params. + * - ZSTD_c_minMatch MUST be set as less than or equal to the smallest match generated by the match finder. It has a minimum value of ZSTD_MINMATCH_MIN. + * - ZSTD_c_compressionLevel accordingly adjusts the strength of the entropy coder, as it would in typical compression. + * - ZSTD_c_windowLog affects offset validation: this function will return an error at higher debug levels if a provided offset + * is larger than what the spec allows for a given window log and dictionary (if present). See: doc/zstd_compression_format.md + * + * Note: Repcodes are, as of now, always re-calculated within this function, so ZSTD_Sequence::rep is unused. + * Note 2: Once we integrate ability to ingest repcodes, the explicit block delims mode must respect those repcodes exactly, + * and cannot emit an RLE block that disagrees with the repcode history + * @return : final compressed size, or a ZSTD error code. + */ +ZSTDLIB_STATIC_API size_t +ZSTD_compressSequences( ZSTD_CCtx* cctx, void* dst, size_t dstSize, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, + const void* src, size_t srcSize); + + +/*! ZSTD_writeSkippableFrame() : + * Generates a zstd skippable frame containing data given by src, and writes it to dst buffer. + * + * Skippable frames begin with a 4-byte magic number. There are 16 possible choices of magic number, + * ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15. + * As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so + * the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant. + * + * Returns an error if destination buffer is not large enough, if the source size is not representable + * with a 4-byte unsigned int, or if the parameter magicVariant is greater than 15 (and therefore invalid). + * + * @return : number of bytes written or a ZSTD error. + */ +ZSTDLIB_STATIC_API size_t ZSTD_writeSkippableFrame(void* dst, size_t dstCapacity, + const void* src, size_t srcSize, unsigned magicVariant); + +/*! ZSTD_readSkippableFrame() : + * Retrieves a zstd skippable frame containing data given by src, and writes it to dst buffer. + * + * The parameter magicVariant will receive the magicVariant that was supplied when the frame was written, + * i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. This can be NULL if the caller is not interested + * in the magicVariant. + * + * Returns an error if destination buffer is not large enough, or if the frame is not skippable. + * + * @return : number of bytes written or a ZSTD error. + */ +ZSTDLIB_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, unsigned* magicVariant, + const void* src, size_t srcSize); + +/*! ZSTD_isSkippableFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier for a skippable frame. + */ +ZSTDLIB_API unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size); + + + +/*************************************** +* Memory management +***************************************/ + +/*! ZSTD_estimate*() : + * These functions make it possible to estimate memory usage + * of a future {D,C}Ctx, before its creation. + * This is useful in combination with ZSTD_initStatic(), + * which makes it possible to employ a static buffer for ZSTD_CCtx* state. + * + * ZSTD_estimateCCtxSize() will provide a memory budget large enough + * to compress data of any size using one-shot compression ZSTD_compressCCtx() or ZSTD_compress2() + * associated with any compression level up to max specified one. + * The estimate will assume the input may be arbitrarily large, + * which is the worst case. + * + * Note that the size estimation is specific for one-shot compression, + * it is not valid for streaming (see ZSTD_estimateCStreamSize*()) + * nor other potential ways of using a ZSTD_CCtx* state. + * + * When srcSize can be bound by a known and rather "small" value, + * this knowledge can be used to provide a tighter budget estimation + * because the ZSTD_CCtx* state will need less memory for small inputs. + * This tighter estimation can be provided by employing more advanced functions + * ZSTD_estimateCCtxSize_usingCParams(), which can be used in tandem with ZSTD_getCParams(), + * and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter(). + * Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits. + * + * Note : only single-threaded compression is supported. + * ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1. + */ +ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize(int maxCompressionLevel); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDCtxSize(void); + +/*! ZSTD_estimateCStreamSize() : + * ZSTD_estimateCStreamSize() will provide a memory budget large enough for streaming compression + * using any compression level up to the max specified one. + * It will also consider src size to be arbitrarily "large", which is a worst case scenario. + * If srcSize is known to always be small, ZSTD_estimateCStreamSize_usingCParams() can provide a tighter estimation. + * ZSTD_estimateCStreamSize_usingCParams() can be used in tandem with ZSTD_getCParams() to create cParams from compressionLevel. + * ZSTD_estimateCStreamSize_usingCCtxParams() can be used in tandem with ZSTD_CCtxParams_setParameter(). Only single-threaded compression is supported. This function will return an error code if ZSTD_c_nbWorkers is >= 1. + * Note : CStream size estimation is only correct for single-threaded compression. + * ZSTD_estimateCStreamSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1. + * Note 2 : ZSTD_estimateCStreamSize* functions are not compatible with the Block-Level Sequence Producer API at this time. + * Size estimates assume that no external sequence producer is registered. + * + * ZSTD_DStream memory budget depends on frame's window Size. + * This information can be passed manually, using ZSTD_estimateDStreamSize, + * or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame(); + * Any frame requesting a window size larger than max specified one will be rejected. + * Note : if streaming is init with function ZSTD_init?Stream_usingDict(), + * an internal ?Dict will be created, which additional size is not estimated here. + * In this case, get total size by adding ZSTD_estimate?DictSize + */ +ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize(int maxCompressionLevel); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize(size_t maxWindowSize); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDStreamSize_fromFrame(const void* src, size_t srcSize); + +/*! ZSTD_estimate?DictSize() : + * ZSTD_estimateCDictSize() will bet that src size is relatively "small", and content is copied, like ZSTD_createCDict(). + * ZSTD_estimateCDictSize_advanced() makes it possible to control compression parameters precisely, like ZSTD_createCDict_advanced(). + * Note : dictionaries created by reference (`ZSTD_dlm_byRef`) are logically smaller. + */ +ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize(size_t dictSize, int compressionLevel); +ZSTDLIB_STATIC_API size_t ZSTD_estimateCDictSize_advanced(size_t dictSize, ZSTD_compressionParameters cParams, ZSTD_dictLoadMethod_e dictLoadMethod); +ZSTDLIB_STATIC_API size_t ZSTD_estimateDDictSize(size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod); + +/*! ZSTD_initStatic*() : + * Initialize an object using a pre-allocated fixed-size buffer. + * workspace: The memory area to emplace the object into. + * Provided pointer *must be 8-bytes aligned*. + * Buffer must outlive object. + * workspaceSize: Use ZSTD_estimate*Size() to determine + * how large workspace must be to support target scenario. + * @return : pointer to object (same address as workspace, just different type), + * or NULL if error (size too small, incorrect alignment, etc.) + * Note : zstd will never resize nor malloc() when using a static buffer. + * If the object requires more memory than available, + * zstd will just error out (typically ZSTD_error_memory_allocation). + * Note 2 : there is no corresponding "free" function. + * Since workspace is allocated externally, it must be freed externally too. + * Note 3 : cParams : use ZSTD_getCParams() to convert a compression level + * into its associated cParams. + * Limitation 1 : currently not compatible with internal dictionary creation, triggered by + * ZSTD_CCtx_loadDictionary(), ZSTD_initCStream_usingDict() or ZSTD_initDStream_usingDict(). + * Limitation 2 : static cctx currently not compatible with multi-threading. + * Limitation 3 : static dctx is incompatible with legacy support. + */ +ZSTDLIB_STATIC_API ZSTD_CCtx* ZSTD_initStaticCCtx(void* workspace, size_t workspaceSize); +ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_initStaticCStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticCCtx() */ + +ZSTDLIB_STATIC_API ZSTD_DCtx* ZSTD_initStaticDCtx(void* workspace, size_t workspaceSize); +ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_initStaticDStream(void* workspace, size_t workspaceSize); /**< same as ZSTD_initStaticDCtx() */ + +ZSTDLIB_STATIC_API const ZSTD_CDict* ZSTD_initStaticCDict( + void* workspace, size_t workspaceSize, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_compressionParameters cParams); + +ZSTDLIB_STATIC_API const ZSTD_DDict* ZSTD_initStaticDDict( + void* workspace, size_t workspaceSize, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType); + + +/*! Custom memory allocation : + * These prototypes make it possible to pass your own allocation/free functions. + * ZSTD_customMem is provided at creation time, using ZSTD_create*_advanced() variants listed below. + * All allocation/free operations will be completed using these custom variants instead of regular ones. + */ +typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size); +typedef void (*ZSTD_freeFunction) (void* opaque, void* address); +typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem; +static +#ifdef __GNUC__ +__attribute__((__unused__)) +#endif +ZSTD_customMem const ZSTD_defaultCMem = { NULL, NULL, NULL }; /**< this constant defers to stdlib's functions */ + +ZSTDLIB_STATIC_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); +ZSTDLIB_STATIC_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); +ZSTDLIB_STATIC_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); +ZSTDLIB_STATIC_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); + +ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_compressionParameters cParams, + ZSTD_customMem customMem); + +/*! Thread pool : + * These prototypes make it possible to share a thread pool among multiple compression contexts. + * This can limit resources for applications with multiple threads where each one uses + * a threaded compression mode (via ZSTD_c_nbWorkers parameter). + * ZSTD_createThreadPool creates a new thread pool with a given number of threads. + * Note that the lifetime of such pool must exist while being used. + * ZSTD_CCtx_refThreadPool assigns a thread pool to a context (use NULL argument value + * to use an internal thread pool). + * ZSTD_freeThreadPool frees a thread pool, accepts NULL pointer. + */ +typedef struct POOL_ctx_s ZSTD_threadPool; +ZSTDLIB_STATIC_API ZSTD_threadPool* ZSTD_createThreadPool(size_t numThreads); +ZSTDLIB_STATIC_API void ZSTD_freeThreadPool (ZSTD_threadPool* pool); /* accept NULL pointer */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refThreadPool(ZSTD_CCtx* cctx, ZSTD_threadPool* pool); + + +/* + * This API is temporary and is expected to change or disappear in the future! + */ +ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_advanced2( + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + const ZSTD_CCtx_params* cctxParams, + ZSTD_customMem customMem); + +ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_advanced( + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType, + ZSTD_customMem customMem); + + +/*************************************** +* Advanced compression functions +***************************************/ + +/*! ZSTD_createCDict_byReference() : + * Create a digested dictionary for compression + * Dictionary content is just referenced, not duplicated. + * As a consequence, `dictBuffer` **must** outlive CDict, + * and its content must remain unmodified throughout the lifetime of CDict. + * note: equivalent to ZSTD_createCDict_advanced(), with dictLoadMethod==ZSTD_dlm_byRef */ +ZSTDLIB_STATIC_API ZSTD_CDict* ZSTD_createCDict_byReference(const void* dictBuffer, size_t dictSize, int compressionLevel); + +/*! ZSTD_getCParams() : + * @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize. + * `estimatedSrcSize` value is optional, select 0 if not known */ +ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); + +/*! ZSTD_getParams() : + * same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`. + * All fields of `ZSTD_frameParameters` are set to default : contentSize=1, checksum=0, noDictID=0 */ +ZSTDLIB_STATIC_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); + +/*! ZSTD_checkCParams() : + * Ensure param values remain within authorized range. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()) */ +ZSTDLIB_STATIC_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); + +/*! ZSTD_adjustCParams() : + * optimize params for a given `srcSize` and `dictSize`. + * `srcSize` can be unknown, in which case use ZSTD_CONTENTSIZE_UNKNOWN. + * `dictSize` must be `0` when there is no dictionary. + * cPar can be invalid : all parameters will be clamped within valid range in the @return struct. + * This function never fails (wide contract) */ +ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize); + +/*! ZSTD_CCtx_setCParams() : + * Set all parameters provided within @p cparams into the working @p cctx. + * Note : if modifying parameters during compression (MT mode only), + * note that changes to the .windowLog parameter will be ignored. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()). + * On failure, no parameters are updated. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams); + +/*! ZSTD_CCtx_setFParams() : + * Set all parameters provided within @p fparams into the working @p cctx. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setFParams(ZSTD_CCtx* cctx, ZSTD_frameParameters fparams); + +/*! ZSTD_CCtx_setParams() : + * Set all parameters provided within @p params into the working @p cctx. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParams(ZSTD_CCtx* cctx, ZSTD_parameters params); + +/*! ZSTD_compress_advanced() : + * Note : this function is now DEPRECATED. + * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters. + * This prototype will generate compilation warnings. */ +ZSTD_DEPRECATED("use ZSTD_compress2") +ZSTDLIB_STATIC_API +size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + ZSTD_parameters params); + +/*! ZSTD_compress_usingCDict_advanced() : + * Note : this function is now DEPRECATED. + * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters. + * This prototype will generate compilation warnings. */ +ZSTD_DEPRECATED("use ZSTD_compress2 with ZSTD_CCtx_loadDictionary") +ZSTDLIB_STATIC_API +size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict, + ZSTD_frameParameters fParams); + + +/*! ZSTD_CCtx_loadDictionary_byReference() : + * Same as ZSTD_CCtx_loadDictionary(), but dictionary content is referenced, instead of being copied into CCtx. + * It saves some memory, but also requires that `dict` outlives its usage within `cctx` */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_byReference(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); + +/*! ZSTD_CCtx_loadDictionary_advanced() : + * Same as ZSTD_CCtx_loadDictionary(), but gives finer control over + * how to load the dictionary (by copy ? by reference ?) + * and how to interpret it (automatic ? force raw mode ? full mode only ?) */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_loadDictionary_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); + +/*! ZSTD_CCtx_refPrefix_advanced() : + * Same as ZSTD_CCtx_refPrefix(), but gives finer control over + * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); + +/* === experimental parameters === */ +/* these parameters can be used with ZSTD_setParameter() + * they are not guaranteed to remain supported in the future */ + + /* Enables rsyncable mode, + * which makes compressed files more rsync friendly + * by adding periodic synchronization points to the compressed data. + * The target average block size is ZSTD_c_jobSize / 2. + * It's possible to modify the job size to increase or decrease + * the granularity of the synchronization point. + * Once the jobSize is smaller than the window size, + * it will result in compression ratio degradation. + * NOTE 1: rsyncable mode only works when multithreading is enabled. + * NOTE 2: rsyncable performs poorly in combination with long range mode, + * since it will decrease the effectiveness of synchronization points, + * though mileage may vary. + * NOTE 3: Rsyncable mode limits maximum compression speed to ~400 MB/s. + * If the selected compression level is already running significantly slower, + * the overall speed won't be significantly impacted. + */ + #define ZSTD_c_rsyncable ZSTD_c_experimentalParam1 + +/* Select a compression format. + * The value must be of type ZSTD_format_e. + * See ZSTD_format_e enum definition for details */ +#define ZSTD_c_format ZSTD_c_experimentalParam2 + +/* Force back-reference distances to remain < windowSize, + * even when referencing into Dictionary content (default:0) */ +#define ZSTD_c_forceMaxWindow ZSTD_c_experimentalParam3 + +/* Controls whether the contents of a CDict + * are used in place, or copied into the working context. + * Accepts values from the ZSTD_dictAttachPref_e enum. + * See the comments on that enum for an explanation of the feature. */ +#define ZSTD_c_forceAttachDict ZSTD_c_experimentalParam4 + +/* Controlled with ZSTD_paramSwitch_e enum. + * Default is ZSTD_ps_auto. + * Set to ZSTD_ps_disable to never compress literals. + * Set to ZSTD_ps_enable to always compress literals. (Note: uncompressed literals + * may still be emitted if huffman is not beneficial to use.) + * + * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use + * literals compression based on the compression parameters - specifically, + * negative compression levels do not use literal compression. + */ +#define ZSTD_c_literalCompressionMode ZSTD_c_experimentalParam5 + +/* User's best guess of source size. + * Hint is not valid when srcSizeHint == 0. + * There is no guarantee that hint is close to actual source size, + * but compression ratio may regress significantly if guess considerably underestimates */ +#define ZSTD_c_srcSizeHint ZSTD_c_experimentalParam7 + +/* Controls whether the new and experimental "dedicated dictionary search + * structure" can be used. This feature is still rough around the edges, be + * prepared for surprising behavior! + * + * How to use it: + * + * When using a CDict, whether to use this feature or not is controlled at + * CDict creation, and it must be set in a CCtxParams set passed into that + * construction (via ZSTD_createCDict_advanced2()). A compression will then + * use the feature or not based on how the CDict was constructed; the value of + * this param, set in the CCtx, will have no effect. + * + * However, when a dictionary buffer is passed into a CCtx, such as via + * ZSTD_CCtx_loadDictionary(), this param can be set on the CCtx to control + * whether the CDict that is created internally can use the feature or not. + * + * What it does: + * + * Normally, the internal data structures of the CDict are analogous to what + * would be stored in a CCtx after compressing the contents of a dictionary. + * To an approximation, a compression using a dictionary can then use those + * data structures to simply continue what is effectively a streaming + * compression where the simulated compression of the dictionary left off. + * Which is to say, the search structures in the CDict are normally the same + * format as in the CCtx. + * + * It is possible to do better, since the CDict is not like a CCtx: the search + * structures are written once during CDict creation, and then are only read + * after that, while the search structures in the CCtx are both read and + * written as the compression goes along. This means we can choose a search + * structure for the dictionary that is read-optimized. + * + * This feature enables the use of that different structure. + * + * Note that some of the members of the ZSTD_compressionParameters struct have + * different semantics and constraints in the dedicated search structure. It is + * highly recommended that you simply set a compression level in the CCtxParams + * you pass into the CDict creation call, and avoid messing with the cParams + * directly. + * + * Effects: + * + * This will only have any effect when the selected ZSTD_strategy + * implementation supports this feature. Currently, that's limited to + * ZSTD_greedy, ZSTD_lazy, and ZSTD_lazy2. + * + * Note that this means that the CDict tables can no longer be copied into the + * CCtx, so the dict attachment mode ZSTD_dictForceCopy will no longer be + * usable. The dictionary can only be attached or reloaded. + * + * In general, you should expect compression to be faster--sometimes very much + * so--and CDict creation to be slightly slower. Eventually, we will probably + * make this mode the default. + */ +#define ZSTD_c_enableDedicatedDictSearch ZSTD_c_experimentalParam8 + +/* ZSTD_c_stableInBuffer + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable. + * + * Tells the compressor that input data presented with ZSTD_inBuffer + * will ALWAYS be the same between calls. + * Technically, the @src pointer must never be changed, + * and the @pos field can only be updated by zstd. + * However, it's possible to increase the @size field, + * allowing scenarios where more data can be appended after compressions starts. + * These conditions are checked by the compressor, + * and compression will fail if they are not respected. + * Also, data in the ZSTD_inBuffer within the range [src, src + pos) + * MUST not be modified during compression or it will result in data corruption. + * + * When this flag is enabled zstd won't allocate an input window buffer, + * because the user guarantees it can reference the ZSTD_inBuffer until + * the frame is complete. But, it will still allocate an output buffer + * large enough to fit a block (see ZSTD_c_stableOutBuffer). This will also + * avoid the memcpy() from the input buffer to the input window buffer. + * + * NOTE: So long as the ZSTD_inBuffer always points to valid memory, using + * this flag is ALWAYS memory safe, and will never access out-of-bounds + * memory. However, compression WILL fail if conditions are not respected. + * + * WARNING: The data in the ZSTD_inBuffer in the range [src, src + pos) MUST + * not be modified during compression or it will result in data corruption. + * This is because zstd needs to reference data in the ZSTD_inBuffer to find + * matches. Normally zstd maintains its own window buffer for this purpose, + * but passing this flag tells zstd to rely on user provided buffer instead. + */ +#define ZSTD_c_stableInBuffer ZSTD_c_experimentalParam9 + +/* ZSTD_c_stableOutBuffer + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable. + * + * Tells he compressor that the ZSTD_outBuffer will not be resized between + * calls. Specifically: (out.size - out.pos) will never grow. This gives the + * compressor the freedom to say: If the compressed data doesn't fit in the + * output buffer then return ZSTD_error_dstSizeTooSmall. This allows us to + * always decompress directly into the output buffer, instead of decompressing + * into an internal buffer and copying to the output buffer. + * + * When this flag is enabled zstd won't allocate an output buffer, because + * it can write directly to the ZSTD_outBuffer. It will still allocate the + * input window buffer (see ZSTD_c_stableInBuffer). + * + * Zstd will check that (out.size - out.pos) never grows and return an error + * if it does. While not strictly necessary, this should prevent surprises. + */ +#define ZSTD_c_stableOutBuffer ZSTD_c_experimentalParam10 + +/* ZSTD_c_blockDelimiters + * Default is 0 == ZSTD_sf_noBlockDelimiters. + * + * For use with sequence compression API: ZSTD_compressSequences(). + * + * Designates whether or not the given array of ZSTD_Sequence contains block delimiters + * and last literals, which are defined as sequences with offset == 0 and matchLength == 0. + * See the definition of ZSTD_Sequence for more specifics. + */ +#define ZSTD_c_blockDelimiters ZSTD_c_experimentalParam11 + +/* ZSTD_c_validateSequences + * Default is 0 == disabled. Set to 1 to enable sequence validation. + * + * For use with sequence compression API: ZSTD_compressSequences(). + * Designates whether or not we validate sequences provided to ZSTD_compressSequences() + * during function execution. + * + * Without validation, providing a sequence that does not conform to the zstd spec will cause + * undefined behavior, and may produce a corrupted block. + * + * With validation enabled, if sequence is invalid (see doc/zstd_compression_format.md for + * specifics regarding offset/matchlength requirements) then the function will bail out and + * return an error. + * + */ +#define ZSTD_c_validateSequences ZSTD_c_experimentalParam12 + +/* ZSTD_c_useBlockSplitter + * Controlled with ZSTD_paramSwitch_e enum. + * Default is ZSTD_ps_auto. + * Set to ZSTD_ps_disable to never use block splitter. + * Set to ZSTD_ps_enable to always use block splitter. + * + * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use + * block splitting based on the compression parameters. + */ +#define ZSTD_c_useBlockSplitter ZSTD_c_experimentalParam13 + +/* ZSTD_c_useRowMatchFinder + * Controlled with ZSTD_paramSwitch_e enum. + * Default is ZSTD_ps_auto. + * Set to ZSTD_ps_disable to never use row-based matchfinder. + * Set to ZSTD_ps_enable to force usage of row-based matchfinder. + * + * By default, in ZSTD_ps_auto, the library will decide at runtime whether to use + * the row-based matchfinder based on support for SIMD instructions and the window log. + * Note that this only pertains to compression strategies: greedy, lazy, and lazy2 + */ +#define ZSTD_c_useRowMatchFinder ZSTD_c_experimentalParam14 + +/* ZSTD_c_deterministicRefPrefix + * Default is 0 == disabled. Set to 1 to enable. + * + * Zstd produces different results for prefix compression when the prefix is + * directly adjacent to the data about to be compressed vs. when it isn't. + * This is because zstd detects that the two buffers are contiguous and it can + * use a more efficient match finding algorithm. However, this produces different + * results than when the two buffers are non-contiguous. This flag forces zstd + * to always load the prefix in non-contiguous mode, even if it happens to be + * adjacent to the data, to guarantee determinism. + * + * If you really care about determinism when using a dictionary or prefix, + * like when doing delta compression, you should select this option. It comes + * at a speed penalty of about ~2.5% if the dictionary and data happened to be + * contiguous, and is free if they weren't contiguous. We don't expect that + * intentionally making the dictionary and data contiguous will be worth the + * cost to memcpy() the data. + */ +#define ZSTD_c_deterministicRefPrefix ZSTD_c_experimentalParam15 + +/* ZSTD_c_prefetchCDictTables + * Controlled with ZSTD_paramSwitch_e enum. Default is ZSTD_ps_auto. + * + * In some situations, zstd uses CDict tables in-place rather than copying them + * into the working context. (See docs on ZSTD_dictAttachPref_e above for details). + * In such situations, compression speed is seriously impacted when CDict tables are + * "cold" (outside CPU cache). This parameter instructs zstd to prefetch CDict tables + * when they are used in-place. + * + * For sufficiently small inputs, the cost of the prefetch will outweigh the benefit. + * For sufficiently large inputs, zstd will by default memcpy() CDict tables + * into the working context, so there is no need to prefetch. This parameter is + * targeted at a middle range of input sizes, where a prefetch is cheap enough to be + * useful but memcpy() is too expensive. The exact range of input sizes where this + * makes sense is best determined by careful experimentation. + * + * Note: for this parameter, ZSTD_ps_auto is currently equivalent to ZSTD_ps_disable, + * but in the future zstd may conditionally enable this feature via an auto-detection + * heuristic for cold CDicts. + * Use ZSTD_ps_disable to opt out of prefetching under any circumstances. + */ +#define ZSTD_c_prefetchCDictTables ZSTD_c_experimentalParam16 + +/* ZSTD_c_enableSeqProducerFallback + * Allowed values are 0 (disable) and 1 (enable). The default setting is 0. + * + * Controls whether zstd will fall back to an internal sequence producer if an + * external sequence producer is registered and returns an error code. This fallback + * is block-by-block: the internal sequence producer will only be called for blocks + * where the external sequence producer returns an error code. Fallback parsing will + * follow any other cParam settings, such as compression level, the same as in a + * normal (fully-internal) compression operation. + * + * The user is strongly encouraged to read the full Block-Level Sequence Producer API + * documentation (below) before setting this parameter. */ +#define ZSTD_c_enableSeqProducerFallback ZSTD_c_experimentalParam17 + +/* ZSTD_c_maxBlockSize + * Allowed values are between 1KB and ZSTD_BLOCKSIZE_MAX (128KB). + * The default is ZSTD_BLOCKSIZE_MAX, and setting to 0 will set to the default. + * + * This parameter can be used to set an upper bound on the blocksize + * that overrides the default ZSTD_BLOCKSIZE_MAX. It cannot be used to set upper + * bounds greater than ZSTD_BLOCKSIZE_MAX or bounds lower than 1KB (will make + * compressBound() inaccurate). Only currently meant to be used for testing. + * + */ +#define ZSTD_c_maxBlockSize ZSTD_c_experimentalParam18 + +/* ZSTD_c_searchForExternalRepcodes + * This parameter affects how zstd parses external sequences, such as sequences + * provided through the compressSequences() API or from an external block-level + * sequence producer. + * + * If set to ZSTD_ps_enable, the library will check for repeated offsets in + * external sequences, even if those repcodes are not explicitly indicated in + * the "rep" field. Note that this is the only way to exploit repcode matches + * while using compressSequences() or an external sequence producer, since zstd + * currently ignores the "rep" field of external sequences. + * + * If set to ZSTD_ps_disable, the library will not exploit repeated offsets in + * external sequences, regardless of whether the "rep" field has been set. This + * reduces sequence compression overhead by about 25% while sacrificing some + * compression ratio. + * + * The default value is ZSTD_ps_auto, for which the library will enable/disable + * based on compression level. + * + * Note: for now, this param only has an effect if ZSTD_c_blockDelimiters is + * set to ZSTD_sf_explicitBlockDelimiters. That may change in the future. + */ +#define ZSTD_c_searchForExternalRepcodes ZSTD_c_experimentalParam19 + +/*! ZSTD_CCtx_getParameter() : + * Get the requested compression parameter value, selected by enum ZSTD_cParameter, + * and store it into int* value. + * @return : 0, or an error code (which can be tested with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_getParameter(const ZSTD_CCtx* cctx, ZSTD_cParameter param, int* value); + + +/*! ZSTD_CCtx_params : + * Quick howto : + * - ZSTD_createCCtxParams() : Create a ZSTD_CCtx_params structure + * - ZSTD_CCtxParams_setParameter() : Push parameters one by one into + * an existing ZSTD_CCtx_params structure. + * This is similar to + * ZSTD_CCtx_setParameter(). + * - ZSTD_CCtx_setParametersUsingCCtxParams() : Apply parameters to + * an existing CCtx. + * These parameters will be applied to + * all subsequent frames. + * - ZSTD_compressStream2() : Do compression using the CCtx. + * - ZSTD_freeCCtxParams() : Free the memory, accept NULL pointer. + * + * This can be used with ZSTD_estimateCCtxSize_advanced_usingCCtxParams() + * for static allocation of CCtx for single-threaded compression. + */ +ZSTDLIB_STATIC_API ZSTD_CCtx_params* ZSTD_createCCtxParams(void); +ZSTDLIB_STATIC_API size_t ZSTD_freeCCtxParams(ZSTD_CCtx_params* params); /* accept NULL pointer */ + +/*! ZSTD_CCtxParams_reset() : + * Reset params to default values. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_reset(ZSTD_CCtx_params* params); + +/*! ZSTD_CCtxParams_init() : + * Initializes the compression parameters of cctxParams according to + * compression level. All other parameters are reset to their default values. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel); + +/*! ZSTD_CCtxParams_init_advanced() : + * Initializes the compression and frame parameters of cctxParams according to + * params. All other parameters are reset to their default values. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_parameters params); + +/*! ZSTD_CCtxParams_setParameter() : Requires v1.4.0+ + * Similar to ZSTD_CCtx_setParameter. + * Set one compression parameter, selected by enum ZSTD_cParameter. + * Parameters must be applied to a ZSTD_CCtx using + * ZSTD_CCtx_setParametersUsingCCtxParams(). + * @result : a code representing success or failure (which can be tested with + * ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* params, ZSTD_cParameter param, int value); + +/*! ZSTD_CCtxParams_getParameter() : + * Similar to ZSTD_CCtx_getParameter. + * Get the requested value of one compression parameter, selected by enum ZSTD_cParameter. + * @result : 0, or an error code (which can be tested with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtxParams_getParameter(const ZSTD_CCtx_params* params, ZSTD_cParameter param, int* value); + +/*! ZSTD_CCtx_setParametersUsingCCtxParams() : + * Apply a set of ZSTD_CCtx_params to the compression context. + * This can be done even after compression is started, + * if nbWorkers==0, this will have no impact until a new compression is started. + * if nbWorkers>=1, new parameters will be picked up at next job, + * with a few restrictions (windowLog, pledgedSrcSize, nbWorkers, jobSize, and overlapLog are not updated). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParametersUsingCCtxParams( + ZSTD_CCtx* cctx, const ZSTD_CCtx_params* params); + +/*! ZSTD_compressStream2_simpleArgs() : + * Same as ZSTD_compressStream2(), + * but using only integral types as arguments. + * This variant might be helpful for binders from dynamic languages + * which have troubles handling structures containing memory pointers. + */ +ZSTDLIB_STATIC_API size_t ZSTD_compressStream2_simpleArgs ( + ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, size_t* dstPos, + const void* src, size_t srcSize, size_t* srcPos, + ZSTD_EndDirective endOp); + + +/*************************************** +* Advanced decompression functions +***************************************/ + +/*! ZSTD_isFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier. + * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. + * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled. + * Note 3 : Skippable Frame Identifiers are considered valid. */ +ZSTDLIB_STATIC_API unsigned ZSTD_isFrame(const void* buffer, size_t size); + +/*! ZSTD_createDDict_byReference() : + * Create a digested dictionary, ready to start decompression operation without startup delay. + * Dictionary content is referenced, and therefore stays in dictBuffer. + * It is important that dictBuffer outlives DDict, + * it must remain read accessible throughout the lifetime of DDict */ +ZSTDLIB_STATIC_API ZSTD_DDict* ZSTD_createDDict_byReference(const void* dictBuffer, size_t dictSize); + +/*! ZSTD_DCtx_loadDictionary_byReference() : + * Same as ZSTD_DCtx_loadDictionary(), + * but references `dict` content instead of copying it into `dctx`. + * This saves memory if `dict` remains around., + * However, it's imperative that `dict` remains accessible (and unmodified) while being used, so it must outlive decompression. */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_byReference(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); + +/*! ZSTD_DCtx_loadDictionary_advanced() : + * Same as ZSTD_DCtx_loadDictionary(), + * but gives direct control over + * how to load the dictionary (by copy ? by reference ?) + * and how to interpret it (automatic ? force raw mode ? full mode only ?). */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_loadDictionary_advanced(ZSTD_DCtx* dctx, const void* dict, size_t dictSize, ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType); + +/*! ZSTD_DCtx_refPrefix_advanced() : + * Same as ZSTD_DCtx_refPrefix(), but gives finer control over + * how to interpret prefix content (automatic ? force raw mode (default) ? full mode only ?) */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_refPrefix_advanced(ZSTD_DCtx* dctx, const void* prefix, size_t prefixSize, ZSTD_dictContentType_e dictContentType); + +/*! ZSTD_DCtx_setMaxWindowSize() : + * Refuses allocating internal buffers for frames requiring a window size larger than provided limit. + * This protects a decoder context from reserving too much memory for itself (potential attack scenario). + * This parameter is only useful in streaming mode, since no internal buffer is allocated in single-pass mode. + * By default, a decompression context accepts all window sizes <= (1 << ZSTD_WINDOWLOG_LIMIT_DEFAULT) + * @return : 0, or an error code (which can be tested using ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_setMaxWindowSize(ZSTD_DCtx* dctx, size_t maxWindowSize); + +/*! ZSTD_DCtx_getParameter() : + * Get the requested decompression parameter value, selected by enum ZSTD_dParameter, + * and store it into int* value. + * @return : 0, or an error code (which can be tested with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value); + +/* ZSTD_d_format + * experimental parameter, + * allowing selection between ZSTD_format_e input compression formats + */ +#define ZSTD_d_format ZSTD_d_experimentalParam1 +/* ZSTD_d_stableOutBuffer + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable. + * + * Tells the decompressor that the ZSTD_outBuffer will ALWAYS be the same + * between calls, except for the modifications that zstd makes to pos (the + * caller must not modify pos). This is checked by the decompressor, and + * decompression will fail if it ever changes. Therefore the ZSTD_outBuffer + * MUST be large enough to fit the entire decompressed frame. This will be + * checked when the frame content size is known. The data in the ZSTD_outBuffer + * in the range [dst, dst + pos) MUST not be modified during decompression + * or you will get data corruption. + * + * When this flag is enabled zstd won't allocate an output buffer, because + * it can write directly to the ZSTD_outBuffer, but it will still allocate + * an input buffer large enough to fit any compressed block. This will also + * avoid the memcpy() from the internal output buffer to the ZSTD_outBuffer. + * If you need to avoid the input buffer allocation use the buffer-less + * streaming API. + * + * NOTE: So long as the ZSTD_outBuffer always points to valid memory, using + * this flag is ALWAYS memory safe, and will never access out-of-bounds + * memory. However, decompression WILL fail if you violate the preconditions. + * + * WARNING: The data in the ZSTD_outBuffer in the range [dst, dst + pos) MUST + * not be modified during decompression or you will get data corruption. This + * is because zstd needs to reference data in the ZSTD_outBuffer to regenerate + * matches. Normally zstd maintains its own buffer for this purpose, but passing + * this flag tells zstd to use the user provided buffer. + */ +#define ZSTD_d_stableOutBuffer ZSTD_d_experimentalParam2 + +/* ZSTD_d_forceIgnoreChecksum + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable + * + * Tells the decompressor to skip checksum validation during decompression, regardless + * of whether checksumming was specified during compression. This offers some + * slight performance benefits, and may be useful for debugging. + * Param has values of type ZSTD_forceIgnoreChecksum_e + */ +#define ZSTD_d_forceIgnoreChecksum ZSTD_d_experimentalParam3 + +/* ZSTD_d_refMultipleDDicts + * Experimental parameter. + * Default is 0 == disabled. Set to 1 to enable + * + * If enabled and dctx is allocated on the heap, then additional memory will be allocated + * to store references to multiple ZSTD_DDict. That is, multiple calls of ZSTD_refDDict() + * using a given ZSTD_DCtx, rather than overwriting the previous DDict reference, will instead + * store all references. At decompression time, the appropriate dictID is selected + * from the set of DDicts based on the dictID in the frame. + * + * Usage is simply calling ZSTD_refDDict() on multiple dict buffers. + * + * Param has values of byte ZSTD_refMultipleDDicts_e + * + * WARNING: Enabling this parameter and calling ZSTD_DCtx_refDDict(), will trigger memory + * allocation for the hash table. ZSTD_freeDCtx() also frees this memory. + * Memory is allocated as per ZSTD_DCtx::customMem. + * + * Although this function allocates memory for the table, the user is still responsible for + * memory management of the underlying ZSTD_DDict* themselves. + */ +#define ZSTD_d_refMultipleDDicts ZSTD_d_experimentalParam4 + +/* ZSTD_d_disableHuffmanAssembly + * Set to 1 to disable the Huffman assembly implementation. + * The default value is 0, which allows zstd to use the Huffman assembly + * implementation if available. + * + * This parameter can be used to disable Huffman assembly at runtime. + * If you want to disable it at compile time you can define the macro + * ZSTD_DISABLE_ASM. + */ +#define ZSTD_d_disableHuffmanAssembly ZSTD_d_experimentalParam5 + +/* ZSTD_d_maxBlockSize + * Allowed values are between 1KB and ZSTD_BLOCKSIZE_MAX (128KB). + * The default is ZSTD_BLOCKSIZE_MAX, and setting to 0 will set to the default. + * + * Forces the decompressor to reject blocks whose content size is + * larger than the configured maxBlockSize. When maxBlockSize is + * larger than the windowSize, the windowSize is used instead. + * This saves memory on the decoder when you know all blocks are small. + * + * This option is typically used in conjunction with ZSTD_c_maxBlockSize. + * + * WARNING: This causes the decoder to reject otherwise valid frames + * that have block sizes larger than the configured maxBlockSize. + */ +#define ZSTD_d_maxBlockSize ZSTD_d_experimentalParam6 + + +/*! ZSTD_DCtx_setFormat() : + * This function is REDUNDANT. Prefer ZSTD_DCtx_setParameter(). + * Instruct the decoder context about what kind of data to decode next. + * This instruction is mandatory to decode data without a fully-formed header, + * such ZSTD_f_zstd1_magicless for example. + * @return : 0, or an error code (which can be tested using ZSTD_isError()). */ +ZSTD_DEPRECATED("use ZSTD_DCtx_setParameter() instead") +ZSTDLIB_STATIC_API +size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format); + +/*! ZSTD_decompressStream_simpleArgs() : + * Same as ZSTD_decompressStream(), + * but using only integral types as arguments. + * This can be helpful for binders from dynamic languages + * which have troubles handling structures containing memory pointers. + */ +ZSTDLIB_STATIC_API size_t ZSTD_decompressStream_simpleArgs ( + ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, size_t* dstPos, + const void* src, size_t srcSize, size_t* srcPos); + + +/******************************************************************** +* Advanced streaming functions +* Warning : most of these functions are now redundant with the Advanced API. +* Once Advanced API reaches "stable" status, +* redundant functions will be deprecated, and then at some point removed. +********************************************************************/ + +/*===== Advanced Streaming compression functions =====*/ + +/*! ZSTD_initCStream_srcSize() : + * This function is DEPRECATED, and equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) + * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * + * pledgedSrcSize must be correct. If it is not known at init time, use + * ZSTD_CONTENTSIZE_UNKNOWN. Note that, for compatibility with older programs, + * "0" also disables frame content size field. It may be enabled in the future. + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, + int compressionLevel, + unsigned long long pledgedSrcSize); + +/*! ZSTD_initCStream_usingDict() : + * This function is DEPRECATED, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); + * + * Creates of an internal CDict (incompatible with static CCtx), except if + * dict == NULL or dictSize < 8, in which case no dict is used. + * Note: dict is loaded with ZSTD_dct_auto (treated as a full zstd dictionary if + * it begins with ZSTD_MAGIC_DICTIONARY, else as raw content) and ZSTD_dlm_byCopy. + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + int compressionLevel); + +/*! ZSTD_initCStream_advanced() : + * This function is DEPRECATED, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setParams(zcs, params); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); + * + * dict is loaded with ZSTD_dct_auto and ZSTD_dlm_byCopy. + * pledgedSrcSize must be correct. + * If srcSize is not known at init time, use value ZSTD_CONTENTSIZE_UNKNOWN. + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + ZSTD_parameters params, + unsigned long long pledgedSrcSize); + +/*! ZSTD_initCStream_usingCDict() : + * This function is DEPRECATED, and equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_refCDict(zcs, cdict); + * + * note : cdict will just be referenced, and must outlive compression session + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); + +/*! ZSTD_initCStream_usingCDict_advanced() : + * This function is DEPRECATED, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setFParams(zcs, fParams); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * ZSTD_CCtx_refCDict(zcs, cdict); + * + * same as ZSTD_initCStream_usingCDict(), with control over frame parameters. + * pledgedSrcSize must be correct. If srcSize is not known at init time, use + * value ZSTD_CONTENTSIZE_UNKNOWN. + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, + const ZSTD_CDict* cdict, + ZSTD_frameParameters fParams, + unsigned long long pledgedSrcSize); + +/*! ZSTD_resetCStream() : + * This function is DEPRECATED, and is equivalent to: + * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); + * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); + * Note: ZSTD_resetCStream() interprets pledgedSrcSize == 0 as ZSTD_CONTENTSIZE_UNKNOWN, but + * ZSTD_CCtx_setPledgedSrcSize() does not do the same, so ZSTD_CONTENTSIZE_UNKNOWN must be + * explicitly specified. + * + * start a new frame, using same parameters from previous frame. + * This is typically useful to skip dictionary loading stage, since it will reuse it in-place. + * Note that zcs must be init at least once before using ZSTD_resetCStream(). + * If pledgedSrcSize is not known at reset time, use macro ZSTD_CONTENTSIZE_UNKNOWN. + * If pledgedSrcSize > 0, its value must be correct, as it will be written in header, and controlled at the end. + * For the time being, pledgedSrcSize==0 is interpreted as "srcSize unknown" for compatibility with older programs, + * but it will change to mean "empty" in future version, so use macro ZSTD_CONTENTSIZE_UNKNOWN instead. + * @return : 0, or an error code (which can be tested using ZSTD_isError()) + * This prototype will generate compilation warnings. + */ +ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API +size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); + + +typedef struct { + unsigned long long ingested; /* nb input bytes read and buffered */ + unsigned long long consumed; /* nb input bytes actually compressed */ + unsigned long long produced; /* nb of compressed bytes generated and buffered */ + unsigned long long flushed; /* nb of compressed bytes flushed : not provided; can be tracked from caller side */ + unsigned currentJobID; /* MT only : latest started job nb */ + unsigned nbActiveWorkers; /* MT only : nb of workers actively compressing at probe time */ +} ZSTD_frameProgression; + +/* ZSTD_getFrameProgression() : + * tells how much data has been ingested (read from input) + * consumed (input actually compressed) and produced (output) for current frame. + * Note : (ingested - consumed) is amount of input data buffered internally, not yet compressed. + * Aggregates progression inside active worker threads. + */ +ZSTDLIB_STATIC_API ZSTD_frameProgression ZSTD_getFrameProgression(const ZSTD_CCtx* cctx); + +/*! ZSTD_toFlushNow() : + * Tell how many bytes are ready to be flushed immediately. + * Useful for multithreading scenarios (nbWorkers >= 1). + * Probe the oldest active job, defined as oldest job not yet entirely flushed, + * and check its output buffer. + * @return : amount of data stored in oldest job and ready to be flushed immediately. + * if @return == 0, it means either : + * + there is no active job (could be checked with ZSTD_frameProgression()), or + * + oldest job is still actively compressing data, + * but everything it has produced has also been flushed so far, + * therefore flush speed is limited by production speed of oldest job + * irrespective of the speed of concurrent (and newer) jobs. + */ +ZSTDLIB_STATIC_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx); + + +/*===== Advanced Streaming decompression functions =====*/ + +/*! + * This function is deprecated, and is equivalent to: + * + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * ZSTD_DCtx_loadDictionary(zds, dict, dictSize); + * + * note: no dictionary will be used if dict == NULL or dictSize < 8 + */ +ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_loadDictionary, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); + +/*! + * This function is deprecated, and is equivalent to: + * + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * ZSTD_DCtx_refDDict(zds, ddict); + * + * note : ddict is referenced, it must outlive decompression session + */ +ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_refDDict, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); + +/*! + * This function is deprecated, and is equivalent to: + * + * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); + * + * reuse decompression parameters from previous init; saves dictionary loading + */ +ZSTD_DEPRECATED("use ZSTD_DCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); + + +/* ********************* BLOCK-LEVEL SEQUENCE PRODUCER API ********************* + * + * *** OVERVIEW *** + * The Block-Level Sequence Producer API allows users to provide their own custom + * sequence producer which libzstd invokes to process each block. The produced list + * of sequences (literals and matches) is then post-processed by libzstd to produce + * valid compressed blocks. + * + * This block-level offload API is a more granular complement of the existing + * frame-level offload API compressSequences() (introduced in v1.5.1). It offers + * an easier migration story for applications already integrated with libzstd: the + * user application continues to invoke the same compression functions + * ZSTD_compress2() or ZSTD_compressStream2() as usual, and transparently benefits + * from the specific advantages of the external sequence producer. For example, + * the sequence producer could be tuned to take advantage of known characteristics + * of the input, to offer better speed / ratio, or could leverage hardware + * acceleration not available within libzstd itself. + * + * See contrib/externalSequenceProducer for an example program employing the + * Block-Level Sequence Producer API. + * + * *** USAGE *** + * The user is responsible for implementing a function of type + * ZSTD_sequenceProducer_F. For each block, zstd will pass the following + * arguments to the user-provided function: + * + * - sequenceProducerState: a pointer to a user-managed state for the sequence + * producer. + * + * - outSeqs, outSeqsCapacity: an output buffer for the sequence producer. + * outSeqsCapacity is guaranteed >= ZSTD_sequenceBound(srcSize). The memory + * backing outSeqs is managed by the CCtx. + * + * - src, srcSize: an input buffer for the sequence producer to parse. + * srcSize is guaranteed to be <= ZSTD_BLOCKSIZE_MAX. + * + * - dict, dictSize: a history buffer, which may be empty, which the sequence + * producer may reference as it parses the src buffer. Currently, zstd will + * always pass dictSize == 0 into external sequence producers, but this will + * change in the future. + * + * - compressionLevel: a signed integer representing the zstd compression level + * set by the user for the current operation. The sequence producer may choose + * to use this information to change its compression strategy and speed/ratio + * tradeoff. Note: the compression level does not reflect zstd parameters set + * through the advanced API. + * + * - windowSize: a size_t representing the maximum allowed offset for external + * sequences. Note that sequence offsets are sometimes allowed to exceed the + * windowSize if a dictionary is present, see doc/zstd_compression_format.md + * for details. + * + * The user-provided function shall return a size_t representing the number of + * sequences written to outSeqs. This return value will be treated as an error + * code if it is greater than outSeqsCapacity. The return value must be non-zero + * if srcSize is non-zero. The ZSTD_SEQUENCE_PRODUCER_ERROR macro is provided + * for convenience, but any value greater than outSeqsCapacity will be treated as + * an error code. + * + * If the user-provided function does not return an error code, the sequences + * written to outSeqs must be a valid parse of the src buffer. Data corruption may + * occur if the parse is not valid. A parse is defined to be valid if the + * following conditions hold: + * - The sum of matchLengths and literalLengths must equal srcSize. + * - All sequences in the parse, except for the final sequence, must have + * matchLength >= ZSTD_MINMATCH_MIN. The final sequence must have + * matchLength >= ZSTD_MINMATCH_MIN or matchLength == 0. + * - All offsets must respect the windowSize parameter as specified in + * doc/zstd_compression_format.md. + * - If the final sequence has matchLength == 0, it must also have offset == 0. + * + * zstd will only validate these conditions (and fail compression if they do not + * hold) if the ZSTD_c_validateSequences cParam is enabled. Note that sequence + * validation has a performance cost. + * + * If the user-provided function returns an error, zstd will either fall back + * to an internal sequence producer or fail the compression operation. The user can + * choose between the two behaviors by setting the ZSTD_c_enableSeqProducerFallback + * cParam. Fallback compression will follow any other cParam settings, such as + * compression level, the same as in a normal compression operation. + * + * The user shall instruct zstd to use a particular ZSTD_sequenceProducer_F + * function by calling + * ZSTD_registerSequenceProducer(cctx, + * sequenceProducerState, + * sequenceProducer) + * This setting will persist until the next parameter reset of the CCtx. + * + * The sequenceProducerState must be initialized by the user before calling + * ZSTD_registerSequenceProducer(). The user is responsible for destroying the + * sequenceProducerState. + * + * *** LIMITATIONS *** + * This API is compatible with all zstd compression APIs which respect advanced parameters. + * However, there are three limitations: + * + * First, the ZSTD_c_enableLongDistanceMatching cParam is not currently supported. + * COMPRESSION WILL FAIL if it is enabled and the user tries to compress with a block-level + * external sequence producer. + * - Note that ZSTD_c_enableLongDistanceMatching is auto-enabled by default in some + * cases (see its documentation for details). Users must explicitly set + * ZSTD_c_enableLongDistanceMatching to ZSTD_ps_disable in such cases if an external + * sequence producer is registered. + * - As of this writing, ZSTD_c_enableLongDistanceMatching is disabled by default + * whenever ZSTD_c_windowLog < 128MB, but that's subject to change. Users should + * check the docs on ZSTD_c_enableLongDistanceMatching whenever the Block-Level Sequence + * Producer API is used in conjunction with advanced settings (like ZSTD_c_windowLog). + * + * Second, history buffers are not currently supported. Concretely, zstd will always pass + * dictSize == 0 to the external sequence producer (for now). This has two implications: + * - Dictionaries are not currently supported. Compression will *not* fail if the user + * references a dictionary, but the dictionary won't have any effect. + * - Stream history is not currently supported. All advanced compression APIs, including + * streaming APIs, work with external sequence producers, but each block is treated as + * an independent chunk without history from previous blocks. + * + * Third, multi-threading within a single compression is not currently supported. In other words, + * COMPRESSION WILL FAIL if ZSTD_c_nbWorkers > 0 and an external sequence producer is registered. + * Multi-threading across compressions is fine: simply create one CCtx per thread. + * + * Long-term, we plan to overcome all three limitations. There is no technical blocker to + * overcoming them. It is purely a question of engineering effort. + */ + +#define ZSTD_SEQUENCE_PRODUCER_ERROR ((size_t)(-1)) + +typedef size_t (*ZSTD_sequenceProducer_F) ( + void* sequenceProducerState, + ZSTD_Sequence* outSeqs, size_t outSeqsCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize, + int compressionLevel, + size_t windowSize +); + +/*! ZSTD_registerSequenceProducer() : + * Instruct zstd to use a block-level external sequence producer function. + * + * The sequenceProducerState must be initialized by the caller, and the caller is + * responsible for managing its lifetime. This parameter is sticky across + * compressions. It will remain set until the user explicitly resets compression + * parameters. + * + * Sequence producer registration is considered to be an "advanced parameter", + * part of the "advanced API". This means it will only have an effect on compression + * APIs which respect advanced parameters, such as compress2() and compressStream2(). + * Older compression APIs such as compressCCtx(), which predate the introduction of + * "advanced parameters", will ignore any external sequence producer setting. + * + * The sequence producer can be "cleared" by registering a NULL function pointer. This + * removes all limitations described above in the "LIMITATIONS" section of the API docs. + * + * The user is strongly encouraged to read the full API documentation (above) before + * calling this function. */ +ZSTDLIB_STATIC_API void +ZSTD_registerSequenceProducer( + ZSTD_CCtx* cctx, + void* sequenceProducerState, + ZSTD_sequenceProducer_F sequenceProducer +); + +/*! ZSTD_CCtxParams_registerSequenceProducer() : + * Same as ZSTD_registerSequenceProducer(), but operates on ZSTD_CCtx_params. + * This is used for accurate size estimation with ZSTD_estimateCCtxSize_usingCCtxParams(), + * which is needed when creating a ZSTD_CCtx with ZSTD_initStaticCCtx(). + * + * If you are using the external sequence producer API in a scenario where ZSTD_initStaticCCtx() + * is required, then this function is for you. Otherwise, you probably don't need it. + * + * See tests/zstreamtest.c for example usage. */ +ZSTDLIB_STATIC_API void +ZSTD_CCtxParams_registerSequenceProducer( + ZSTD_CCtx_params* params, + void* sequenceProducerState, + ZSTD_sequenceProducer_F sequenceProducer +); + + +/********************************************************************* +* Buffer-less and synchronous inner streaming functions (DEPRECATED) +* +* This API is deprecated, and will be removed in a future version. +* It allows streaming (de)compression with user allocated buffers. +* However, it is hard to use, and not as well tested as the rest of +* our API. +* +* Please use the normal streaming API instead: ZSTD_compressStream2, +* and ZSTD_decompressStream. +* If there is functionality that you need, but it doesn't provide, +* please open an issue on our GitHub. +********************************************************************* */ + +/** + Buffer-less streaming compression (synchronous mode) + + A ZSTD_CCtx object is required to track streaming operations. + Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource. + ZSTD_CCtx object can be reused multiple times within successive compression operations. + + Start by initializing a context. + Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression. + + Then, consume your input using ZSTD_compressContinue(). + There are some important considerations to keep in mind when using this advanced function : + - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffers only. + - Interface is synchronous : input is consumed entirely and produces 1+ compressed blocks. + - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario. + Worst case evaluation is provided by ZSTD_compressBound(). + ZSTD_compressContinue() doesn't guarantee recover after a failed compression. + - ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, see WindowLog). + It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks) + - ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps. + In which case, it will "discard" the relevant memory section from its history. + + Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum. + It's possible to use srcSize==0, in which case, it will write a final empty block to end the frame. + Without last block mark, frames are considered unfinished (hence corrupted) by compliant decoders. + + `ZSTD_CCtx` object can be reused (ZSTD_compressBegin()) to compress again. +*/ + +/*===== Buffer-less streaming compression functions =====*/ +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */ + +ZSTD_DEPRECATED("This function will likely be removed in a future release. It is misleading and has very limited utility.") +ZSTDLIB_STATIC_API +size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */ + +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +/* The ZSTD_compressBegin_advanced() and ZSTD_compressBegin_usingCDict_advanced() are now DEPRECATED and will generate a compiler warning */ +ZSTD_DEPRECATED("use advanced API to access custom parameters") +ZSTDLIB_STATIC_API +size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */ +ZSTD_DEPRECATED("use advanced API to access custom parameters") +ZSTDLIB_STATIC_API +size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */ +/** + Buffer-less streaming decompression (synchronous mode) + + A ZSTD_DCtx object is required to track streaming operations. + Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it. + A ZSTD_DCtx object can be reused multiple times. + + First typical operation is to retrieve frame parameters, using ZSTD_getFrameHeader(). + Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough. + Data fragment must be large enough to ensure successful decoding. + `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough. + result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled. + >0 : `srcSize` is too small, please provide at least result bytes on next attempt. + errorCode, which can be tested using ZSTD_isError(). + + It fills a ZSTD_frameHeader structure with important information to correctly decode the frame, + such as the dictionary ID, content size, or maximum back-reference distance (`windowSize`). + Note that these values could be wrong, either because of data corruption, or because a 3rd party deliberately spoofs false information. + As a consequence, check that values remain within valid application range. + For example, do not allocate memory blindly, check that `windowSize` is within expectation. + Each application can set its own limits, depending on local restrictions. + For extended interoperability, it is recommended to support `windowSize` of at least 8 MB. + + ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize` bytes. + ZSTD_decompressContinue() is very sensitive to contiguity, + if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place, + or that previous contiguous segment is large enough to properly handle maximum back-reference distance. + There are multiple ways to guarantee this condition. + + The most memory efficient way is to use a round buffer of sufficient size. + Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(), + which can return an error code if required value is too large for current system (in 32-bits mode). + In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one, + up to the moment there is not enough room left in the buffer to guarantee decoding another full block, + which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`. + At which point, decoding can resume from the beginning of the buffer. + Note that already decoded data stored in the buffer should be flushed before being overwritten. + + There are alternatives possible, for example using two or more buffers of size `windowSize` each, though they consume more memory. + + Finally, if you control the compression process, you can also ignore all buffer size rules, + as long as the encoder and decoder progress in "lock-step", + aka use exactly the same buffer sizes, break contiguity at the same place, etc. + + Once buffers are setup, start decompression, with ZSTD_decompressBegin(). + If decompression requires a dictionary, use ZSTD_decompressBegin_usingDict() or ZSTD_decompressBegin_usingDDict(). + + Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively. + ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue(). + ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail. + + result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity). + It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item. + It can also be an error code, which can be tested with ZSTD_isError(). + + A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero. + Context can then be reset to start a new decompression. + + Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType(). + This information is not required to properly decode a frame. + + == Special case : skippable frames == + + Skippable frames allow integration of user-defined data into a flow of concatenated frames. + Skippable frames will be ignored (skipped) by decompressor. + The format of skippable frames is as follows : + a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F + b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits + c) Frame Content - any content (User Data) of length equal to Frame Size + For skippable frames ZSTD_getFrameHeader() returns zfhPtr->frameType==ZSTD_skippableFrame. + For skippable frames ZSTD_decompressContinue() always returns 0 : it only skips the content. +*/ + +/*===== Buffer-less streaming decompression functions =====*/ + +ZSTDLIB_STATIC_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */ + +ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); +ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); +ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin_usingDDict(ZSTD_DCtx* dctx, const ZSTD_DDict* ddict); + +ZSTDLIB_STATIC_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); +ZSTDLIB_STATIC_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + +/* misc */ +ZSTD_DEPRECATED("This function will likely be removed in the next minor release. It is misleading and has very limited utility.") +ZSTDLIB_STATIC_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); +typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e; +ZSTDLIB_STATIC_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); + + + + +/* ========================================= */ +/** Block level API (DEPRECATED) */ +/* ========================================= */ + +/*! + + This API is deprecated in favor of the regular compression API. + You can get the frame header down to 2 bytes by setting: + - ZSTD_c_format = ZSTD_f_zstd1_magicless + - ZSTD_c_contentSizeFlag = 0 + - ZSTD_c_checksumFlag = 0 + - ZSTD_c_dictIDFlag = 0 + + This API is not as well tested as our normal API, so we recommend not using it. + We will be removing it in a future version. If the normal API doesn't provide + the functionality you need, please open a GitHub issue. + + Block functions produce and decode raw zstd blocks, without frame metadata. + Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes). + But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes. + + A few rules to respect : + - Compressing and decompressing require a context structure + + Use ZSTD_createCCtx() and ZSTD_createDCtx() + - It is necessary to init context before starting + + compression : any ZSTD_compressBegin*() variant, including with dictionary + + decompression : any ZSTD_decompressBegin*() variant, including with dictionary + - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB + + If input is larger than a block size, it's necessary to split input data into multiple blocks + + For inputs larger than a single block, consider using regular ZSTD_compress() instead. + Frame metadata is not that costly, and quickly becomes negligible as source size grows larger than a block. + - When a block is considered not compressible enough, ZSTD_compressBlock() result will be 0 (zero) ! + ===> In which case, nothing is produced into `dst` ! + + User __must__ test for such outcome and deal directly with uncompressed data + + A block cannot be declared incompressible if ZSTD_compressBlock() return value was != 0. + Doing so would mess up with statistics history, leading to potential data corruption. + + ZSTD_decompressBlock() _doesn't accept uncompressed data as input_ !! + + In case of multiple successive blocks, should some of them be uncompressed, + decoder must be informed of their existence in order to follow proper history. + Use ZSTD_insertBlock() for such a case. +*/ + +/*===== Raw zstd block functions =====*/ +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx); +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") +ZSTDLIB_STATIC_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */ + +#endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */ + +#if defined (__cplusplus) +} +#endif diff --git a/externals/zstd/lib/zstd_errors.h b/externals/zstd/lib/zstd_errors.h new file mode 100644 index 0000000..dc75eee --- /dev/null +++ b/externals/zstd/lib/zstd_errors.h @@ -0,0 +1,114 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_ERRORS_H_398273423 +#define ZSTD_ERRORS_H_398273423 + +#if defined (__cplusplus) +extern "C" { +#endif + +/*===== dependency =====*/ +#include /* size_t */ + + +/* ===== ZSTDERRORLIB_API : control library symbols visibility ===== */ +#ifndef ZSTDERRORLIB_VISIBLE + /* Backwards compatibility with old macro name */ +# ifdef ZSTDERRORLIB_VISIBILITY +# define ZSTDERRORLIB_VISIBLE ZSTDERRORLIB_VISIBILITY +# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDERRORLIB_VISIBLE __attribute__ ((visibility ("default"))) +# else +# define ZSTDERRORLIB_VISIBLE +# endif +#endif + +#ifndef ZSTDERRORLIB_HIDDEN +# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDERRORLIB_HIDDEN __attribute__ ((visibility ("hidden"))) +# else +# define ZSTDERRORLIB_HIDDEN +# endif +#endif + +#if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBLE +#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBLE +#endif + +/*-********************************************* + * Error codes list + *-********************************************* + * Error codes _values_ are pinned down since v1.3.1 only. + * Therefore, don't rely on values if you may link to any version < v1.3.1. + * + * Only values < 100 are considered stable. + * + * note 1 : this API shall be used with static linking only. + * dynamic linking is not yet officially supported. + * note 2 : Prefer relying on the enum than on its value whenever possible + * This is the only supported way to use the error list < v1.3.1 + * note 3 : ZSTD_isError() is always correct, whatever the library version. + **********************************************/ +typedef enum { + ZSTD_error_no_error = 0, + ZSTD_error_GENERIC = 1, + ZSTD_error_prefix_unknown = 10, + ZSTD_error_version_unsupported = 12, + ZSTD_error_frameParameter_unsupported = 14, + ZSTD_error_frameParameter_windowTooLarge = 16, + ZSTD_error_corruption_detected = 20, + ZSTD_error_checksum_wrong = 22, + ZSTD_error_literals_headerWrong = 24, + ZSTD_error_dictionary_corrupted = 30, + ZSTD_error_dictionary_wrong = 32, + ZSTD_error_dictionaryCreation_failed = 34, + ZSTD_error_parameter_unsupported = 40, + ZSTD_error_parameter_combination_unsupported = 41, + ZSTD_error_parameter_outOfBound = 42, + ZSTD_error_tableLog_tooLarge = 44, + ZSTD_error_maxSymbolValue_tooLarge = 46, + ZSTD_error_maxSymbolValue_tooSmall = 48, + ZSTD_error_stabilityCondition_notRespected = 50, + ZSTD_error_stage_wrong = 60, + ZSTD_error_init_missing = 62, + ZSTD_error_memory_allocation = 64, + ZSTD_error_workSpace_tooSmall= 66, + ZSTD_error_dstSize_tooSmall = 70, + ZSTD_error_srcSize_wrong = 72, + ZSTD_error_dstBuffer_null = 74, + ZSTD_error_noForwardProgress_destFull = 80, + ZSTD_error_noForwardProgress_inputEmpty = 82, + /* following error codes are __NOT STABLE__, they can be removed or changed in future versions */ + ZSTD_error_frameIndex_tooLarge = 100, + ZSTD_error_seekableIO = 102, + ZSTD_error_dstBuffer_wrong = 104, + ZSTD_error_srcBuffer_wrong = 105, + ZSTD_error_sequenceProducer_failed = 106, + ZSTD_error_externalSequences_invalid = 107, + ZSTD_error_maxCode = 120 /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */ +} ZSTD_ErrorCode; + +/*! ZSTD_getErrorCode() : + convert a `size_t` function result into a `ZSTD_ErrorCode` enum type, + which can be used to compare with enum list published above */ +ZSTDERRORLIB_API ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); +ZSTDERRORLIB_API const char* ZSTD_getErrorString(ZSTD_ErrorCode code); /**< Same as ZSTD_getErrorName, but using a `ZSTD_ErrorCode` enum argument */ + + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_ERRORS_H_398273423 */ diff --git a/mk/analysis.mk b/mk/analysis.mk index 246c948..b8e5311 100644 --- a/mk/analysis.mk +++ b/mk/analysis.mk @@ -14,10 +14,13 @@ SHELL_SCRIPTS := $(shell git ls-files --cached --others --exclude-standard \ PYTHON_FORMAT_FILES := $(shell git ls-files --cached --others --exclude-standard \ -- '*.py') -## Run clang-tidy on all source files +## Run clang-tidy on all source files. ZSTD_DIR is exported by the parent +## Makefile so src/oci/decompress.c, which is the only translation unit +## that #includes , can resolve the header during analysis. lint: $(BUILD_DIR)/shim_blob.h $(BUILD_DIR)/version.h @echo " TIDY src/" - $(Q)$(CLANG_TIDY) $(SRCS) -- $(CFLAGS) -Isrc -I$(BUILD_DIR) + $(Q)$(CLANG_TIDY) $(SRCS) -- $(CFLAGS) -Isrc -I$(BUILD_DIR) \ + -I$(ZSTD_DIR)/lib ## Run clang static analyzer (scan-build) analyze: diff --git a/mk/config.mk b/mk/config.mk index 232da91..eca20c1 100644 --- a/mk/config.mk +++ b/mk/config.mk @@ -15,7 +15,16 @@ ifeq ($(origin GUEST_TEST_BINARIES), undefined) endif # Exclude native macOS test files from cross-compilation -NATIVE_TESTS := tests/test-multi-vcpu.c tests/test-rwx.c +NATIVE_TESTS := tests/test-multi-vcpu.c tests/test-rwx.c tests/test-oci-ref.c \ + tests/test-oci-digest.c tests/test-oci-blob-store.c \ + tests/test-oci-manifest.c tests/test-oci-fetch.c \ + tests/test-oci-store.c tests/test-oci-pull.c \ + tests/test-oci-inspect.c tests/test-oci-tar.c \ + tests/test-oci-decompress.c tests/test-oci-meta.c \ + tests/test-oci-layer-apply.c tests/test-oci-volume.c \ + tests/test-oci-clone.c tests/test-oci-unpack.c \ + tests/test-oci-runspec.c tests/test-oci-path-resolve.c \ + tests/test-oci-run.c SPECIAL_TEST_SRCS := tests/test-lowbase-mem.c SPECIAL_TEST_BINS := $(BUILD_DIR)/test-lowbase-mem-200000 $(BUILD_DIR)/test-lowbase-mem-300000 diff --git a/mk/tests.mk b/mk/tests.mk index 71014b3..9be3037 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -8,7 +8,19 @@ test-rosetta-alpine test-rosetta-audit test-rosetta-jit \ test-rosetta-glibc test-rosetta-all bench-rosetta \ test-matrix test-matrix-elfuse-aarch64 test-matrix-qemu-aarch64 \ - test-full test-multi-vcpu test-rwx test-sysroot-rename \ + test-full test-multi-vcpu test-rwx \ + test-oci-ref test-oci-digest test-oci-blob-store test-oci-manifest \ + test-oci-fetch test-oci-fetch-online test-oci-store test-oci-pull \ + test-oci-inspect test-oci-dedup-metrics test-oci-rebuild-cache \ + test-oci-status \ + test-oci-policy \ + test-oci-tar test-oci-decompress test-oci-meta \ + test-oci-origin \ + test-oci-layer-apply test-oci-volume test-oci-clone \ + test-oci-unpack test-oci-runspec test-oci-user test-oci-path-resolve \ + test-oci-runtime-files \ + test-oci-run test-oci-compat oci-fixture-builder \ + test-sysroot-rename \ test-case-collision test-case-collision-fallback test-sysroot-create-paths \ test-proctitle-host test-proctitle-low-stack \ test-sysroot-procfs-exec test-timeout-disable test-fuse-alpine \ @@ -51,6 +63,215 @@ check: $(ELFUSE_BIN) $(TEST_DEPS) check-syscall-coverage @$(MAKE) --no-print-directory test-timeout-disable @printf "\n$(BLUE)━━━ rosetta CLI gating ━━━$(RESET)\n" @$(MAKE) --no-print-directory test-rosetta-cli + @printf "\n$(BLUE)━━━ OCI reference parser unit tests ━━━$(RESET)\n" + @$(MAKE) --no-print-directory test-oci-ref + @printf "\n$(BLUE)━━━ OCI digest unit tests ━━━$(RESET)\n" + @$(MAKE) --no-print-directory test-oci-digest + @printf "\n$(BLUE)━━━ OCI blob store unit tests ━━━$(RESET)\n" + @$(MAKE) --no-print-directory test-oci-blob-store + @printf "\n$(BLUE)━━━ OCI manifest parser unit tests ━━━$(RESET)\n" + @$(MAKE) --no-print-directory test-oci-manifest + @printf "\n$(BLUE)━━━ OCI fetch unit tests (offline mock HTTP) ━━━$(RESET)\n" + @$(MAKE) --no-print-directory test-oci-fetch + @printf "\n$(BLUE)━━━ OCI store unit tests ━━━$(RESET)\n" + @$(MAKE) --no-print-directory test-oci-store + @printf "\n$(BLUE)━━━ OCI pull pipeline unit tests ━━━$(RESET)\n" + @$(MAKE) --no-print-directory test-oci-pull + @printf "\n$(BLUE)━━━ OCI inspect renderer unit tests ━━━$(RESET)\n" + @$(MAKE) --no-print-directory test-oci-inspect + @printf "\n$(BLUE)━━━ OCI cross-image dedup metrics unit tests ━━━$(RESET)\n" + @$(MAKE) --no-print-directory test-oci-dedup-metrics + @printf "\n$(BLUE)━━━ OCI rebuild-cache unit tests ━━━$(RESET)\n" + @$(MAKE) --no-print-directory test-oci-rebuild-cache + @printf "\n$(BLUE)━━━ OCI store-wide status unit tests ━━━$(RESET)\n" + @$(MAKE) --no-print-directory test-oci-status + @printf "\n$(BLUE)━━━ OCI policy.json loader unit tests ━━━$(RESET)\n" + @$(MAKE) --no-print-directory test-oci-policy + @printf "\n$(BLUE)━━━ OCI tar reader unit tests ━━━$(RESET)\n" + @$(MAKE) --no-print-directory test-oci-tar + @printf "\n$(BLUE)━━━ OCI decompression dispatch unit tests ━━━$(RESET)\n" + @$(MAKE) --no-print-directory test-oci-decompress + @printf "\n$(BLUE)━━━ OCI sidecar metadata unit tests ━━━$(RESET)\n" + @$(MAKE) --no-print-directory test-oci-meta + @printf "\n$(BLUE)━━━ OCI origin sidecar unit tests ━━━$(RESET)\n" + @$(MAKE) --no-print-directory test-oci-origin + @printf "\n$(BLUE)━━━ OCI layer applier unit tests ━━━$(RESET)\n" + @$(MAKE) --no-print-directory test-oci-layer-apply + @printf "\n$(BLUE)━━━ OCI volume bootstrap unit tests ━━━$(RESET)\n" + @$(MAKE) --no-print-directory test-oci-volume + @printf "\n$(BLUE)━━━ OCI clone-rootfs unit tests ━━━$(RESET)\n" + @$(MAKE) --no-print-directory test-oci-clone + @printf "\n$(BLUE)━━━ OCI unpack orchestrator smoke ━━━$(RESET)\n" + @$(MAKE) --no-print-directory test-oci-unpack + @printf "\n$(BLUE)━━━ OCI runspec resolver unit tests ━━━$(RESET)\n" + @$(MAKE) --no-print-directory test-oci-runspec + @printf "\n$(BLUE)━━━ OCI User-field resolver unit tests ━━━$(RESET)\n" + @$(MAKE) --no-print-directory test-oci-user + @printf "\n$(BLUE)━━━ OCI path-resolve unit tests ━━━$(RESET)\n" + @$(MAKE) --no-print-directory test-oci-path-resolve + @printf "\n$(BLUE)━━━ OCI runtime-files injection unit tests ━━━$(RESET)\n" + @$(MAKE) --no-print-directory test-oci-runtime-files + @printf "\n$(BLUE)━━━ OCI run orchestrator unit tests ━━━$(RESET)\n" + @$(MAKE) --no-print-directory test-oci-run + @printf "\n$(BLUE)━━━ OCI compat shell smoke ━━━$(RESET)\n" + @$(MAKE) --no-print-directory test-oci-compat + +## Run the OCI image reference parser unit tests (native, no HVF) +test-oci-ref: $(BUILD_DIR)/test-oci-ref + @$(BUILD_DIR)/test-oci-ref + +## Run the OCI digest unit tests (native, no HVF) +test-oci-digest: $(BUILD_DIR)/test-oci-digest + @$(BUILD_DIR)/test-oci-digest + +## Run the OCI blob store unit tests (native, no HVF) +test-oci-blob-store: $(BUILD_DIR)/test-oci-blob-store + @$(BUILD_DIR)/test-oci-blob-store + +## Run the OCI manifest / index / config parser unit tests (native, no HVF) +test-oci-manifest: $(BUILD_DIR)/test-oci-manifest + @$(BUILD_DIR)/test-oci-manifest + +## Run the OCI fetch unit tests against an in-process mock HTTP server +## (native, no HVF, no network). +test-oci-fetch: $(BUILD_DIR)/test-oci-fetch + @$(BUILD_DIR)/test-oci-fetch + +## Pull alpine:3.20 from Docker Hub anonymously, verify manifest parse and +## blob digests against a real registry. Opt-in; requires network. Not run by +## `make check`. +test-oci-fetch-online: $(BUILD_DIR)/test-oci-fetch + @OCI_FETCH_ONLINE=1 $(BUILD_DIR)/test-oci-fetch + +## Run the OCI local store unit tests (native, no HVF) +test-oci-store: $(BUILD_DIR)/test-oci-store + @$(BUILD_DIR)/test-oci-store + +## Run the OCI pull pipeline unit tests (native, no HVF, no network) +test-oci-pull: $(BUILD_DIR)/test-oci-pull + @$(BUILD_DIR)/test-oci-pull + +## Run the OCI inspect renderer unit tests (native, no HVF, no network) +test-oci-inspect: $(BUILD_DIR)/test-oci-inspect + @$(BUILD_DIR)/test-oci-inspect + +## Run the OCI cross-image dedup metrics unit tests (native, no HVF, no network). +## Phase 1 Plan 3 C3.4: validates oci_dedup_metrics_compute against pin-only +## and pin + unpacked-tree scratch stores. +test-oci-dedup-metrics: $(BUILD_DIR)/test-oci-dedup-metrics + @$(BUILD_DIR)/test-oci-dedup-metrics + +## Run the OCI rebuild-cache unit tests (native, no HVF, no network). +## Phase 1 Plan 3 C3.5: validates oci_rebuild_cache against scratch +## stores hand-populated via oci_origin_write into a fixture +## /images/sha256-/ tree. +test-oci-rebuild-cache: $(BUILD_DIR)/test-oci-rebuild-cache + @$(BUILD_DIR)/test-oci-rebuild-cache + +## Run the OCI store-wide status unit tests (native, no HVF, no network). +## Phase 1 Plan 4 C4.1: validates oci_status_compute against scratch stores +## hand-populated via stage_image + oci_origin_write fixture helpers. +test-oci-status: $(BUILD_DIR)/test-oci-status + @$(BUILD_DIR)/test-oci-status + +## Run the OCI policy.json schema and loader unit tests (native, no HVF, +## no network). Phase 1 Plan 6 C6.1: validates oci_policy_load against +## scratch HOME / XDG / override trees, the load-order chain, and the +## per-host effective view returned by oci_policy_lookup. +test-oci-policy: $(BUILD_DIR)/test-oci-policy + @$(BUILD_DIR)/test-oci-policy + +## Run the OCI tar reader unit tests (native, no HVF, no network) +test-oci-tar: $(BUILD_DIR)/test-oci-tar + @$(BUILD_DIR)/test-oci-tar + +## Run the OCI decompression dispatch unit tests (native, no HVF, no network) +test-oci-decompress: $(BUILD_DIR)/test-oci-decompress + @$(BUILD_DIR)/test-oci-decompress + +## Run the OCI sidecar metadata unit tests (native, no HVF, no network) +test-oci-meta: $(BUILD_DIR)/test-oci-meta + @$(BUILD_DIR)/test-oci-meta + +## Run the OCI origin sidecar unit tests (native, no HVF, no network). +## Covers oci_origin_write + cJSON parse-back round-trips. Phase 3 sees +## the file in unpacked image directories; Plan 1's root-set walker +## consumes it to attribute layer blobs back to live sysroots. +test-oci-origin: $(BUILD_DIR)/test-oci-origin + @$(BUILD_DIR)/test-oci-origin + +## Run the OCI layer applier unit tests (native, no HVF, no network) +test-oci-layer-apply: $(BUILD_DIR)/test-oci-layer-apply + @$(BUILD_DIR)/test-oci-layer-apply + +## Run the OCI volume bootstrap unit tests (native, no HVF). The +## default-sparsebundle case is gated behind OCI_VOLUME_TEST=1 because +## hdiutil orchestration is slow. +test-oci-volume: $(BUILD_DIR)/test-oci-volume + @$(BUILD_DIR)/test-oci-volume + +## Run the OCI clone-rootfs unit tests (native, no HVF). Skips itself +## if the test scratch directory does not support clonefile. +test-oci-clone: $(BUILD_DIR)/test-oci-clone + @$(BUILD_DIR)/test-oci-clone + +## Run the OCI unpack orchestrator smoke (native, no HVF). The full +## end-to-end fixture is gated behind OCI_VOLUME_TEST=1. +test-oci-unpack: $(BUILD_DIR)/test-oci-unpack + @$(BUILD_DIR)/test-oci-unpack + +## Run the OCI runspec resolver unit tests (native, no HVF, no network). +## Feeds hand-built oci_image_runtime_t literals plus synthetic CLI flags +## through oci_runspec_build and asserts argv / envp / uid / cwd outputs +## against the Phase 3 override matrix and Env policy. Phase 4 symbolic +## User cases write scratch /tmp rootfses for /etc/passwd lookup. +test-oci-runspec: $(BUILD_DIR)/test-oci-runspec + @$(BUILD_DIR)/test-oci-runspec + +## Run the OCI User-field resolver unit tests (native, no HVF, no network). +## Phase 4 F4.7: validates oci_user_lookup against scratch rootfses +## carrying synthetic /etc/passwd / /etc/group; covers the seven OCI +## image-spec User shapes plus the policy edges (digit-name collision, +## missing passwd, name-not-found, invalid characters). +test-oci-user: $(BUILD_DIR)/test-oci-user + @$(BUILD_DIR)/test-oci-user + +## Run the OCI guest PATH resolver unit tests (native, no HVF, no network). +## Builds a fake sysroot tree under /tmp and drives oci_path_resolve +## against it: PATH search, symlink-follow, escape-symlink skip, +## EACCES on noexec, ENOENT diagnostics with searched-dirs list. +test-oci-path-resolve: $(BUILD_DIR)/test-oci-path-resolve + @$(BUILD_DIR)/test-oci-path-resolve + +## Run the OCI runtime-files injection unit tests (native, no HVF, no network). +## Phase 4 F4.2 / F4.3: validates oci_runtime_files_inject against scratch +## run directories, covering fresh-/etc creation, symlink overwrite, +## regular-file overwrite, and the synthesised /etc/{resolv.conf, +## hosts, hostname} content. +test-oci-runtime-files: $(BUILD_DIR)/test-oci-runtime-files + @$(BUILD_DIR)/test-oci-runtime-files + +## Run the OCI run orchestrator unit tests (native, no HVF, no network). +## Covers oci_cli_run argument parsing plus oci_run early-failure +## paths against a case-insensitive volume; the launch backend is +## stubbed via oci_run_set_launch_for_testing so the test never spins +## up a real HVF VM. End-to-end launch coverage lives in the Phase 3 +## commit 6 compat shell suite. +test-oci-run: $(BUILD_DIR)/test-oci-run + @$(BUILD_DIR)/test-oci-run + +## Build the OCI fixture builder tool. Standalone executable used by +## tests/test-oci-compat.sh and available for hand-rolled fixtures. +oci-fixture-builder: $(BUILD_DIR)/oci-fixture-builder + +## Run the OCI run compatibility shell smoke (native, no HVF). Default +## mode covers CLI surface + fixture-builder integration; OCI_COMPAT_TEST=1 +## gates the heavy end-to-end harness (hdiutil sparsebundle + actual +## elfuse oci run launches); OCI_FETCH_ONLINE=1 gates the docker.io +## pull + run sibling. Requires test-hello (assembly aarch64 ELF) + +## elfuse + oci-fixture-builder pre-built. +test-oci-compat: $(ELFUSE_BIN) $(BUILD_DIR)/oci-fixture-builder $(TEST_HELLO_DEP) + @bash tests/test-oci-compat.sh test-sysroot-rename: $(ELFUSE_BIN) $(BUILD_DIR)/test-sysroot-rename @set -e; \ diff --git a/mk/toolchain.mk b/mk/toolchain.mk index e0f6be4..ec00aa9 100644 --- a/mk/toolchain.mk +++ b/mk/toolchain.mk @@ -42,3 +42,25 @@ SHIM_ASFLAGS ?= -arch arm64 # clang-format CLANG_FORMAT ?= clang-format + +# OpenSSL (Homebrew) for the OCI fetch test scaffolding. The mock HTTP server +# uses libssl/libcrypto to terminate TLS with a self-signed certificate so the +# ca_file negative cases exercise a real handshake. macOS ships LibreSSL +# headers in a private framework and does not publish a usable include path +# under /usr; brew openssl@3 is the documented public location. +ifeq ($(origin OPENSSL_PREFIX),undefined) + ifneq ($(wildcard /opt/homebrew/opt/openssl@3/include/openssl/ssl.h),) + OPENSSL_PREFIX := /opt/homebrew/opt/openssl@3 + else ifneq ($(wildcard /usr/local/opt/openssl@3/include/openssl/ssl.h),) + OPENSSL_PREFIX := /usr/local/opt/openssl@3 + else + OPENSSL_PREFIX := + endif +endif +ifneq ($(OPENSSL_PREFIX),) + OPENSSL_CFLAGS := -I$(OPENSSL_PREFIX)/include + OPENSSL_LDFLAGS := -L$(OPENSSL_PREFIX)/lib -lssl -lcrypto +else + OPENSSL_CFLAGS := + OPENSSL_LDFLAGS := -lssl -lcrypto +endif diff --git a/src/core/launch.c b/src/core/launch.c new file mode 100644 index 0000000..32f6de5 --- /dev/null +++ b/src/core/launch.c @@ -0,0 +1,148 @@ +/* elfuse VM launch: bring-up + GDB + run loop + teardown + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Extracted from src/main.c so the Phase 3 oci run orchestrator and the + * legacy positional-ELF main can share one bring-up path. The function + * deliberately does NOT touch the original CLI argv block: proctitle + * rewriting must happen in the caller, before elfuse_launch, because the + * caller owns the only pointer to the original argv (the heap-copied + * guest_argv hands a different string region to the guest). + * + * The function does not save/restore host cwd. Callers that care about + * post-exit host cwd preservation snapshot it themselves; this matches + * the pre-refactor main() shape where the cwd save lived alongside the + * CLI parsing rather than the bring-up. Same goes for sysroot_mount + * cleanup -- the --create-sysroot detach belongs to whoever provisioned + * the mount, not to the launch. + * + * shim_blob.h carries the embedded EL1 kernel shim. It is included here + * rather than in src/main.c so symbol shim_bin / shim_bin_len has a + * single definition site once the legacy main was reshaped to call + * elfuse_launch. Any other future caller that needs the shim bytes + * directly should pull them from here. + */ + +#include "launch.h" + +#include +#include +#include +#include + +#include "core/bootstrap.h" +#include "core/guest.h" +#include "core/sysroot.h" + +#include "syscall/proc.h" + +#include "debug/gdbstub.h" +#include "debug/log.h" + +/* Embedded shim binary (generated by xxd -i from shim.bin). */ +#include "shim_blob.h" + +int elfuse_launch(const launch_args_t *args) +{ + if (!args) { + log_error("elfuse_launch: NULL args"); + return 1; + } + + extern char **environ; + char **envp_use = args->envp ? (char **) (uintptr_t) args->envp : environ; + + guest_t g; + bool guest_initialized = false; + guest_bootstrap_t boot; + /* oci_run pre-resolves the entrypoint: args->elf_path is the host path + * under the cloned rootfs and guest_argv[0] is the guest-absolute path + * the guest should see (/proc/self/exe, argv[0]). The binary is a real + * file in the rootfs, never a FUSE-materialized temp, so the host-path + * temp flag is false. (guest_bootstrap_prepare's split of host vs guest + * path landed with the rosetta work; the caller resolves the host path.) + */ + const char *elf_guest_path = (args->guest_argc > 0 && args->guest_argv) + ? args->guest_argv[0] + : args->elf_path; + if (guest_bootstrap_prepare( + &g, args->elf_path, false, elf_guest_path, args->sysroot, + args->guest_argc, args->guest_argv, envp_use, shim_bin, + shim_bin_len, args->verbose, &guest_initialized, &boot) < 0) { + if (guest_initialized) + guest_destroy(&g); + return 1; + } + + if (args->sysroot) { + bool case_sensitive = true; + bool case_preserving = true; + if (sysroot_probe_case_sensitivity(args->sysroot, &case_sensitive, + &case_preserving) == 0) + proc_set_sysroot_casefold(case_preserving && !case_sensitive); + else + proc_set_sysroot_casefold(false); + } else { + proc_set_sysroot_casefold(false); + } + + if (args->has_creds) + proc_set_ids(args->uid, args->uid, args->uid, args->gid, args->gid, + args->gid); + + /* Phase 3 commit 5 will wire cwd_guest here once oci_run materializes + * the image WorkingDir under the cloned rootfs. Today's legacy main + * passes NULL and the guest inherits the host cwd, matching the + * pre-refactor behavior. + */ + (void) args->cwd_guest; + + /* Phase 3 placeholder: fork_child / vfork_notify dispatch stays in + * main() for now (early return before reaching elfuse_launch). The + * fields are kept on launch_args_t so callers route through one + * launch struct shape even when the IPC plumbing changes. + */ + (void) args->fork_child_fd; + (void) args->vfork_notify_fd; + + hv_vcpu_t vcpu; + hv_vcpu_exit_t *vexit; + if (guest_bootstrap_create_vcpu(&g, &boot, args->verbose, &vcpu, &vexit) < + 0) { + if (guest_initialized) + guest_destroy(&g); + return 1; + } + + /* GDB setup must happen before the first run so entry-stop and + * hardware breakpoints can affect the initial vCPU. + */ + if (args->gdb_port > 0) { + if (gdb_stub_init(args->gdb_port, &g) < 0) { + log_error("failed to initialize GDB stub"); + if (guest_initialized) + guest_destroy(&g); + return 1; + } + /* Mirror any preconfigured breakpoints/watchpoints into this + * vCPU. + */ + gdb_stub_sync_debug_regs(vcpu); + if (args->gdb_stop_on_entry) + gdb_stub_wait_for_attach(); + } + + /* vcpu_run_loop owns guest execution until exit, fatal signal, or + * timeout. + */ + int exit_code = + vcpu_run_loop(vcpu, vexit, &g, args->verbose, args->timeout_sec); + + /* Tear down debugger state before freeing guest/vCPU resources. */ + gdb_stub_shutdown(); + if (guest_initialized) + guest_destroy(&g); + + return exit_code; +} diff --git a/src/core/launch.h b/src/core/launch.h new file mode 100644 index 0000000..3422794 --- /dev/null +++ b/src/core/launch.h @@ -0,0 +1,84 @@ +/* elfuse VM launch entry: post-CLI bring-up + run loop + teardown + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * elfuse_launch is the single entry point for "run a guest binary in a + * fresh HVF VM until it exits". It is shared between main() (legacy + * positional-ELF CLI) and the Phase 3 oci run orchestrator. The function + * owns the guest_t, the vCPU, the GDB stub, and the run loop; it does NOT + * own the elf_path / sysroot / guest_argv heap copies or the + * sysroot_mount the host CLI may have provisioned -- those stay with the + * caller so behaviors that need the original CLI argv (proctitle + * rewriting, --create-sysroot detach on exit, host cwd save+restore) + * remain coherent regardless of how the launch was kicked off. + * + * Lifetime / ownership contract: + * + * - The caller owns every pointer in launch_args_t. elfuse_launch reads + * them and does not free them; const-qualified pointers stay valid + * for the duration of the call. + * - envp may be NULL; the host process environ is used in that case. + * - guest_argv is the NULL-terminated string array the guest sees as + * its argv. It must already be heap-copied because the caller may + * have clobbered the original CLI argv with proctitle. + * - has_creds=false means "inherit the host uid/gid"; uid/gid are + * ignored. has_creds=true forces the elfuse guest identity model + * via proc_set_ids. + * - cwd_guest reserves a slot for Phase 3 commit 5 (oci run) to set + * the guest's initial working directory. main()'s legacy positional- + * ELF path passes NULL and the guest inherits the host cwd, matching + * pre-refactor behavior. + * - fork_child_fd / vfork_notify_fd are forwarded for future + * fork-child-routed launches; main() currently dispatches the + * fork-child path before reaching elfuse_launch and so passes -1. + */ + +#pragma once + +#include +#include + +typedef struct { + /* Host filesystem path to the guest ELF (absolute). */ + const char *elf_path; + /* Host filesystem path to the sysroot the guest sees as / (absolute), + * or NULL when the guest runs without a sysroot. + */ + const char *sysroot; + /* NULL-terminated guest argv shape. guest_argc is the count of + * non-NULL entries (matches the legacy main() call shape). + */ + int guest_argc; + const char **guest_argv; + /* NULL-terminated guest environ. NULL means "use host environ". */ + const char **envp; + /* Override host uid/gid when true. Phase 3 commit 5 sets this from + * the image User field; main()'s legacy path leaves it false. + */ + bool has_creds; + uint32_t uid; + uint32_t gid; + /* Guest-absolute initial working directory. NULL inherits the host + * cwd. Wired up by Phase 3 commit 5. + */ + const char *cwd_guest; + /* GDB Remote Serial Protocol port. 0 disables the stub. */ + int gdb_port; + bool gdb_stop_on_entry; + /* Per-iteration vCPU run timeout. 0 disables (no alarm()). */ + int timeout_sec; + /* Fork-child IPC handles. -1 means "not a fork child". main()'s + * --fork-child dispatch handles the >= 0 case before reaching + * elfuse_launch; oci run never sets these. + */ + int fork_child_fd; + int vfork_notify_fd; + bool verbose; +} launch_args_t; + +/* Bring up the guest VM, run it to exit / signal / timeout, tear down, + * return the exit code. Returns 1 on bring-up failure (with a log + * message) and the guest's exit status otherwise. + */ +int elfuse_launch(const launch_args_t *args); diff --git a/src/core/sysroot.c b/src/core/sysroot.c index 188c783..d8a4442 100644 --- a/src/core/sysroot.c +++ b/src/core/sysroot.c @@ -5,6 +5,7 @@ */ #include +#include #include #include #include @@ -222,10 +223,35 @@ static int spawn_capture_stdout(char *const argv[], return 0; } -static int spawn_simple(char *const argv[]) +static int spawn_simple_common(char *const argv[], bool silence_stdout) { + posix_spawn_file_actions_t actions; + posix_spawn_file_actions_t *actions_ptr = NULL; + if (silence_stdout) { + /* hdiutil prints messages like '"diskN" ejected.' to stdout, + * even on success. Callers that promise a clean stdout contract + * (oci unpack / oci clone) must not inherit that noise, so + * redirect the child's fd 1 to /dev/null. stderr is left alone + * so genuine error messages still surface for diagnostics. + */ + if (posix_spawn_file_actions_init(&actions) != 0) { + errno = ENOMEM; + return -1; + } + if (posix_spawn_file_actions_addopen(&actions, STDOUT_FILENO, + "/dev/null", O_WRONLY, 0) != 0) { + posix_spawn_file_actions_destroy(&actions); + errno = EIO; + return -1; + } + actions_ptr = &actions; + } + pid_t pid = -1; - int spawn_ret = posix_spawnp(&pid, argv[0], NULL, NULL, argv, environ); + int spawn_ret = + posix_spawnp(&pid, argv[0], actions_ptr, NULL, argv, environ); + if (actions_ptr) + posix_spawn_file_actions_destroy(actions_ptr); if (spawn_ret != 0) { errno = spawn_ret; return -1; @@ -244,6 +270,11 @@ static int spawn_simple(char *const argv[]) return 0; } +static int spawn_simple_silent(char *const argv[]) +{ + return spawn_simple_common(argv, true); +} + static int parse_attach_mountpoint(const char *plist, char *mount_path, size_t mount_path_sz) @@ -419,11 +450,11 @@ static int sysroot_detach_mountpoint_force(const char *mount_path, bool force) if (force) { char *const argv[] = {"hdiutil", "detach", "-force", (char *) mount_path, NULL}; - return spawn_simple(argv); + return spawn_simple_silent(argv); } char *const argv[] = {"hdiutil", "detach", (char *) mount_path, NULL}; - return spawn_simple(argv); + return spawn_simple_silent(argv); } static bool sysroot_mountpoint_is_active(const char *mount_path) @@ -505,7 +536,7 @@ int sysroot_create_mount(const char *mount_path, sysroot_mount_t *mount) "elfuse_sysroot", mount->image_path, NULL}; - if (spawn_simple(create_argv) < 0) { + if (spawn_simple_silent(create_argv) < 0) { log_error("sysroot: hdiutil create failed for %s: %s", mount->image_path, strerror(errno)); return -1; diff --git a/src/main.c b/src/main.c index 30eb23d..429cb51 100644 --- a/src/main.c +++ b/src/main.c @@ -32,6 +32,8 @@ #include "core/rosetta.h" #include "core/sysroot.h" +#include "oci/cli.h" + #include "runtime/forkipc.h" #include "runtime/proctitle.h" @@ -192,6 +194,13 @@ int main(int argc, char **argv) argc = 5; } + /* `elfuse oci ...` is a self-contained CLI subcommand: image distribution + * never touches Hypervisor.framework, so dispatch before any guest setup + * to avoid host-DC-ZVA / entitlement checks the user never asked for. + */ + if (argc > 1 && !strcmp(argv[1], "oci")) + return oci_cli_main(argc - 1, argv + 1); + /* --help and --version do not require an ELF path. */ if (argc > 1) { if (!strcmp(argv[1], "--version") || !strcmp(argv[1], "-V")) { diff --git a/src/oci/blob-store.c b/src/oci/blob-store.c new file mode 100644 index 0000000..3c98f4a --- /dev/null +++ b/src/oci/blob-store.c @@ -0,0 +1,677 @@ +/* Content-addressable blob store for OCI image data + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The commit path uses link(2) rather than rename(2) so that a second writer + * racing on the same digest cannot silently overwrite a blob that another + * process already finalized. link returning EEXIST is treated as a dedup + * hit; both clients then unlink their staging file and report success. This + * matches the content-addressable invariant: identical bytes map to one + * inode, regardless of how many concurrent writers raced to produce them. + */ + +#include "blob-store.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "digest.h" + +/* Largest path the store will materialize. Comfortably above PATH_MAX so + * snprintf truncation never silently corrupts a path; callers that pass an + * out_size smaller than this can still recover via the returned length. + */ +#define STORE_PATH_MAX 4096 + +struct oci_blob_store { + char *root; +}; + +struct oci_blob_writer { + oci_blob_store_t *store; + oci_digest_algo_t algo; + char expected_hex[OCI_DIGEST_HEX_MAX + 1]; + char tmp_path[STORE_PATH_MAX]; + int fd; + oci_digester_t *digester; + bool failed; +}; + +static int mkdir_one(const char *path) +{ + if (mkdir(path, 0755) == 0) + return 0; + if (errno == EEXIST) { + struct stat st; + if (stat(path, &st) == 0 && S_ISDIR(st.st_mode)) + return 0; + errno = ENOTDIR; + return -1; + } + return -1; +} + +/* Create every directory along path. Walks component by component so that a + * missing intermediate directory does not abort the whole open. path must + * fit in STORE_PATH_MAX; the caller is responsible for upstream length + * checks (only internal call sites build these paths from store->root plus + * fixed suffixes, all of which stay well under the limit). + */ +static int mkdir_p(const char *path) +{ + char buf[STORE_PATH_MAX]; + size_t len = strlen(path); + if (len == 0 || len >= sizeof(buf)) { + errno = ENAMETOOLONG; + return -1; + } + memcpy(buf, path, len + 1); + + for (size_t i = 1; i < len; i++) { + if (buf[i] != '/') + continue; + buf[i] = '\0'; + if (mkdir_one(buf) < 0) + return -1; + buf[i] = '/'; + } + return mkdir_one(buf); +} + +static int join2(char *out, size_t out_size, const char *a, const char *b) +{ + int n = snprintf(out, out_size, "%s/%s", a, b); + if (n < 0 || (size_t) n >= out_size) { + errno = ENAMETOOLONG; + return -1; + } + return n; +} + +static int ensure_layout(const char *root) +{ + char path[STORE_PATH_MAX]; + if (mkdir_p(root) < 0) + return -1; + if (join2(path, sizeof(path), root, "blobs") < 0 || mkdir_one(path) < 0) + return -1; + if (join2(path, sizeof(path), root, "tmp") < 0 || mkdir_one(path) < 0) + return -1; + + static const char *const algos[] = {"sha256", "sha512"}; + for (size_t i = 0; i < sizeof(algos) / sizeof(algos[0]); i++) { + int n = snprintf(path, sizeof(path), "%s/blobs/%s", root, algos[i]); + if (n < 0 || (size_t) n >= sizeof(path)) { + errno = ENAMETOOLONG; + return -1; + } + if (mkdir_one(path) < 0) + return -1; + } + return 0; +} + +oci_blob_store_t *oci_blob_store_open(const char *root) +{ + if (!root || !*root) { + errno = EINVAL; + return NULL; + } + if (ensure_layout(root) < 0) + return NULL; + + oci_blob_store_t *s = calloc(1, sizeof(*s)); + if (!s) + return NULL; + s->root = strdup(root); + if (!s->root) { + free(s); + return NULL; + } + return s; +} + +void oci_blob_store_close(oci_blob_store_t *s) +{ + if (!s) + return; + free(s->root); + free(s); +} + +int oci_blob_store_path(const oci_blob_store_t *s, + oci_digest_algo_t algo, + const char *hex, + char *out, + size_t out_size) +{ + if (!s || !out || out_size == 0) { + if (out && out_size) + out[0] = '\0'; + return -1; + } + const char *name = oci_digest_algo_name(algo); + if (!name || !oci_digest_hex_valid(algo, hex)) { + out[0] = '\0'; + return -1; + } + int n = snprintf(out, out_size, "%s/blobs/%s/%s", s->root, name, hex); + if (n < 0) { + out[0] = '\0'; + return -1; + } + return n; +} + +bool oci_blob_store_has(const oci_blob_store_t *s, + oci_digest_algo_t algo, + const char *hex) +{ + char path[STORE_PATH_MAX]; + int n = oci_blob_store_path(s, algo, hex, path, sizeof(path)); + if (n < 0 || (size_t) n >= sizeof(path)) + return false; + struct stat st; + return stat(path, &st) == 0 && S_ISREG(st.st_mode); +} + +/* Monotonic counter used to disambiguate concurrent staging files within the + * same process. mkstemp itself supplies the global uniqueness via the random + * XXXXXX suffix; the counter is here only so that read-modify failures of + * the rand pool cannot defeat in-process uniqueness. + */ +static unsigned long writer_seq(void) +{ + static unsigned long n = 0; + return __sync_add_and_fetch(&n, 1); +} + +/* Shared writer construction. tmp_template_suffix is the part after + * "/tmp/" -- the caller composes a pid/seq form (anonymous) or a + * digest-prefix form (named) and this helper opens mkstemp + chmod + the + * digester, identical between the two public entry points. + */ +static oci_blob_writer_t *writer_begin_with_template( + oci_blob_store_t *s, + oci_digest_algo_t algo, + const char *expected_hex, + const char *tmp_template_suffix) +{ + if (!s || !oci_digest_hex_valid(algo, expected_hex)) { + errno = EINVAL; + return NULL; + } + + oci_blob_writer_t *w = calloc(1, sizeof(*w)); + if (!w) + return NULL; + w->store = s; + w->algo = algo; + memcpy(w->expected_hex, expected_hex, oci_digest_hex_len(algo) + 1); + w->fd = -1; + + int n = snprintf(w->tmp_path, sizeof(w->tmp_path), "%s/tmp/%s", s->root, + tmp_template_suffix); + if (n < 0 || (size_t) n >= sizeof(w->tmp_path)) { + free(w); + errno = ENAMETOOLONG; + return NULL; + } + + int fd = mkstemp(w->tmp_path); + if (fd < 0) { + int saved = errno; + free(w); + errno = saved; + return NULL; + } + (void) fcntl(fd, F_SETFD, FD_CLOEXEC); + if (fchmod(fd, 0644) < 0) { + int saved = errno; + (void) close(fd); + (void) unlink(w->tmp_path); + free(w); + errno = saved; + return NULL; + } + w->fd = fd; + + w->digester = oci_digester_new(algo); + if (!w->digester) { + int saved = errno ? errno : ENOMEM; + (void) close(w->fd); + (void) unlink(w->tmp_path); + free(w); + errno = saved; + return NULL; + } + return w; +} + +oci_blob_writer_t *oci_blob_writer_begin(oci_blob_store_t *s, + oci_digest_algo_t algo, + const char *expected_hex) +{ + char tmpl[128]; + int n = snprintf(tmpl, sizeof(tmpl), "blob-%ld-%lu-XXXXXX", (long) getpid(), + writer_seq()); + if (n < 0 || (size_t) n >= sizeof(tmpl)) { + errno = ENAMETOOLONG; + return NULL; + } + return writer_begin_with_template(s, algo, expected_hex, tmpl); +} + +#define OCI_BLOB_NAMED_HEX_PREFIX 16 + +oci_blob_writer_t *oci_blob_writer_begin_named(oci_blob_store_t *s, + oci_digest_algo_t algo, + const char *expected_hex) +{ + if (!expected_hex) { + errno = EINVAL; + return NULL; + } + char prefix[OCI_BLOB_NAMED_HEX_PREFIX + 1]; + size_t hl = strlen(expected_hex); + size_t use = + hl < OCI_BLOB_NAMED_HEX_PREFIX ? hl : OCI_BLOB_NAMED_HEX_PREFIX; + memcpy(prefix, expected_hex, use); + prefix[use] = '\0'; + char tmpl[64]; + int n = snprintf(tmpl, sizeof(tmpl), "blob-%s-XXXXXX", prefix); + if (n < 0 || (size_t) n >= sizeof(tmpl)) { + errno = ENAMETOOLONG; + return NULL; + } + return writer_begin_with_template(s, algo, expected_hex, tmpl); +} + +/* Build the per-store tmp/ path into out. Returns true on success, false on + * overflow. The caller is responsible for sizing out (STORE_PATH_MAX fits). + */ +static bool tmp_dir_path(const oci_blob_store_t *s, char *out, size_t cap) +{ + int n = snprintf(out, cap, "%s/tmp", s->root); + return n > 0 && (size_t) n < cap; +} + +/* Pull the leading hex16 digest prefix used for tmp filenames. expected_hex + * is validated by the caller (oci_digest_hex_valid). + */ +static void named_prefix_for(const char *expected_hex, char *out) +{ + size_t hl = strlen(expected_hex); + size_t use = + hl < OCI_BLOB_NAMED_HEX_PREFIX ? hl : OCI_BLOB_NAMED_HEX_PREFIX; + memcpy(out, expected_hex, use); + out[use] = '\0'; +} + +/* Open an existing partial as a writer. Re-hashes the bytes already on disk + * and positions the fd at end-of-file. Returns the writer on success or + * NULL on any I/O failure; the caller decides whether to fall back to a + * fresh writer. The partial file at path is NOT unlinked on failure -- + * caller policy. + */ +static oci_blob_writer_t *open_partial_as_writer(oci_blob_store_t *s, + oci_digest_algo_t algo, + const char *expected_hex, + const char *path, + int64_t partial_size) +{ + oci_blob_writer_t *w = calloc(1, sizeof(*w)); + if (!w) + return NULL; + w->store = s; + w->algo = algo; + memcpy(w->expected_hex, expected_hex, oci_digest_hex_len(algo) + 1); + size_t plen = strlen(path); + if (plen + 1 > sizeof(w->tmp_path)) { + free(w); + errno = ENAMETOOLONG; + return NULL; + } + memcpy(w->tmp_path, path, plen + 1); + w->fd = open(path, O_RDWR); + if (w->fd < 0) { + free(w); + return NULL; + } + (void) fcntl(w->fd, F_SETFD, FD_CLOEXEC); + w->digester = oci_digester_new(algo); + if (!w->digester) { + int saved = errno ? errno : ENOMEM; + (void) close(w->fd); + free(w); + errno = saved; + return NULL; + } + if (lseek(w->fd, 0, SEEK_SET) < 0) + goto fail_io; + int64_t consumed = 0; + char buf[64 * 1024]; + while (consumed < partial_size) { + ssize_t got = read(w->fd, buf, sizeof(buf)); + if (got == 0) + break; + if (got < 0) { + if (errno == EINTR) + continue; + goto fail_io; + } + oci_digester_update(w->digester, buf, (size_t) got); + consumed += got; + } + if (consumed != partial_size) + goto fail_io; + if (lseek(w->fd, 0, SEEK_END) < 0) + goto fail_io; + return w; + +fail_io: { + int saved = errno ? errno : EIO; + oci_digester_free(w->digester); + (void) close(w->fd); + free(w); + errno = saved; + return NULL; +} +} + +oci_blob_writer_t *oci_blob_writer_resume_named(oci_blob_store_t *s, + oci_digest_algo_t algo, + const char *expected_hex, + int64_t expected_size, + int64_t *out_resume_offset) +{ + if (out_resume_offset) + *out_resume_offset = 0; + if (!s || !oci_digest_hex_valid(algo, expected_hex)) { + errno = EINVAL; + return NULL; + } + + char tmp_dir[STORE_PATH_MAX]; + if (!tmp_dir_path(s, tmp_dir, sizeof(tmp_dir))) + return oci_blob_writer_begin_named(s, algo, expected_hex); + + char prefix[OCI_BLOB_NAMED_HEX_PREFIX + 1]; + named_prefix_for(expected_hex, prefix); + char glob[8 + OCI_BLOB_NAMED_HEX_PREFIX]; + int gn = snprintf(glob, sizeof(glob), "blob-%s-", prefix); + if (gn <= 0 || (size_t) gn >= sizeof(glob)) + return oci_blob_writer_begin_named(s, algo, expected_hex); + size_t glen = (size_t) gn; + + DIR *d = opendir(tmp_dir); + if (!d) + return oci_blob_writer_begin_named(s, algo, expected_hex); + + char best_path[STORE_PATH_MAX] = {0}; + int64_t best_size = -1; + struct dirent *de; + while ((de = readdir(d)) != NULL) { + if (strncmp(de->d_name, glob, glen) != 0) + continue; + char cand[STORE_PATH_MAX]; + int cn = snprintf(cand, sizeof(cand), "%s/%s", tmp_dir, de->d_name); + if (cn <= 0 || (size_t) cn >= sizeof(cand)) + continue; + struct stat st; + if (stat(cand, &st) < 0 || !S_ISREG(st.st_mode)) + continue; + int64_t sz = (int64_t) st.st_size; + /* Keep the largest partial; unlink everything else. A partial that is + * already at or past the declared size is corrupt or stale -- the + * caller cannot send a useful Range from it -- so drop it here and + * fall through to the fresh-writer path on no surviving partial. + */ + if (sz <= 0 || sz >= expected_size) { + (void) unlink(cand); + continue; + } + if (sz > best_size) { + if (best_path[0]) + (void) unlink(best_path); + memcpy(best_path, cand, (size_t) cn + 1); + best_size = sz; + } else { + (void) unlink(cand); + } + } + closedir(d); + + if (best_size <= 0 || !best_path[0]) + return oci_blob_writer_begin_named(s, algo, expected_hex); + + oci_blob_writer_t *w = + open_partial_as_writer(s, algo, expected_hex, best_path, best_size); + if (!w) { + (void) unlink(best_path); + return oci_blob_writer_begin_named(s, algo, expected_hex); + } + if (out_resume_offset) + *out_resume_offset = best_size; + return w; +} + +void oci_blob_store_sweep_partials(oci_blob_store_t *s, long ttl_secs) +{ + if (!s) + return; + char tmp_dir[STORE_PATH_MAX]; + if (!tmp_dir_path(s, tmp_dir, sizeof(tmp_dir))) + return; + DIR *d = opendir(tmp_dir); + if (!d) + return; + time_t now = time(NULL); + struct dirent *de; + while ((de = readdir(d)) != NULL) { + if (strncmp(de->d_name, "blob-", 5) != 0) + continue; + char path[STORE_PATH_MAX]; + int n = snprintf(path, sizeof(path), "%s/%s", tmp_dir, de->d_name); + if (n <= 0 || (size_t) n >= sizeof(path)) + continue; + struct stat st; + if (stat(path, &st) < 0 || !S_ISREG(st.st_mode)) + continue; + if ((long) (now - st.st_mtime) >= ttl_secs) + (void) unlink(path); + } + closedir(d); +} + +bool oci_blob_writer_write(oci_blob_writer_t *w, const void *buf, size_t len) +{ + if (!w || w->failed || (!buf && len)) { + if (w) + w->failed = true; + errno = EINVAL; + return false; + } + const uint8_t *p = buf; + while (len > 0) { + ssize_t n = write(w->fd, p, len); + if (n < 0) { + if (errno == EINTR) + continue; + w->failed = true; + return false; + } + if (n == 0) { + w->failed = true; + errno = EIO; + return false; + } + oci_digester_update(w->digester, p, (size_t) n); + p += n; + len -= (size_t) n; + } + return true; +} + +/* Discard staging file, free fd and digester. Errno is preserved across the + * cleanup so the caller can return its own diagnostic. + */ +static void writer_cleanup_fail(oci_blob_writer_t *w) +{ + int saved = errno; + if (w->fd >= 0) + (void) close(w->fd); + (void) unlink(w->tmp_path); + oci_digester_free(w->digester); + free(w); + errno = saved; +} + +/* fsync the directory that contains path so a newly linked/renamed entry is + * durable across a crash: fsync on the file persists its data but not the + * parent directory entry that names it. Best-effort -- the file-level fsync + * is the primary guarantee, and some filesystems reject a directory fsync, so + * a failure here must not fail the surrounding commit. + */ +static void fsync_parent_dir(const char *path) +{ + const char *slash = strrchr(path, '/'); + char dir[STORE_PATH_MAX]; + if (!slash) { + dir[0] = '.'; + dir[1] = '\0'; + } else if (slash == path) { + dir[0] = '/'; + dir[1] = '\0'; + } else { + size_t n = (size_t) (slash - path); + if (n >= sizeof(dir)) + return; + memcpy(dir, path, n); + dir[n] = '\0'; + } + int dfd = open(dir, O_RDONLY | O_DIRECTORY | O_CLOEXEC); + if (dfd < 0) + return; + (void) fsync(dfd); + (void) close(dfd); +} + +int oci_blob_writer_commit(oci_blob_writer_t *w) +{ + if (!w) { + errno = EINVAL; + return -1; + } + if (w->failed) { + writer_cleanup_fail(w); + errno = EIO; + return -1; + } + + char got_hex[OCI_DIGEST_HEX_MAX + 1]; + if (oci_digester_finish_hex(w->digester, got_hex) == 0) { + writer_cleanup_fail(w); + errno = EIO; + return -1; + } + oci_digester_free(w->digester); + w->digester = NULL; + + if (strcmp(got_hex, w->expected_hex) != 0) { + if (w->fd >= 0) + (void) close(w->fd); + (void) unlink(w->tmp_path); + free(w); + errno = EINVAL; + return -1; + } + + if (fsync(w->fd) < 0) { + int saved = errno; + (void) close(w->fd); + (void) unlink(w->tmp_path); + free(w); + errno = saved; + return -1; + } + if (close(w->fd) < 0) { + int saved = errno; + w->fd = -1; + (void) unlink(w->tmp_path); + free(w); + errno = saved; + return -1; + } + w->fd = -1; + + char final_path[STORE_PATH_MAX]; + int n = oci_blob_store_path(w->store, w->algo, w->expected_hex, final_path, + sizeof(final_path)); + if (n < 0 || (size_t) n >= sizeof(final_path)) { + (void) unlink(w->tmp_path); + free(w); + errno = ENAMETOOLONG; + return -1; + } + + if (link(w->tmp_path, final_path) < 0) { + if (errno != EEXIST) { + int saved = errno; + (void) unlink(w->tmp_path); + free(w); + errno = saved; + return -1; + } + /* Dedup hit: another writer beat this one. Content is identical + * because the digest matched, so dropping the staging file is the + * correct action. + */ + } + /* Persist the directory entry just created by link(2); without this a + * crash can leave the blob's data on disk but unreferenced by its name. + */ + fsync_parent_dir(final_path); + (void) unlink(w->tmp_path); + free(w); + return 0; +} + +void oci_blob_writer_abort(oci_blob_writer_t *w) +{ + if (!w) + return; + if (w->fd >= 0) + (void) close(w->fd); + (void) unlink(w->tmp_path); + oci_digester_free(w->digester); + free(w); +} + +int oci_blob_store_put_bytes(oci_blob_store_t *s, + oci_digest_algo_t algo, + const char *expected_hex, + const void *buf, + size_t len) +{ + oci_blob_writer_t *w = oci_blob_writer_begin(s, algo, expected_hex); + if (!w) + return -1; + if (!oci_blob_writer_write(w, buf, len)) { + int saved = errno; + oci_blob_writer_abort(w); + errno = saved; + return -1; + } + return oci_blob_writer_commit(w); +} diff --git a/src/oci/blob-store.h b/src/oci/blob-store.h new file mode 100644 index 0000000..5345571 --- /dev/null +++ b/src/oci/blob-store.h @@ -0,0 +1,157 @@ +/* Content-addressable blob store for OCI image data + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Layout matches the OCI image-layout convention: + * + * /blobs// finalized blob, immutable + * /tmp/blob-- in-flight staging file + * + * Every blob is committed by writing the staging file, fsync'ing it, hashing + * the bytes as they stream through the writer, comparing the actual hex to + * the expected hex from the manifest descriptor, and then atomically renaming + * the staging file into its final blobs// slot. A digest mismatch + * unlinks the staging file before returning -1, so an interrupted or hostile + * pull leaves no visible-complete blob behind. Repeated commits of the same + * digest are dedup'd in place (final path already exists -> drop staging, + * report success). + * + * The store path is opaque to this module; the caller picks it. Phase 1 + * targets ~/Library/Application Support/elfuse/blobs/ on macOS; a later + * slice moves the root onto a case-sensitive APFS sparse volume (oci-roadmap + * Q1) but the store API does not change. + */ + +#pragma once + +#include +#include +#include + +#include "digest.h" + +typedef struct oci_blob_store oci_blob_store_t; +typedef struct oci_blob_writer oci_blob_writer_t; + +/* Open or create the store rooted at `root`. The directory tree (root, + * blobs/, tmp) is created with mode 0755 if missing. Returns NULL on + * failure with errno preserved. + */ +oci_blob_store_t *oci_blob_store_open(const char *root); + +/* Release the store handle. Does not delete on-disk state. Safe on NULL. */ +void oci_blob_store_close(oci_blob_store_t *s); + +/* Resolve the final on-disk path for algo:hex. Returns the number of bytes + * the full path occupies excluding the trailing NUL, or -1 if algo or hex + * is malformed. Always writes a NUL terminator when out_size > 0; if the + * full path does not fit, out is truncated but still NUL-terminated and the + * caller can detect overflow by comparing the return value to out_size. + */ +int oci_blob_store_path(const oci_blob_store_t *s, + oci_digest_algo_t algo, + const char *hex, + char *out, + size_t out_size); + +/* True when blobs// exists as a regular file. */ +bool oci_blob_store_has(const oci_blob_store_t *s, + oci_digest_algo_t algo, + const char *hex); + +/* Begin a streaming write keyed by the descriptor digest. The writer hashes + * payload bytes as they stream and verifies the result against expected_hex + * during commit. Returns NULL on failure with errno preserved. expected_hex + * must be lowercase and the correct length for algo. + */ +oci_blob_writer_t *oci_blob_writer_begin(oci_blob_store_t *s, + oci_digest_algo_t algo, + const char *expected_hex); + +/* Same contract as oci_blob_writer_begin but stages into + * tmp/blob--XXXXXX. The digest prefix in the filename lets + * parallel batch callers find their in-flight partials by digest (used by + * the curl_multi pull path's resume + sweep, plan-doc Plan 5). Both writer + * entry points produce final blobs at the same blobs// path and + * are otherwise interchangeable; pickers can choose based on whether they + * need digest-keyed staging. + */ +oci_blob_writer_t *oci_blob_writer_begin_named(oci_blob_store_t *s, + oci_digest_algo_t algo, + const char *expected_hex); + +/* Open a writer that resumes from an existing tmp/blob--* + * partial when one is present. Scans tmp/ for files whose name starts with + * the digest prefix; selects the largest matching file; reopens it O_RDWR, + * re-hashes the bytes that are already on disk into the digester, and + * positions the fd at end-of-file so the next write appends. The selected + * partial keeps its name; siblings with the same prefix are unlinked. + * + * Falls back to oci_blob_writer_begin_named (fresh mkstemp staging file, + * offset 0) when any of these holds: + * - tmp/ has no matching partial + * - partial size >= expected_size (corrupt or stale; would defeat the + * descriptor size cap during the resumed transfer) + * - partial size is zero + * - reopen / re-hash fails for any reason + * + * On success returns a writer and, when out_resume_offset is non-NULL, + * writes the partial byte count there (0 on the fallback paths). Returns + * NULL only on the hard-error conditions that oci_blob_writer_begin_named + * also returns NULL for (EINVAL on bad arguments, ENOMEM on calloc, etc.). + * + * expected_size must be the descriptor's declared blob size in bytes. The + * store does not know what the caller will subsequently issue as a Range + * request; the parameter is here so the store can pre-reject partials that + * are at or past the declared size (which would tip the streaming overflow + * gate downstream into a "blob exceeded declared size" failure rather than + * a clean restart). + */ +oci_blob_writer_t *oci_blob_writer_resume_named(oci_blob_store_t *s, + oci_digest_algo_t algo, + const char *expected_hex, + int64_t expected_size, + int64_t *out_resume_offset); + +/* Delete tmp/ partials whose mtime is older than ttl_secs seconds ago. + * Matches any file in tmp/ whose name starts with "blob-"; non-matching + * names and subdirectories are skipped. Errors during the scan are + * silent: a missing tmp/ directory, a permission denial, or a per-file + * unlink failure leaves the rest of the sweep running. + * + * The store retains exclusive ownership of tmp/, so an aggressive prefix + * filter would not catch any third-party content. ttl_secs is the caller's + * choice; the fetcher invokes this once per batch with seven days. + */ +void oci_blob_store_sweep_partials(oci_blob_store_t *s, long ttl_secs); + +/* Append data to the staging file and the running digest. Returns true on + * success or false on a short write / I/O error with errno preserved. On + * failure the writer is left in a state where the only valid next call is + * oci_blob_writer_abort. + */ +bool oci_blob_writer_write(oci_blob_writer_t *w, const void *buf, size_t len); + +/* Finalize the digest, fsync, verify against expected_hex, then atomically + * rename into place. On success returns 0 and releases the writer. On digest + * mismatch returns -1 with errno set to EINVAL. On I/O failure returns -1 + * with errno preserved. The staging file is always unlinked on failure so + * an aborted pull never leaves a visible-complete blob. + */ +int oci_blob_writer_commit(oci_blob_writer_t *w); + +/* Discard the staging file and release the writer. Always succeeds; safe on + * NULL. + */ +void oci_blob_writer_abort(oci_blob_writer_t *w); + +/* One-shot helper: write a memory buffer into the store. Returns 0 on + * success or -1 on failure (errno preserved); semantics match the streaming + * commit path including dedup and atomic rename. + */ +int oci_blob_store_put_bytes(oci_blob_store_t *s, + oci_digest_algo_t algo, + const char *expected_hex, + const void *buf, + size_t len); diff --git a/src/oci/cli.c b/src/oci/cli.c new file mode 100644 index 0000000..f132137 --- /dev/null +++ b/src/oci/cli.c @@ -0,0 +1,1543 @@ +/* `elfuse oci` subcommand dispatch + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Slice 5a turns pull into a real subcommand: argument parsing for --store, + * -u USER[:PASS], --insecure-ca PEM, --insecure, -q, plus the actual oci_pull + * invocation against a freshly opened store and fetcher. Slice 5b extends + * inspect with --store and --all-platforms and an offline manifest tree + * renderer (src/oci/inspect.c). prune and list still return rc=2 "not + * implemented yet". + */ + +#include "cli.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "clone-rootfs.h" +#include "fetch.h" +#include "inspect.h" +#include "policy.h" +#include "pull.h" +#include "rebuild-cache.h" +#include "ref.h" +#include "run.h" +#include "status.h" +#include "store.h" +#include "unpack.h" +#include "volume.h" + +static int print_usage(FILE *out) +{ + fputs( + "usage: elfuse oci [args]\n" + "\n" + "Subcommands:\n" + " pull [OPTIONS] Download an image into the local store\n" + " inspect [OPTIONS] Show the canonical reference and parsed " + "fields\n" + " unpack [OPTIONS] Apply layers into a case-sensitive " + "sysroot\n" + " clone [OPTIONS] Create a per-run rootfs via APFS " + "clonefile\n" + " run [OPTIONS] [ARG...]\n" + " Launch a guest binary from a pulled image\n" + " prune Remove unreferenced blobs from the local " + "store\n" + " rebuild-cache Back-fill stack cache from unpacked " + "sysroots\n" + " status Report pins, unpacked sysroots, and cache " + "totals\n" + " list List images in the local store\n" + "\n" + "Pull options:\n" + " --store DIR Override the local store root\n" + " (default: ~/Library/Application " + "Support/elfuse/store)\n" + " -u, --user USER[:PASS] HTTP Basic auth for private registries\n" + " --insecure-ca PEM Trust PEM as the registry CA bundle\n" + " --insecure Skip TLS verify (loopback registries only)\n" + " --refresh Revalidate the pinned tag via " + "If-None-Match:\n" + " on 304 reuse the cached manifest and only\n" + " re-fetch missing layer blobs\n" + " -q, --quiet Suppress per-blob progress output\n" + "\n" + "Env: ELFUSE_OCI_PROGRESS=plain disables the in-place TTY\n" + " redraw (use when the terminal mis-handles CSI cursor-up;\n" + " prints one summary line per blob on completion).\n" + "\n" + "Policy: optional policy.json controls per-registry insecure /\n" + " ca_bundle / auth_file. Read from $ELFUSE_POLICY_FILE >\n" + " $XDG_CONFIG_HOME/elfuse/policy.json >\n" + " $HOME/.config/elfuse/policy.json >\n" + " $HOME/Library/Application Support/elfuse/policy.json.\n" + " CLI flags override; --quiet silences override warnings.\n" + "\n" + "Inspect options:\n" + " --store DIR Override the local store root\n" + " --volume DIR Include unpacked sysroots under DIR/images/\n" + " in the layer reuse comparison\n" + " --all-platforms List every platform entry of an image index\n" + " instead of drilling into linux/arm64\n" + "\n" + "Unpack options:\n" + " --store DIR Override the local store root\n" + " --volume DIR Override the sysroot APFS volume mount point\n" + " (default: auto-provisioned sparsebundle " + "under\n" + " ~/Library/Application " + "Support/elfuse/sysroots/)\n" + " --force Re-extract even if the image sysroot exists\n" + " -q, --quiet Suppress per-layer progress output\n" + "\n" + "Clone options:\n" + " --store DIR Override the local store root\n" + " --volume DIR Override the sysroot APFS volume mount point\n" + " --name NAME Human-friendly suffix for the per-run rootfs\n" + " --keep Do not register the run dir for cleanup " + "(no-op)\n" + "\n" + "Prune options:\n" + " --store DIR Override the local store root\n" + " --volume DIR Treat unpacked sysroots under DIR/images/ as " + "roots\n" + " --commit Actually unlink dangling blobs " + "(default: dry-run)\n" + " --older-than DUR Skip dangling blobs younger than DUR\n" + " (suffixes: s, m, h, d, w; plain integer = " + "seconds;\n" + " 0 = no filter)\n" + " --keep-bytes SIZE Keep up to SIZE bytes of newest dangling " + "blobs;\n" + " (suffixes: K, M, G; KiB-based; 0 = no " + "budget)\n" + "\n" + "Rebuild-cache options:\n" + " --store DIR Override the local store root\n" + " --volume DIR Override the sysroot APFS volume mount point\n" + " --commit Actually write stack snapshots " + "(default: dry-run)\n" + "\n" + "Status options:\n" + " --store DIR Override the local store root\n" + " --volume DIR Include unpacked sysroots under DIR/images/\n" + " in the report\n" + " --json Emit machine-readable JSON (schemaVersion 1)\n" + " --no-disk-usage Skip recursive size sums (faster on large " + "stores)\n" + "\n" + "Refs follow the docker/containerd grammar:\n" + " alpine, alpine:3.20, user/repo, ghcr.io/owner/img:tag,\n" + " repo@sha256:, repo:tag@sha256:\n", + out); + return out == stderr ? 2 : 0; +} + +/* Argument parser state for `oci inspect`. Mirrors pull_args_t in shape so a + * future cleanup could share the flag-loop, but the option set is disjoint + * enough that today the two parsers live side by side. + */ +typedef struct { + const char *store_root; + const char *volume_root; + bool show_all_platforms; + const char *ref_str; +} inspect_args_t; + +static int parse_inspect_args(int argc, char **argv, inspect_args_t *out) +{ + int i = 1; + while (i < argc) { + const char *a = argv[i]; + if (a[0] != '-') + break; + if (!strcmp(a, "--")) { + i++; + break; + } + if (!strcmp(a, "-h") || !strcmp(a, "--help")) { + return 1; + } else if (!strcmp(a, "--all-platforms")) { + out->show_all_platforms = true; + } else if (!strcmp(a, "--store")) { + if (++i >= argc) { + fputs("error: --store needs an argument\n", stderr); + return -1; + } + out->store_root = argv[i]; + } else if (!strcmp(a, "--volume")) { + if (++i >= argc) { + fputs("error: --volume needs an argument\n", stderr); + return -1; + } + out->volume_root = argv[i]; + } else { + fprintf(stderr, "error: unknown inspect option: %s\n", a); + return -1; + } + i++; + } + if (i >= argc) { + fputs("error: inspect needs a reference argument\n", stderr); + return -1; + } + if (i != argc - 1) { + fputs("error: extra arguments after inspect reference\n", stderr); + return -1; + } + out->ref_str = argv[i]; + return 0; +} + +static int cmd_inspect(int argc, char **argv) +{ + inspect_args_t args = {0}; + int prc = parse_inspect_args(argc, argv, &args); + if (prc == 1) + return print_usage(stdout); + if (prc < 0) + return 2; + + oci_ref_t ref = {0}; + const char *err = NULL; + if (oci_ref_parse(args.ref_str, &ref, &err) < 0) { + fprintf(stderr, "error: %s\n", err ? err : "invalid reference"); + return 1; + } + char *canonical = oci_ref_canonical(&ref); + if (!canonical) { + fputs("error: out of memory rendering canonical reference\n", stderr); + oci_ref_free(&ref); + return 1; + } + printf("canonical: %s\n", canonical); + printf("registry: %s\n", ref.registry); + printf("repository: %s\n", ref.repository); + printf("tag: %s\n", ref.tag ? ref.tag : "(none)"); + printf("digest: %s\n", ref.digest ? ref.digest : "(none)"); + free(canonical); + + /* Resolve store root: --store override or platform default. */ + char *default_root = NULL; + const char *store_root = args.store_root; + if (!store_root) { + default_root = oci_store_default_root(); + if (!default_root) { + fprintf(stderr, + "error: could not determine default store root " + "(HOME not set?)\n"); + oci_ref_free(&ref); + return 1; + } + store_root = default_root; + } + + oci_store_t *store = oci_store_open(store_root); + if (!store) { + fprintf(stderr, "error: could not open store at %s: %s\n", store_root, + strerror(errno)); + oci_ref_free(&ref); + free(default_root); + return 1; + } + + oci_inspect_options_t opts = { + .out = stdout, + .show_all_platforms = args.show_all_platforms, + .volume_root = args.volume_root, + }; + err = NULL; + int rc = oci_inspect(store, &ref, &opts, &err); + if (rc < 0 && err) + fprintf(stderr, "error: %s\n", err); + + oci_store_close(store); + oci_ref_free(&ref); + free(default_root); + return rc < 0 ? 1 : 0; +} + +/* Argument parser state for `oci pull`. Defaults are populated by the caller, + * then patched by parse_pull_args. + */ +typedef struct { + const char *store_root; /* heap-owned by main, not by parse */ + const char *user; + const char *password; + const char *ca_file; + bool allow_insecure; + bool quiet; + bool refresh; + const char *ref_str; + char *user_pass_buf; /* heap; freed by caller */ +} pull_args_t; + +/* Split USER[:PASS] in-place. Returns 0 on success or -1 with errno=ENOMEM. */ +static int split_userpass(const char *spec, pull_args_t *out) +{ + free(out->user_pass_buf); + out->user_pass_buf = strdup(spec); + if (!out->user_pass_buf) { + errno = ENOMEM; + return -1; + } + char *colon = strchr(out->user_pass_buf, ':'); + if (colon) { + *colon = '\0'; + out->user = out->user_pass_buf; + out->password = colon + 1; + } else { + out->user = out->user_pass_buf; + out->password = ""; + } + return 0; +} + +/* argv layout coming in: ["pull", "--flag", "...", ""]. argv[0] is the + * subcommand name; argv[argc-1] is the ref. Anything in between is options. + * Returns 0 on success, -1 on bad arguments (after printing an error). + */ +static int parse_pull_args(int argc, char **argv, pull_args_t *out) +{ + int i = 1; + while (i < argc) { + const char *a = argv[i]; + if (a[0] != '-') + break; + if (!strcmp(a, "--")) { + i++; + break; + } + if (!strcmp(a, "-h") || !strcmp(a, "--help")) { + return 1; + } else if (!strcmp(a, "-q") || !strcmp(a, "--quiet")) { + out->quiet = true; + } else if (!strcmp(a, "--refresh")) { + out->refresh = true; + } else if (!strcmp(a, "--insecure")) { + out->allow_insecure = true; + } else if (!strcmp(a, "--store")) { + if (++i >= argc) { + fputs("error: --store needs an argument\n", stderr); + return -1; + } + out->store_root = argv[i]; + } else if (!strcmp(a, "-u") || !strcmp(a, "--user")) { + if (++i >= argc) { + fputs("error: -u needs USER[:PASS]\n", stderr); + return -1; + } + if (split_userpass(argv[i], out) < 0) { + fputs("error: out of memory parsing credentials\n", stderr); + return -1; + } + } else if (!strcmp(a, "--insecure-ca")) { + if (++i >= argc) { + fputs("error: --insecure-ca needs a PEM path\n", stderr); + return -1; + } + out->ca_file = argv[i]; + } else { + fprintf(stderr, "error: unknown pull option: %s\n", a); + return -1; + } + i++; + } + if (i >= argc) { + fputs("error: pull needs a reference argument\n", stderr); + return -1; + } + if (i != argc - 1) { + fputs("error: extra arguments after pull reference\n", stderr); + return -1; + } + out->ref_str = argv[i]; + return 0; +} + +static int cmd_pull(int argc, char **argv) +{ + pull_args_t args = {0}; + int prc = parse_pull_args(argc, argv, &args); + if (prc == 1) { + free(args.user_pass_buf); + return print_usage(stdout); + } + if (prc < 0) { + free(args.user_pass_buf); + return 2; + } + + /* Default store root: either --store override or the platform default. */ + char *default_root = NULL; + const char *store_root = args.store_root; + if (!store_root) { + default_root = oci_store_default_root(); + if (!default_root) { + fprintf(stderr, + "error: could not determine default store root " + "(HOME not set?)\n"); + free(args.user_pass_buf); + return 1; + } + store_root = default_root; + } + + oci_ref_t ref = {0}; + const char *err = NULL; + if (oci_ref_parse(args.ref_str, &ref, &err) < 0) { + fprintf(stderr, "error: invalid reference: %s\n", + err ? err : "(unknown)"); + free(default_root); + free(args.user_pass_buf); + return 1; + } + + oci_store_t *store = oci_store_open(store_root); + if (!store) { + fprintf(stderr, "error: could not open store at %s: %s\n", store_root, + strerror(errno)); + oci_ref_free(&ref); + free(default_root); + free(args.user_pass_buf); + return 1; + } + + oci_policy_t *policy = NULL; + const char *perr = NULL; + if (oci_policy_load(&policy, &perr) < 0) { + fprintf(stderr, "error: policy load failed: %s\n", + perr ? perr : strerror(errno)); + oci_policy_free(policy); + oci_store_close(store); + oci_ref_free(&ref); + free(default_root); + free(args.user_pass_buf); + return 1; + } + + /* Warn when a CLI flag overrides a policy-declared value for the same + * registry. The check is gated on having actually loaded a policy file + * (source != "") so the user sees nothing when no policy is configured. + * The warn surface is intentionally minimal: one line per overridden + * field, host-scoped, no JSON pointer plumbing. --quiet silences all. + */ + if (!args.quiet && policy && oci_policy_source(policy)[0] != '\0') { + oci_policy_effective_t pol; + oci_policy_lookup(policy, ref.registry, &pol); + if (args.allow_insecure && !pol.insecure) + fprintf(stderr, + "warning: --insecure overrides policy.insecure for %s\n", + ref.registry); + if (args.user && pol.auth_file) + fprintf(stderr, "warning: -u overrides policy.auth_file for %s\n", + ref.registry); + if (args.ca_file && pol.ca_bundle) + fprintf( + stderr, + "warning: --insecure-ca overrides policy.ca_bundle for %s\n", + ref.registry); + } + + oci_fetcher_options_t fopts = { + .username = args.user, + .password = args.password, + .ca_file = args.ca_file, + .allow_insecure = args.allow_insecure, + .policy = policy, + }; + oci_fetcher_t *fetcher = oci_fetcher_new(&fopts); + if (!fetcher) { + fprintf(stderr, "error: could not create fetcher: %s\n", + strerror(errno)); + oci_policy_free(policy); + oci_store_close(store); + oci_ref_free(&ref); + free(default_root); + free(args.user_pass_buf); + return 1; + } + + if (!args.quiet) { + char *canon = oci_ref_canonical(&ref); + fprintf(stderr, "elfuse oci pull %s\n store: %s\n", + canon ? canon : args.ref_str, store_root); + free(canon); + } + + oci_pull_options_t popts = { + .quiet = args.quiet, + .refresh = args.refresh, + }; + err = NULL; + int rc = oci_pull(fetcher, store, &ref, &popts, &err); + if (rc < 0) { + fprintf(stderr, "error: pull failed: %s\n", + err ? err : strerror(errno)); + } else if (!args.quiet) { + fputs("done.\n", stderr); + } + + oci_fetcher_free(fetcher); + oci_policy_free(policy); + oci_store_close(store); + oci_ref_free(&ref); + free(default_root); + free(args.user_pass_buf); + return rc < 0 ? 1 : 0; +} + +typedef struct { + const char *store_root; + const char *volume_root; + const char *ref_str; + const char *name; /* clone only */ + bool quiet; + bool force_relayer; + bool keep_on_exit; /* clone only */ +} unpack_args_t; + +static int parse_unpack_args(int argc, + char **argv, + unpack_args_t *out, + bool clone_mode) +{ + int i = 1; + while (i < argc) { + const char *a = argv[i]; + if (a[0] != '-') + break; + if (!strcmp(a, "--")) { + i++; + break; + } + if (!strcmp(a, "-h") || !strcmp(a, "--help")) { + return 1; + } else if (!strcmp(a, "-q") || !strcmp(a, "--quiet")) { + out->quiet = true; + } else if (!strcmp(a, "--force")) { + if (clone_mode) { + fputs("error: --force is not valid for oci clone\n", stderr); + return -1; + } + out->force_relayer = true; + } else if (!strcmp(a, "--keep")) { + if (!clone_mode) { + fputs("error: --keep is only valid for oci clone\n", stderr); + return -1; + } + out->keep_on_exit = true; + } else if (!strcmp(a, "--store")) { + if (++i >= argc) { + fputs("error: --store needs an argument\n", stderr); + return -1; + } + out->store_root = argv[i]; + } else if (!strcmp(a, "--volume")) { + if (++i >= argc) { + fputs("error: --volume needs an argument\n", stderr); + return -1; + } + out->volume_root = argv[i]; + } else if (clone_mode && !strcmp(a, "--name")) { + if (++i >= argc) { + fputs("error: --name needs an argument\n", stderr); + return -1; + } + out->name = argv[i]; + } else { + fprintf(stderr, "error: unknown option: %s\n", a); + return -1; + } + i++; + } + if (i >= argc) { + fputs("error: subcommand needs a reference argument\n", stderr); + return -1; + } + if (i != argc - 1) { + fputs("error: extra arguments after reference\n", stderr); + return -1; + } + out->ref_str = argv[i]; + return 0; +} + +static int do_unpack(const unpack_args_t *args, + char **out_image_dir, + oci_store_t **out_store_keep) +{ + char *default_root = NULL; + const char *store_root = args->store_root; + if (!store_root) { + default_root = oci_store_default_root(); + if (!default_root) { + fprintf(stderr, + "error: could not determine default store root (HOME?)\n"); + return 1; + } + store_root = default_root; + } + + oci_ref_t ref = {0}; + const char *err = NULL; + if (oci_ref_parse(args->ref_str, &ref, &err) < 0) { + fprintf(stderr, "error: invalid reference: %s\n", + err ? err : "(unknown)"); + free(default_root); + return 1; + } + + oci_store_t *store = oci_store_open(store_root); + if (!store) { + fprintf(stderr, "error: could not open store at %s: %s\n", store_root, + strerror(errno)); + oci_ref_free(&ref); + free(default_root); + return 1; + } + + oci_unpack_options_t uopts = { + .volume_root = args->volume_root, + .quiet = args->quiet, + .force_relayer = args->force_relayer, + }; + err = NULL; + int rc = oci_unpack(store, &ref, &uopts, out_image_dir, &err); + if (rc < 0) { + fprintf(stderr, "error: unpack failed: %s\n", + err ? err : strerror(errno)); + oci_store_close(store); + oci_ref_free(&ref); + free(default_root); + return 1; + } + oci_ref_free(&ref); + free(default_root); + if (out_store_keep) + *out_store_keep = store; + else + oci_store_close(store); + return 0; +} + +static int cmd_unpack(int argc, char **argv) +{ + unpack_args_t args = {0}; + int prc = parse_unpack_args(argc, argv, &args, false); + if (prc == 1) + return print_usage(stdout); + if (prc < 0) + return 2; + char *image_dir = NULL; + int rc = do_unpack(&args, &image_dir, NULL); + if (rc != 0) { + free(image_dir); + return rc; + } + /* stdout: just the absolute path so $(elfuse oci unpack ref) composes. */ + printf("%s\n", image_dir); + free(image_dir); + return 0; +} + +static int cmd_clone(int argc, char **argv) +{ + unpack_args_t args = {0}; + int prc = parse_unpack_args(argc, argv, &args, true); + if (prc == 1) + return print_usage(stdout); + if (prc < 0) + return 2; + + char *image_dir = NULL; + oci_store_t *store = NULL; + int rc = do_unpack(&args, &image_dir, &store); + if (rc != 0) { + free(image_dir); + return rc; + } + oci_store_close(store); + + /* Resolve the volume root the same way unpack did so clone-rootfs + * lands in the same sparsebundle. + */ + char *volume_root = NULL; + const char *err = NULL; + if (oci_volume_ensure(args.volume_root, &volume_root, &err) < 0) { + fprintf(stderr, "error: volume_ensure failed: %s\n", + err ? err : strerror(errno)); + free(image_dir); + return 1; + } + + /* image_dir has a trailing slash; strip it for the clone source. */ + size_t il = strlen(image_dir); + if (il > 1 && image_dir[il - 1] == '/') + image_dir[il - 1] = '\0'; + + char *run_dir = NULL; + err = NULL; + if (oci_clone_rootfs(image_dir, volume_root, &run_dir, &err) < 0) { + fprintf(stderr, "error: clone failed: %s\n", + err ? err : strerror(errno)); + free(image_dir); + free(volume_root); + return 1; + } + /* --keep is forward-looking; Phase 2 does not auto-clean either way. */ + (void) args.keep_on_exit; + printf("%s\n", run_dir); + free(run_dir); + free(image_dir); + free(volume_root); + return 0; +} + +static int cmd_not_implemented(const char *name) +{ + fprintf(stderr, + "error: 'oci %s' is not implemented yet (see issue #31 Phase 1)\n", + name); + return 2; +} + +/* Argument parser state for `oci prune`. The flag set is intentionally + * minimal: dry-run is the default (so the operator can review what would + * be reclaimed before committing) and --commit is the only switch that + * actually unlinks. --volume mirrors the same flag in unpack/clone so + * the same volume root the user uses for unpacked sysroots also feeds + * the keep-set walk; without --volume only pins contribute. + * + * older_than_sec / keep_bytes default to 0, which the store API + * interprets as "no filter" so an operator that does not opt in sees + * the C1.3 behaviour (every dangling blob is pruned). The CLI does + * not distinguish between "not specified" and "--older-than 0" / + * "--keep-bytes 0" because both compose to the same zero-filter + * behaviour; a future structured-output mode (Plan 4 oci status) can + * surface filter state from the rendered options struct directly. + */ +typedef struct { + const char *store_root; + const char *volume_root; + bool commit; + uint64_t older_than_sec; + uint64_t keep_bytes; +} prune_args_t; + +/* Parse a duration string into seconds. Accepted shapes are + * pure integer interpreted as seconds + * s seconds + * m minutes (60s) + * h hours (3600s) + * d days (86400s) + * w weeks (604800s) + * where is a decimal unsigned integer with no sign character. The + * trailing suffix, when present, is a single ASCII letter; any other + * trailing bytes are rejected. Overflow is detected by checking the + * intermediate product against UINT64_MAX before applying it. Returns + * 0 on success with the value written to *out; -1 on any parse or + * overflow failure with errno=EINVAL. + */ +static int parse_duration(const char *s, uint64_t *out) +{ + if (!s || !*s) { + errno = EINVAL; + return -1; + } + /* strtoull silently accepts a leading '-' and wraps the result; + * detect a negative sign and the leading-whitespace skip + * explicitly so a user-facing flag never quietly parses "-5d" as + * a huge positive duration. + */ + if (*s == '-' || *s == '+' || *s == ' ' || *s == '\t') { + errno = EINVAL; + return -1; + } + char *endp = NULL; + errno = 0; + unsigned long long raw = strtoull(s, &endp, 10); + if (errno == ERANGE) { + errno = EINVAL; + return -1; + } + if (!endp || endp == s) { + errno = EINVAL; + return -1; + } + uint64_t value = (uint64_t) raw; + uint64_t multiplier = 1; + if (*endp != '\0') { + if (endp[1] != '\0') { + errno = EINVAL; + return -1; + } + switch (*endp) { + case 's': + multiplier = 1; + break; + case 'm': + multiplier = 60; + break; + case 'h': + multiplier = 3600; + break; + case 'd': + multiplier = 86400; + break; + case 'w': + multiplier = 604800; + break; + default: + errno = EINVAL; + return -1; + } + } + if (multiplier != 0 && value > UINT64_MAX / multiplier) { + errno = EINVAL; + return -1; + } + *out = value * multiplier; + return 0; +} + +/* Parse a byte-size string into bytes. Accepted shapes are + * pure integer interpreted as bytes + * K / KB 1024 bytes per unit + * M / MB 1024 * 1024 bytes per unit + * G / GB 1024 * 1024 * 1024 bytes per unit + * matching du / df conventions (KiB-based, not decimal). The + * trailing suffix is at most two letters, case-sensitive, and the + * second letter when present must be 'B'. Negative inputs and + * arithmetic overflow are rejected with EINVAL; on success returns 0 + * and stores the byte count in *out. + */ +static int parse_byte_size(const char *s, uint64_t *out) +{ + if (!s || !*s) { + errno = EINVAL; + return -1; + } + if (*s == '-' || *s == '+' || *s == ' ' || *s == '\t') { + errno = EINVAL; + return -1; + } + char *endp = NULL; + errno = 0; + unsigned long long raw = strtoull(s, &endp, 10); + if (errno == ERANGE) { + errno = EINVAL; + return -1; + } + if (!endp || endp == s) { + errno = EINVAL; + return -1; + } + uint64_t value = (uint64_t) raw; + uint64_t multiplier = 1; + if (*endp != '\0') { + char unit = *endp; + char trailer = endp[1]; + if (trailer != '\0' && (trailer != 'B' || endp[2] != '\0')) { + errno = EINVAL; + return -1; + } + switch (unit) { + case 'K': + multiplier = 1024ULL; + break; + case 'M': + multiplier = 1024ULL * 1024ULL; + break; + case 'G': + multiplier = 1024ULL * 1024ULL * 1024ULL; + break; + default: + errno = EINVAL; + return -1; + } + } + if (multiplier != 0 && value > UINT64_MAX / multiplier) { + errno = EINVAL; + return -1; + } + *out = value * multiplier; + return 0; +} + +static int parse_prune_args(int argc, char **argv, prune_args_t *out) +{ + int i = 1; + while (i < argc) { + const char *a = argv[i]; + if (a[0] != '-') + break; + if (!strcmp(a, "--")) { + i++; + break; + } + if (!strcmp(a, "-h") || !strcmp(a, "--help")) { + return 1; + } else if (!strcmp(a, "--commit")) { + out->commit = true; + } else if (!strcmp(a, "--store")) { + if (++i >= argc) { + fputs("error: --store needs an argument\n", stderr); + return -1; + } + out->store_root = argv[i]; + } else if (!strcmp(a, "--volume")) { + if (++i >= argc) { + fputs("error: --volume needs an argument\n", stderr); + return -1; + } + out->volume_root = argv[i]; + } else if (!strcmp(a, "--older-than")) { + if (++i >= argc) { + fputs("error: --older-than needs an argument\n", stderr); + return -1; + } + if (parse_duration(argv[i], &out->older_than_sec) < 0) { + fprintf(stderr, "error: --older-than: invalid duration '%s'\n", + argv[i]); + return -1; + } + } else if (!strcmp(a, "--keep-bytes")) { + if (++i >= argc) { + fputs("error: --keep-bytes needs an argument\n", stderr); + return -1; + } + if (parse_byte_size(argv[i], &out->keep_bytes) < 0) { + fprintf(stderr, "error: --keep-bytes: invalid byte size '%s'\n", + argv[i]); + return -1; + } + } else { + fprintf(stderr, "error: unknown prune option: %s\n", a); + return -1; + } + i++; + } + if (i != argc) { + fputs("error: prune takes no positional arguments\n", stderr); + return -1; + } + return 0; +} + +static int cmd_prune(int argc, char **argv) +{ + prune_args_t args = {0}; + int prc = parse_prune_args(argc, argv, &args); + if (prc == 1) + return print_usage(stdout); + if (prc < 0) + return 2; + + char *default_root = NULL; + const char *store_root = args.store_root; + if (!store_root) { + default_root = oci_store_default_root(); + if (!default_root) { + fprintf(stderr, + "error: could not determine default store root " + "(HOME not set?)\n"); + return 1; + } + store_root = default_root; + } + + oci_store_t *store = oci_store_open(store_root); + if (!store) { + fprintf(stderr, "error: could not open store at %s: %s\n", store_root, + strerror(errno)); + free(default_root); + return 1; + } + + oci_store_prune_options_t opts = { + .commit = args.commit, + .volume_root = args.volume_root, + .older_than_sec = args.older_than_sec, + .keep_bytes = args.keep_bytes, + }; + const char *err = NULL; + int rc = oci_store_prune(store, &opts, &err); + if (rc < 0) { + fprintf(stderr, "error: prune failed: %s\n", + err ? err : strerror(errno)); + oci_store_close(store); + free(default_root); + return 1; + } + + /* Output preserves the Plan 1 line shape so existing operator scripts + * and the compat smoke continue to match on "reclaimable: N blobs" / + * "reclaimed: N blobs" / "kept: M blobs" / "dry-run". The new layer + * and stack lines (C3.3d) only render when their counter is non-zero + * so a single-family cache still produces the legacy two-line output. + */ + const char *verb_done = args.commit ? "reclaimed" : "reclaimable"; + const char *verb_pre = args.commit ? "reclaimed" : "reclaimable"; + if (args.commit) { + printf("reclaimed: %zu blobs (%llu bytes)\n", opts.pruned_blobs, + (unsigned long long) opts.pruned_bytes); + if (opts.pruned_layers > 0) + printf("layers: %zu %s (%llu bytes)\n", opts.pruned_layers, + verb_done, (unsigned long long) opts.pruned_layer_bytes); + if (opts.pruned_stacks > 0) + printf("stacks: %zu %s (%llu bytes)\n", opts.pruned_stacks, + verb_done, (unsigned long long) opts.pruned_stack_bytes); + if (opts.skipped_blobs > 0) + printf("skipped: %zu blobs (%llu bytes)\n", opts.skipped_blobs, + (unsigned long long) opts.skipped_bytes); + if (opts.skipped_layers > 0) + printf("layers: %zu skipped (%llu bytes)\n", opts.skipped_layers, + (unsigned long long) opts.skipped_layer_bytes); + if (opts.skipped_stacks > 0) + printf("stacks: %zu skipped (%llu bytes)\n", opts.skipped_stacks, + (unsigned long long) opts.skipped_stack_bytes); + printf("kept: %zu blobs\n", opts.kept_blobs); + if (opts.kept_layers > 0) + printf("kept: %zu layers\n", opts.kept_layers); + if (opts.kept_stacks > 0) + printf("kept: %zu stacks\n", opts.kept_stacks); + } else { + printf("reclaimable: %zu blobs (%llu bytes)\n", opts.pruned_blobs, + (unsigned long long) opts.pruned_bytes); + if (opts.pruned_layers > 0) + printf("layers: %zu %s (%llu bytes)\n", opts.pruned_layers, + verb_pre, (unsigned long long) opts.pruned_layer_bytes); + if (opts.pruned_stacks > 0) + printf("stacks: %zu %s (%llu bytes)\n", opts.pruned_stacks, + verb_pre, (unsigned long long) opts.pruned_stack_bytes); + if (opts.skipped_blobs > 0) + printf("skipped: %zu blobs (%llu bytes)\n", opts.skipped_blobs, + (unsigned long long) opts.skipped_bytes); + if (opts.skipped_layers > 0) + printf("layers: %zu skipped (%llu bytes)\n", + opts.skipped_layers, + (unsigned long long) opts.skipped_layer_bytes); + if (opts.skipped_stacks > 0) + printf("stacks: %zu skipped (%llu bytes)\n", + opts.skipped_stacks, + (unsigned long long) opts.skipped_stack_bytes); + printf("kept: %zu blobs\n", opts.kept_blobs); + if (opts.kept_layers > 0) + printf("kept: %zu layers\n", opts.kept_layers); + if (opts.kept_stacks > 0) + printf("kept: %zu stacks\n", opts.kept_stacks); + printf("(dry-run; pass --commit to delete)\n"); + } + + oci_store_close(store); + free(default_root); + return 0; +} + +/* Argument parser state for oci rebuild-cache. Mirrors prune_args_t in + * shape because both subcommands carry --store / --volume / --commit; the + * two parsers stay disjoint so a future option addition to either does not + * surprise the other. + */ +typedef struct { + const char *store_root; + const char *volume_root; + bool commit; +} rebuild_cache_args_t; + +static int parse_rebuild_cache_args(int argc, + char **argv, + rebuild_cache_args_t *out) +{ + int i = 1; + while (i < argc) { + const char *a = argv[i]; + if (a[0] != '-') + break; + if (!strcmp(a, "--")) { + i++; + break; + } + if (!strcmp(a, "-h") || !strcmp(a, "--help")) { + return 1; + } else if (!strcmp(a, "--commit")) { + out->commit = true; + } else if (!strcmp(a, "--store")) { + if (++i >= argc) { + fputs("error: --store needs an argument\n", stderr); + return -1; + } + out->store_root = argv[i]; + } else if (!strcmp(a, "--volume")) { + if (++i >= argc) { + fputs("error: --volume needs an argument\n", stderr); + return -1; + } + out->volume_root = argv[i]; + } else { + fprintf(stderr, "error: unknown rebuild-cache option: %s\n", a); + return -1; + } + i++; + } + if (i != argc) { + fputs("error: rebuild-cache takes no positional arguments\n", stderr); + return -1; + } + return 0; +} + +static int cmd_rebuild_cache(int argc, char **argv) +{ + rebuild_cache_args_t args = {0}; + int prc = parse_rebuild_cache_args(argc, argv, &args); + if (prc == 1) + return print_usage(stdout); + if (prc < 0) + return 2; + + char *default_root = NULL; + const char *store_root = args.store_root; + if (!store_root) { + default_root = oci_store_default_root(); + if (!default_root) { + fprintf(stderr, + "error: could not determine default store root " + "(HOME not set?)\n"); + return 1; + } + store_root = default_root; + } + + oci_store_t *store = oci_store_open(store_root); + if (!store) { + fprintf(stderr, "error: could not open store at %s: %s\n", store_root, + strerror(errno)); + free(default_root); + return 1; + } + + oci_rebuild_cache_options_t opts = { + .commit = args.commit, + }; + const char *err = NULL; + int rc = oci_rebuild_cache(store, args.volume_root, &opts, &err); + if (rc < 0) { + fprintf(stderr, "error: rebuild-cache failed: %s\n", + err ? err : strerror(errno)); + oci_store_close(store); + free(default_root); + return 1; + } + + size_t skipped_bad = opts.trees_skipped_no_origin + + opts.trees_skipped_bad_origin + + opts.trees_skipped_empty_diffids; + + if (args.commit) { + printf("rebuild-cache:\n"); + printf(" scanned: %zu unpacked trees\n", opts.trees_scanned); + printf(" rebuilt: %zu trees (%zu stack entries)\n", + opts.trees_rebuilt, opts.stack_entries_added); + printf(" already cached: %zu trees\n", opts.trees_skipped_cached); + if (skipped_bad > 0) + printf(" skipped (bad): %zu trees\n", skipped_bad); + if (opts.trees_failed > 0) + printf(" failed: %zu trees\n", opts.trees_failed); + } else { + printf("rebuild-cache (dry-run):\n"); + printf(" scanned: %zu unpacked trees\n", opts.trees_scanned); + printf(" would rebuild: %zu trees (%zu stack entries)\n", + opts.trees_rebuilt, opts.stack_entries_added); + printf(" already cached: %zu trees\n", opts.trees_skipped_cached); + if (skipped_bad > 0) + printf(" skipped (bad): %zu trees\n", skipped_bad); + if (opts.trees_failed > 0) + printf(" failed: %zu trees\n", opts.trees_failed); + printf("(dry-run; pass --commit to write)\n"); + } + + oci_store_close(store); + free(default_root); + return 0; +} + +/* Argument parser state for `oci status`. The flag set is intentionally + * small: store / volume mirrors prune / rebuild-cache, --json toggles the + * structured output, --no-disk-usage is the operator escape hatch for very + * large stores where the recursive size walk dominates wall time. + */ +typedef struct { + const char *store_root; + const char *volume_root; + bool json; + bool no_disk_usage; +} status_args_t; + +static int parse_status_args(int argc, char **argv, status_args_t *out) +{ + int i = 1; + while (i < argc) { + const char *a = argv[i]; + if (a[0] != '-') + break; + if (!strcmp(a, "--")) { + i++; + break; + } + if (!strcmp(a, "-h") || !strcmp(a, "--help")) { + return 1; + } else if (!strcmp(a, "--json")) { + out->json = true; + } else if (!strcmp(a, "--no-disk-usage")) { + out->no_disk_usage = true; + } else if (!strcmp(a, "--store")) { + if (++i >= argc) { + fputs("error: --store needs an argument\n", stderr); + return -1; + } + out->store_root = argv[i]; + } else if (!strcmp(a, "--volume")) { + if (++i >= argc) { + fputs("error: --volume needs an argument\n", stderr); + return -1; + } + out->volume_root = argv[i]; + } else { + fprintf(stderr, "error: unknown status option: %s\n", a); + return -1; + } + i++; + } + if (i != argc) { + fputs("error: status takes no positional arguments\n", stderr); + return -1; + } + return 0; +} + +/* Render a byte count compactly. Values >= 1 MiB use "~X.Y MiB", smaller + * non-zero values render as raw bytes, zero stays "0 B". Mirrors the inspect + * renderer's shared_bytes formatter so the two surfaces look consistent. + */ +static void format_bytes(uint64_t bytes, char *out, size_t cap) +{ + if (bytes == 0) { + snprintf(out, cap, "0 B"); + return; + } + if (bytes >= (uint64_t) 1024 * 1024) { + double mib = (double) bytes / (1024.0 * 1024.0); + snprintf(out, cap, "~%.1f MiB", mib); + return; + } + snprintf(out, cap, "%llu B", (unsigned long long) bytes); +} + +/* Render epoch seconds as a fixed-width "YYYY-MM-DD HH:MM" string in local + * time. Out buffer must hold at least 17 bytes. Negative or zero epochs + * render as "(unknown)". + */ +static void format_mtime(int64_t epoch, char *out, size_t cap) +{ + if (epoch <= 0) { + snprintf(out, cap, "(unknown)"); + return; + } + time_t t = (time_t) epoch; + struct tm lt; + if (!localtime_r(&t, <)) { + snprintf(out, cap, "(unknown)"); + return; + } + strftime(out, cap, "%Y-%m-%d %H:%M", <); +} + +/* Truncate a digest to ":<13-hex>..." so wide manifest digests still + * align in the table. Mirrors the short_digest helper in inspect.c without + * the dependency on that translation unit. + */ +static void short_digest(const char *full, char out[24]) +{ + if (!full) { + snprintf(out, 24, "(null)"); + return; + } + size_t len = strlen(full); + if (len <= 22) { + snprintf(out, 24, "%s", full); + return; + } + snprintf(out, 24, "%.19s...", full); +} + +static const char *pin_status_label(oci_status_pin_code_t c) +{ + switch (c) { + case OCI_STATUS_PIN_OK: + return "ok"; + case OCI_STATUS_PIN_MISSING_MANIFEST: + return "missing manifest"; + case OCI_STATUS_PIN_CORRUPT_MANIFEST: + return "corrupt manifest"; + case OCI_STATUS_PIN_CORRUPT_CONFIG: + return "corrupt config"; + case OCI_STATUS_PIN_INDEX_NO_ARM64: + return "no linux/arm64 entry"; + } + return "unknown"; +} + +static const char *unpacked_status_label(oci_status_unpacked_code_t c) +{ + switch (c) { + case OCI_STATUS_UNPACKED_OK: + return "ok"; + case OCI_STATUS_UNPACKED_MISSING_ORIGIN: + return "missing origin"; + case OCI_STATUS_UNPACKED_CORRUPT_ORIGIN: + return "corrupt origin"; + } + return "unknown"; +} + +/* Emit one JSON-quoted token with backslash and double-quote escaping. + * Mirrors the print_quoted_token static in inspect.c without dragging the + * dependency; control chars pass through (operator-facing strings, never + * raw binary). + */ +static void emit_json_quoted(FILE *out, const char *s) +{ + fputc('"', out); + if (s) { + for (const char *p = s; *p; p++) { + if (*p == '"' || *p == '\\') + fputc('\\', out); + fputc(*p, out); + } + } + fputc('"', out); +} + +static void render_status_human(FILE *out, const oci_status_t *st) +{ + /* Pins section. */ + if (st->pin_count == 0) { + fprintf(out, "PINS (0): (none)\n\n"); + } else { + fprintf(out, "PINS (%zu):\n", st->pin_count); + for (size_t i = 0; i < st->pin_count; i++) { + const oci_status_pin_entry_t *p = &st->pins[i]; + char short_d[24]; + short_digest(p->digest, short_d); + if (p->status != OCI_STATUS_PIN_OK) { + fprintf(out, " %-40s %-22s (%s)\n", + p->name ? p->name : "(unknown)", short_d, + pin_status_label(p->status)); + continue; + } + char mtime_s[20]; + format_mtime(p->last_seen_mtime, mtime_s, sizeof(mtime_s)); + fprintf(out, " %-40s %-22s %2zu layers %s\n", + p->name ? p->name : "(unknown)", short_d, p->layer_count, + mtime_s); + } + fputc('\n', out); + } + + /* Unpacked sysroots section. */ + if (st->unpacked_count == 0) { + fprintf(out, "UNPACKED SYSROOTS (0): (none)\n\n"); + } else { + fprintf(out, "UNPACKED SYSROOTS (%zu):\n", st->unpacked_count); + for (size_t i = 0; i < st->unpacked_count; i++) { + const oci_status_unpacked_entry_t *u = &st->unpacked[i]; + if (u->status != OCI_STATUS_UNPACKED_OK) { + fprintf(out, " %s (%s)\n", u->path ? u->path : "(unknown)", + unpacked_status_label(u->status)); + continue; + } + char short_d[24]; + short_digest(u->manifest_digest, short_d); + char bytes_s[24]; + if (st->disk_usage_skipped) + snprintf(bytes_s, sizeof(bytes_s), "(skipped)"); + else + format_bytes(u->tree_bytes, bytes_s, sizeof(bytes_s)); + fprintf(out, " %s %-22s %s\n", u->path ? u->path : "(unknown)", + short_d, bytes_s); + } + fputc('\n', out); + } + + /* Store totals section. */ + char blob_b[24], layer_b[24], stack_b[24], total_b[24]; + if (st->disk_usage_skipped) { + snprintf(blob_b, sizeof(blob_b), "(skipped)"); + snprintf(layer_b, sizeof(layer_b), "(skipped)"); + snprintf(stack_b, sizeof(stack_b), "(skipped)"); + snprintf(total_b, sizeof(total_b), "(skipped)"); + } else { + format_bytes(st->blob_bytes_total, blob_b, sizeof(blob_b)); + format_bytes(st->layer_cache_bytes_total, layer_b, sizeof(layer_b)); + format_bytes(st->stack_cache_bytes_total, stack_b, sizeof(stack_b)); + uint64_t total = st->blob_bytes_total + st->layer_cache_bytes_total + + st->stack_cache_bytes_total; + format_bytes(total, total_b, sizeof(total_b)); + } + fprintf(out, "STORE TOTALS:\n"); + fprintf(out, " blobs: %zu (%s)\n", st->blob_count, blob_b); + fprintf(out, " layers raw: %zu of %zu reachable cached (%s)\n", + st->diff_ids_populated, st->diff_ids_reachable, layer_b); + fprintf(out, " layers stack: %zu of %zu reachable cached (%s)\n", + st->chain_ids_populated, st->chain_ids_reachable, stack_b); + fprintf(out, " total: %s\n", total_b); + if (st->disk_usage_skipped) + fprintf(out, " (disk usage skipped)\n"); +} + +static void render_status_json(FILE *out, const oci_status_t *st) +{ + fprintf(out, "{\"schemaVersion\":1,\"pins\":["); + for (size_t i = 0; i < st->pin_count; i++) { + const oci_status_pin_entry_t *p = &st->pins[i]; + if (i > 0) + fputc(',', out); + fprintf(out, "{\"name\":"); + emit_json_quoted(out, p->name); + fprintf(out, ",\"digest\":"); + emit_json_quoted(out, p->digest); + fprintf(out, + ",\"manifest_size\":%llu,\"config_size\":%llu,\"layer_count\":" + "%zu,\"last_seen_mtime\":%lld,\"status\":\"%s\"}", + (unsigned long long) p->manifest_size, + (unsigned long long) p->config_size, p->layer_count, + (long long) p->last_seen_mtime, pin_status_label(p->status)); + } + fprintf(out, "],\"unpacked\":["); + for (size_t i = 0; i < st->unpacked_count; i++) { + const oci_status_unpacked_entry_t *u = &st->unpacked[i]; + if (i > 0) + fputc(',', out); + fprintf(out, "{\"path\":"); + emit_json_quoted(out, u->path); + fprintf(out, ",\"manifest_digest\":"); + emit_json_quoted(out, u->manifest_digest); + fprintf(out, + ",\"layer_count\":%zu,\"tree_bytes\":%llu,\"status\":\"%s\"}", + u->layer_count, (unsigned long long) u->tree_bytes, + unpacked_status_label(u->status)); + } + fprintf(out, "],\"totals\":{"); + fprintf(out, "\"blob_count\":%zu,\"blob_bytes\":%llu,", st->blob_count, + (unsigned long long) st->blob_bytes_total); + fprintf(out, "\"layer_cache_count\":%zu,\"layer_cache_bytes\":%llu,", + st->layer_cache_count, + (unsigned long long) st->layer_cache_bytes_total); + fprintf(out, "\"stack_cache_count\":%zu,\"stack_cache_bytes\":%llu,", + st->stack_cache_count, + (unsigned long long) st->stack_cache_bytes_total); + fprintf(out, "\"diff_ids_reachable\":%zu,\"diff_ids_populated\":%zu,", + st->diff_ids_reachable, st->diff_ids_populated); + fprintf(out, "\"chain_ids_reachable\":%zu,\"chain_ids_populated\":%zu,", + st->chain_ids_reachable, st->chain_ids_populated); + fprintf(out, "\"disk_usage_skipped\":%s", + st->disk_usage_skipped ? "true" : "false"); + fprintf(out, "}}\n"); +} + +static int cmd_status(int argc, char **argv) +{ + status_args_t args = {0}; + int prc = parse_status_args(argc, argv, &args); + if (prc == 1) + return print_usage(stdout); + if (prc < 0) + return 2; + + char *default_root = NULL; + const char *store_root = args.store_root; + if (!store_root) { + default_root = oci_store_default_root(); + if (!default_root) { + fprintf(stderr, + "error: could not determine default store root " + "(HOME not set?)\n"); + return 1; + } + store_root = default_root; + } + + oci_store_t *store = oci_store_open(store_root); + if (!store) { + fprintf(stderr, "error: could not open store at %s: %s\n", store_root, + strerror(errno)); + free(default_root); + return 1; + } + + oci_status_options_t sopts = { + .volume_root = args.volume_root, + .skip_disk_usage = args.no_disk_usage, + }; + oci_status_t st = {0}; + const char *err = NULL; + if (oci_status_compute(store, &sopts, &st, &err) < 0) { + fprintf(stderr, "error: status failed: %s\n", + err ? err : strerror(errno)); + oci_status_free(&st); + oci_store_close(store); + free(default_root); + return 1; + } + + if (args.json) + render_status_json(stdout, &st); + else + render_status_human(stdout, &st); + + oci_status_free(&st); + oci_store_close(store); + free(default_root); + return 0; +} + +int oci_cli_main(int argc, char **argv) +{ + if (argc < 2) + return print_usage(stderr); + + const char *sub = argv[1]; + if (!strcmp(sub, "-h") || !strcmp(sub, "--help") || !strcmp(sub, "help")) + return print_usage(stdout); + if (!strcmp(sub, "inspect")) + return cmd_inspect(argc - 1, argv + 1); + if (!strcmp(sub, "pull")) + return cmd_pull(argc - 1, argv + 1); + if (!strcmp(sub, "unpack")) + return cmd_unpack(argc - 1, argv + 1); + if (!strcmp(sub, "clone")) + return cmd_clone(argc - 1, argv + 1); + if (!strcmp(sub, "run")) + return oci_cli_run(argc - 1, argv + 1); + if (!strcmp(sub, "prune")) + return cmd_prune(argc - 1, argv + 1); + if (!strcmp(sub, "rebuild-cache")) + return cmd_rebuild_cache(argc - 1, argv + 1); + if (!strcmp(sub, "status")) + return cmd_status(argc - 1, argv + 1); + if (!strcmp(sub, "list") || !strcmp(sub, "ls")) + return cmd_not_implemented("list"); + + fprintf(stderr, "error: unknown oci subcommand: %s\n", sub); + return print_usage(stderr); +} diff --git a/src/oci/cli.h b/src/oci/cli.h new file mode 100644 index 0000000..781efd4 --- /dev/null +++ b/src/oci/cli.h @@ -0,0 +1,18 @@ +/* `elfuse oci` subcommand dispatch + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Sits on the side of the main argv parser: when argv[1] == "oci" the rest + * of the command line is forwarded here. Subcommands are pull, inspect, + * prune, and list. Only inspect parses a reference today; the others return + * a deterministic "not yet implemented" exit so users can discover the + * surface without crashes. + */ + +#pragma once + +/* argc/argv are the slice starting at "oci" (i.e. argv[0] == "oci"). Returns + * a process exit code suitable for main() to return directly. + */ +int oci_cli_main(int argc, char **argv); diff --git a/src/oci/clone-rootfs.c b/src/oci/clone-rootfs.c new file mode 100644 index 0000000..67ee976 --- /dev/null +++ b/src/oci/clone-rootfs.c @@ -0,0 +1,190 @@ +/* OCI per-run rootfs via clonefile(2) + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "oci/clone-rootfs.h" + +#define CR_PATH_MAX 4096 + +static int set_err(const char **err, const char *msg, int err_no) +{ + if (err) + *err = msg; + errno = err_no; + return -1; +} + +static int rand_hex(char *out, size_t n_hex) +{ + /* n_hex is the number of hex chars in the output; the helper reads + * n_hex/2 random bytes from getentropy and prints them. + */ + size_t need = n_hex / 2; + uint8_t buf[32]; + if (need > sizeof(buf)) + return -1; + if (getentropy(buf, need) < 0) + return -1; + static const char hex[] = "0123456789abcdef"; + for (size_t i = 0; i < need; i++) { + out[i * 2] = hex[buf[i] >> 4]; + out[i * 2 + 1] = hex[buf[i] & 0xf]; + } + out[n_hex] = '\0'; + return 0; +} + +static int mkdir_p(const char *path) +{ + char buf[CR_PATH_MAX]; + size_t n = strlen(path); + if (n >= sizeof(buf)) + return -1; + memcpy(buf, path, n + 1); + for (char *p = buf + 1; *p; p++) { + if (*p != '/') + continue; + *p = '\0'; + if (mkdir(buf, 0755) < 0 && errno != EEXIST) + return -1; + *p = '/'; + } + if (mkdir(buf, 0755) < 0 && errno != EEXIST) + return -1; + return 0; +} + +int oci_clone_rootfs(const char *src_image_dir, + const char *volume_root, + char **out_run_dir, + const char **err) +{ + static const char *dummy_err; + if (!err) + err = &dummy_err; + *err = NULL; + if (!src_image_dir || !volume_root || !out_run_dir) + return set_err(err, "clone: NULL argument", EINVAL); + *out_run_dir = NULL; + + /* Image dir must exist and be a directory. */ + struct stat sb; + if (stat(src_image_dir, &sb) < 0) + return set_err(err, "clone: src image stat failed", errno); + if (!S_ISDIR(sb.st_mode)) + return set_err(err, "clone: src image is not a directory", ENOTDIR); + + /* Provision volume_root/runs/. */ + char runs_dir[CR_PATH_MAX]; + if ((size_t) snprintf(runs_dir, sizeof(runs_dir), "%s/runs", volume_root) >= + sizeof(runs_dir)) + return set_err(err, "clone: runs path overflow", ENAMETOOLONG); + if (mkdir_p(runs_dir) < 0) + return set_err(err, "clone: cannot create runs dir", errno); + + /* Pick a fresh run id. 12 hex chars = 48 bits of randomness; ample + * for elfuse process lifetimes. + */ + char id[13]; + if (rand_hex(id, 12) < 0) + return set_err(err, "clone: getentropy failed", errno); + + char run_dir[CR_PATH_MAX]; + if ((size_t) snprintf(run_dir, sizeof(run_dir), "%s/%s", runs_dir, id) >= + sizeof(run_dir)) + return set_err(err, "clone: run path overflow", ENAMETOOLONG); + + /* clonefile creates the destination atomically: pass dst that does + * NOT exist (the call itself creates the directory). Use + * CLONE_NOFOLLOW so a symlink at the src root is not followed off + * the immutable image. + */ + if (clonefile(src_image_dir, run_dir, CLONE_NOFOLLOW) < 0) + return set_err(err, "clone: clonefile failed", errno); + + char *dup = strdup(run_dir); + if (!dup) { + /* Roll back the clone so the caller sees no half-state. */ + (void) oci_clone_rootfs_remove(run_dir, NULL); + return set_err(err, "clone: strdup failed", ENOMEM); + } + *out_run_dir = dup; + return 0; +} + +int oci_clone_rootfs_remove(const char *run_dir, const char **err) +{ + static const char *dummy_err; + if (!err) + err = &dummy_err; + *err = NULL; + if (!run_dir) + return set_err(err, "clone remove: NULL path", EINVAL); + + struct stat st; + if (lstat(run_dir, &st) < 0) { + if (errno == ENOENT) + return 0; + return set_err(err, "clone remove: lstat failed", errno); + } + if (!S_ISDIR(st.st_mode)) { + if (unlink(run_dir) < 0) + return set_err(err, "clone remove: unlink failed", errno); + return 0; + } + DIR *d = opendir(run_dir); + if (!d) + return set_err(err, "clone remove: opendir failed", errno); + struct dirent *de; + int rc = 0; + while ((de = readdir(d))) { + if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) + continue; + char child[CR_PATH_MAX]; + if ((size_t) snprintf(child, sizeof(child), "%s/%s", run_dir, + de->d_name) >= sizeof(child)) { + rc = -1; + errno = ENAMETOOLONG; + break; + } + if (oci_clone_rootfs_remove(child, NULL) < 0) { + rc = -1; + break; + } + } + closedir(d); + if (rc == 0 && rmdir(run_dir) < 0) + return set_err(err, "clone remove: rmdir failed", errno); + return rc; +} + +int oci_clone_rootfs_gc(const char *volume_root, + time_t older_than, + const char **err) +{ + (void) volume_root; + (void) older_than; + if (err) + *err = NULL; + /* Phase 2 stub: `elfuse oci prune` does not yet sweep stale runs. + * Phase 3 will walk volume_root/runs/ and unlink entries older + * than older_than. The stub returns 0 so the CLI can call it + * unconditionally without a feature gate. + */ + return 0; +} diff --git a/src/oci/clone-rootfs.h b/src/oci/clone-rootfs.h new file mode 100644 index 0000000..a4b1918 --- /dev/null +++ b/src/oci/clone-rootfs.h @@ -0,0 +1,54 @@ +/* OCI per-run rootfs via clonefile(2) + * + * Phase 2 commits Q2 of oci-roadmap.md to APFS clonefile-based + * copy-up: each `elfuse oci clone` invocation gets a fresh directory + * tree cloned from the immutable image sysroot. APFS file-level CoW + * makes the clone nearly O(1) at start, and only modified files + * allocate new blocks. + * + * Apple's clonefile(2) is recursive across directories since macOS + * 10.12. Hardlinks inside the source tree survive the clone metadata + * pass; cross-tree hardlinks back to the immutable image are NOT + * created, so the layer applier in commit 5 must materialize intra- + * image hardlinks via link(2) rather than relying on the clone. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include + +/* Clone the immutable image directory at src_image_dir into a fresh + * run directory under volume_root/runs/. On success *out_run_dir + * receives a heap-allocated absolute path (caller frees) and returns + * 0. On failure returns -1 with errno set and *err populated. + * + * Errors: + * ENOTSUP the volume does not support APFS clones (non-APFS + * scratch or a future fs without clonefile) + * ENOSPC sparsebundle full + * EACCES volume read-only or run_dir creation denied + * ENAMETOOLONG generated run_dir path overflows the buffer + */ +int oci_clone_rootfs(const char *src_image_dir, + const char *volume_root, + char **out_run_dir, + const char **err); + +/* Recursively remove a run directory previously returned by + * oci_clone_rootfs. Best effort: returns 0 on full removal, -1 on + * the first irrecoverable error. + */ +int oci_clone_rootfs_remove(const char *run_dir, const char **err); + +/* Sweep volume_root/runs/ for stale clones older than older_than + * (epoch seconds). Phase 2 ships this as a stub returning 0 because + * `elfuse oci prune` is not extended in this PR; Phase 3 will + * actually walk the directory. + */ +int oci_clone_rootfs_gc(const char *volume_root, + time_t older_than, + const char **err); diff --git a/src/oci/decompress.c b/src/oci/decompress.c new file mode 100644 index 0000000..20d3134 --- /dev/null +++ b/src/oci/decompress.c @@ -0,0 +1,326 @@ +/* OCI decompression dispatch (gzip + zstd + passthrough) + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include +#include + +#include + +/* Opt into the libzstd static-only interface to access ZSTD_ErrorCode + * and the named constants (ZSTD_error_frameParameter_windowTooLarge in + * particular). The decoder still goes through the public symbols; only + * the error classification consults the static-only enum. + */ +#define ZSTD_STATIC_LINKING_ONLY +#include +#include + +#include "oci/decompress.h" + +/* 128 MiB; rejects pathological zstd headers without hurting real layers. */ +#define OCI_ZSTD_MAX_WINDOW_LOG 27 + +/* Decoder-side input buffer. Sized to one host page so libcurl-style + * pipelines do not pay extra syscalls per read. + */ +#define OCI_DECOMPRESS_IBUF 65536 + +typedef enum { + OCI_STREAM_NONE, + OCI_STREAM_GZIP, + OCI_STREAM_ZSTD, +} oci_stream_kind_t; + +struct oci_stream { + oci_stream_kind_t kind; + int fd; + bool eof; + const char *last_err; + + /* zlib backend */ + z_stream zs; + bool zs_inited; + + /* zstd backend */ + ZSTD_DCtx *zd; + + /* Shared input buffer; both backends pull from it. The position + * advances as the decoder consumes input, and refill_input pulls + * from fd when it is exhausted. + */ + uint8_t *ibuf; + size_t ibuf_len; + size_t ibuf_pos; +}; + +static ssize_t read_some(int fd, void *buf, size_t cap) +{ + while (1) { + ssize_t n = read(fd, buf, cap); + if (n < 0 && errno == EINTR) + continue; + return n; + } +} + +static int refill_input(oci_stream_t *s) +{ + if (s->ibuf_pos < s->ibuf_len) + return 0; + ssize_t n = read_some(s->fd, s->ibuf, OCI_DECOMPRESS_IBUF); + if (n < 0) { + s->last_err = "decompress: input read failed"; + return -1; + } + s->ibuf_pos = 0; + s->ibuf_len = (size_t) n; + return 0; +} + +oci_stream_t *oci_decompress_open(int fd, + oci_compression_t alg, + const char **err) +{ + static const char *dummy_err; + if (!err) + err = &dummy_err; + *err = NULL; + + if (fd < 0) { + errno = EBADF; + *err = "decompress: invalid fd"; + return NULL; + } + + oci_stream_t *s = calloc(1, sizeof(*s)); + if (!s) { + errno = ENOMEM; + *err = "decompress: state allocation failed"; + return NULL; + } + s->fd = fd; + + s->ibuf = malloc(OCI_DECOMPRESS_IBUF); + if (!s->ibuf) { + free(s); + errno = ENOMEM; + *err = "decompress: input buffer allocation failed"; + return NULL; + } + + switch (alg) { + case OCI_COMPRESSION_NONE: + s->kind = OCI_STREAM_NONE; + return s; + case OCI_COMPRESSION_GZIP: { + s->kind = OCI_STREAM_GZIP; + /* windowBits=15 + 32 enables gzip + zlib header auto-detection; + * raw deflate without a header is NOT accepted because real OCI + * gzip layers always carry the standard gzip wrapper. + */ + int zrc = inflateInit2(&s->zs, 15 + 32); + if (zrc != Z_OK) { + free(s->ibuf); + free(s); + errno = EINVAL; + *err = "decompress: zlib inflateInit2 failed"; + return NULL; + } + s->zs_inited = true; + return s; + } + case OCI_COMPRESSION_ZSTD: { + s->kind = OCI_STREAM_ZSTD; + s->zd = ZSTD_createDCtx(); + if (!s->zd) { + free(s->ibuf); + free(s); + errno = ENOMEM; + *err = "decompress: ZSTD_createDCtx failed"; + return NULL; + } + size_t prc = ZSTD_DCtx_setParameter(s->zd, ZSTD_d_windowLogMax, + OCI_ZSTD_MAX_WINDOW_LOG); + if (ZSTD_isError(prc)) { + ZSTD_freeDCtx(s->zd); + free(s->ibuf); + free(s); + errno = EINVAL; + *err = "decompress: ZSTD_d_windowLogMax rejected"; + return NULL; + } + return s; + } + default: + free(s->ibuf); + free(s); + errno = EINVAL; + *err = "decompress: unsupported compression"; + return NULL; + } +} + +static ssize_t read_passthrough(oci_stream_t *s, void *buf, size_t cap) +{ + if (s->ibuf_pos < s->ibuf_len) { + size_t left = s->ibuf_len - s->ibuf_pos; + size_t take = left < cap ? left : cap; + memcpy(buf, s->ibuf + s->ibuf_pos, take); + s->ibuf_pos += take; + return (ssize_t) take; + } + /* The passthrough does not buffer beyond what was pre-read; once + * exhausted, hand the caller's buf directly to read(2) so a tar + * driver can stream large payloads without an extra copy. + */ + return read_some(s->fd, buf, cap); +} + +static ssize_t read_gzip(oci_stream_t *s, void *buf, size_t cap) +{ + if (s->eof) + return 0; + s->zs.next_out = buf; + s->zs.avail_out = (uInt) (cap > UINT32_MAX ? UINT32_MAX : cap); + + while (s->zs.avail_out > 0) { + if (s->ibuf_pos == s->ibuf_len) { + if (refill_input(s) < 0) { + errno = EIO; + return -1; + } + if (s->ibuf_len == 0) { + /* Source EOF before zlib reported Z_STREAM_END means + * the gzip frame was truncated. + */ + if (s->zs.avail_out == cap) { + s->eof = true; + return 0; + } + s->last_err = "decompress: gzip stream truncated"; + errno = EIO; + return -1; + } + } + s->zs.next_in = s->ibuf + s->ibuf_pos; + s->zs.avail_in = (uInt) (s->ibuf_len - s->ibuf_pos); + + int zrc = inflate(&s->zs, Z_NO_FLUSH); + size_t consumed = (s->ibuf_len - s->ibuf_pos) - s->zs.avail_in; + s->ibuf_pos += consumed; + if (zrc == Z_STREAM_END) { + s->eof = true; + break; + } + if (zrc == Z_OK || zrc == Z_BUF_ERROR) { + /* Z_BUF_ERROR with no progress just means the decoder wants + * more input next call; loop and refill. + */ + continue; + } + s->last_err = s->zs.msg ? s->zs.msg : "decompress: zlib inflate failed"; + errno = EIO; + return -1; + } + return (ssize_t) (cap - s->zs.avail_out); +} + +static ssize_t read_zstd(oci_stream_t *s, void *buf, size_t cap) +{ + if (s->eof) + return 0; + ZSTD_outBuffer out = {.dst = buf, .size = cap, .pos = 0}; + + while (out.pos < out.size) { + if (s->ibuf_pos == s->ibuf_len) { + if (refill_input(s) < 0) { + errno = EIO; + return -1; + } + if (s->ibuf_len == 0) { + /* libzstd returns 0 from decompressStream when it + * finishes a frame; if the caller sees source EOF here + * with no output produced yet, it is a clean end. + */ + if (out.pos == 0) { + s->eof = true; + return 0; + } + s->last_err = "decompress: zstd stream truncated"; + errno = EIO; + return -1; + } + } + ZSTD_inBuffer in = { + .src = s->ibuf + s->ibuf_pos, + .size = s->ibuf_len - s->ibuf_pos, + .pos = 0, + }; + size_t rrc = ZSTD_decompressStream(s->zd, &out, &in); + s->ibuf_pos += in.pos; + if (ZSTD_isError(rrc)) { + ZSTD_ErrorCode ec = ZSTD_getErrorCode(rrc); + if (ec == ZSTD_error_frameParameter_windowTooLarge) { + s->last_err = "decompress: zstd window exceeds cap"; + errno = EINVAL; + } else { + s->last_err = ZSTD_getErrorName(rrc); + errno = EIO; + } + return -1; + } + if (rrc == 0) { + /* Frame complete; libzstd may still accept more frames, but + * OCI layers ship single-frame so the reader treats this as + * EOF. + */ + s->eof = true; + break; + } + } + return (ssize_t) out.pos; +} + +ssize_t oci_stream_read(oci_stream_t *s, void *buf, size_t cap) +{ + if (!s || !buf) { + errno = EINVAL; + return -1; + } + if (cap == 0) + return 0; + switch (s->kind) { + case OCI_STREAM_NONE: + return read_passthrough(s, buf, cap); + case OCI_STREAM_GZIP: + return read_gzip(s, buf, cap); + case OCI_STREAM_ZSTD: + return read_zstd(s, buf, cap); + } + errno = EINVAL; + return -1; +} + +void oci_stream_close(oci_stream_t *s) +{ + if (!s) + return; + if (s->zs_inited) + inflateEnd(&s->zs); + if (s->zd) + ZSTD_freeDCtx(s->zd); + free(s->ibuf); + free(s); +} + +const char *oci_stream_last_error(const oci_stream_t *s) +{ + return s ? s->last_err : NULL; +} diff --git a/src/oci/decompress.h b/src/oci/decompress.h new file mode 100644 index 0000000..1bfb0e2 --- /dev/null +++ b/src/oci/decompress.h @@ -0,0 +1,55 @@ +/* OCI layer-blob decompression dispatch + * + * Wraps zlib (gzip), libzstd (vendored decode-only), and a passthrough + * stream behind one read-only oci_stream_t. The tar reader is + * compression-agnostic; oci/decompress.c is the only translation unit + * in the project that includes externals/zstd/lib/zstd.h, so any + * future swap-out of the compression backend is local to this module. + * + * The zstd backend caps the decoder window log at 27 (128 MiB) so a + * hostile or unintentionally fat layer cannot exhaust host memory. + * Real-world OCI registry layers stay well below this; the regression + * test pins the boundary at 28-bit windows rejecting with EINVAL. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include + +#include "oci/media-type.h" + +typedef struct oci_stream oci_stream_t; + +/* Open a decompression stream over fd. The caller retains fd ownership; + * oci_stream_close does NOT close it. On error returns NULL and sets + * *err to a static description string plus errno. + * + * For OCI_COMPRESSION_NONE this is a thin passthrough wrapper; for GZIP + * the implementation uses zlib's inflate (auto-detecting raw vs gzip + * via inflateInit2 with windowBits=47); for ZSTD it uses libzstd's + * streaming ZSTD_DCtx capped to a 128 MiB decoder window. + */ +oci_stream_t *oci_decompress_open(int fd, + oci_compression_t alg, + const char **err); + +/* Read up to cap bytes. Returns: + * >0 bytes copied into buf (may be a short read; caller loops) + * 0 end of compressed stream (clean) + * -1 decompression or I/O error; errno set, callers may surface + * EIO with the decoder error string + */ +ssize_t oci_stream_read(oci_stream_t *s, void *buf, size_t cap); + +/* Release decoder state. Does NOT close the underlying fd. Safe on NULL. */ +void oci_stream_close(oci_stream_t *s); + +/* For diagnostics only: last static error string the decoder produced. + * Returns NULL if the stream has not failed. The string is owned by + * the stream and remains valid until oci_stream_close. + */ +const char *oci_stream_last_error(const oci_stream_t *s); diff --git a/src/oci/dedup-metrics.c b/src/oci/dedup-metrics.c new file mode 100644 index 0000000..6b73102 --- /dev/null +++ b/src/oci/dedup-metrics.c @@ -0,0 +1,565 @@ +/* OCI cross-image layer dedup metrics + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * One walker per source: + * + * - Pins from index.json. For each pin's manifest digest, the manifest blob + * is parsed; an image-manifest contributes its config blob's diff_ids + * directly, an image-index contributes the picked linux/arm64 sub- + * manifest's diff_ids. Anything that fails along the way is a per-pin + * skip, not a hard error (see header for rationale: dedup is + * informational, not GC). + * + * - Unpacked sysroots under volume_root/images/sha256-/. The origin + * sidecar already records the manifest digest, the config digest, and + * the diff_id list, so this walker never has to read a blob. The walker + * dedupes against the pin walk by manifest_digest so a pin pointing at + * the same manifest as an unpacked tree does not count twice. + * + * Once both walkers finish, the result sets are intersected against the + * target's diff_ids and its per-layer ChainID chain. shared_bytes accumulates + * the on-disk tree size of layers/sha256// entries that exist and + * fall in the intersection; entries absent from the raw cache still register + * as shared layers (cross-image overlap is independent of cache populate). + * + * Memory ownership: + * - oci_dedup_metrics_compute zeroes *out on entry and on failure, so the + * caller can pass a stack-resident struct safely. + * - Internal scratch state (oci_digest_set_t accumulators, the target's + * ChainID chain heap, manifest parses) is owned and freed by this file. + * + * Path budget: DEDUP_PATH_MAX = 4096 matches src/oci/store.c::STORE_PATH_MAX + * so a layers/// tree walk has the same headroom store.c assumes. + */ + +#include "dedup-metrics.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "blob-store.h" +#include "digest-set.h" +#include "manifest.h" +#include "origin-meta.h" +#include "volume.h" + +#define DEDUP_PATH_MAX 4096 + +/* Largest blob this helper will memory-map. The manifest renderer in + * src/oci/inspect.c uses 64 MiB; the cross-image walker reads many blobs + * back to back so the cap is the same to keep parser failure modes + * uniform. + */ +#define DEDUP_BLOB_MAX ((size_t) 64 * 1024 * 1024) + +/* Slurp a blob from the store into a fresh heap buffer. The buffer is + * NUL-terminated so the caller can pass the byte range as either a + * length-bounded array or a C string. Returns 0 on success, -1 with + * errno preserved on failure. + */ +static int slurp_blob(oci_blob_store_t *blobs, + const char *digest_str, + char **out_body, + size_t *out_len) +{ + oci_digest_algo_t algo; + char hex[OCI_DIGEST_HEX_MAX + 1]; + if (!oci_digest_parse(digest_str, &algo, hex)) { + errno = EINVAL; + return -1; + } + char path[DEDUP_PATH_MAX]; + int n = oci_blob_store_path(blobs, algo, hex, path, sizeof(path)); + if (n < 0 || (size_t) n >= sizeof(path)) { + errno = ENAMETOOLONG; + return -1; + } + int fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + return -1; + struct stat st; + if (fstat(fd, &st) < 0) { + int saved = errno; + close(fd); + errno = saved; + return -1; + } + if (st.st_size < 0 || (uintmax_t) st.st_size > DEDUP_BLOB_MAX) { + close(fd); + errno = EFBIG; + return -1; + } + size_t want = (size_t) st.st_size; + char *buf = malloc(want + 1); + if (!buf) { + close(fd); + errno = ENOMEM; + return -1; + } + size_t off = 0; + while (off < want) { + ssize_t r = read(fd, buf + off, want - off); + if (r < 0) { + if (errno == EINTR) + continue; + int saved = errno; + free(buf); + close(fd); + errno = saved; + return -1; + } + if (r == 0) + break; + off += (size_t) r; + } + close(fd); + if (off != want) { + free(buf); + errno = EIO; + return -1; + } + buf[want] = '\0'; + *out_body = buf; + *out_len = want; + return 0; +} + +/* Free a NULL-terminated char ** array allocated as { strdup, strdup, NULL }. + */ +static void free_strv(char **v) +{ + if (!v) + return; + for (size_t i = 0; v[i]; i++) + free(v[i]); + free((void *) v); +} + +/* Walk a path tree summing the st_size of every regular file. Returns the + * accumulated total on success or 0 when the entry is absent / unreadable + * (the caller treats absence as "cache entry not populated", which is + * shared_bytes == 0 for that diff_id). Symlinks are skipped (lstat) so a + * stray symlink can never inflate the count by following it into the + * blob store. + */ +static uint64_t sum_tree_size(const char *path) +{ + struct stat st; + if (lstat(path, &st) < 0) + return 0; + if (S_ISREG(st.st_mode)) + return (uint64_t) st.st_size; + if (!S_ISDIR(st.st_mode)) + return 0; + DIR *d = opendir(path); + if (!d) + return 0; + uint64_t total = 0; + struct dirent *de; + while ((de = readdir(d)) != NULL) { + if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) + continue; + char child[DEDUP_PATH_MAX]; + int n = snprintf(child, sizeof(child), "%s/%s", path, de->d_name); + if (n < 0 || (size_t) n >= sizeof(child)) + continue; + total += sum_tree_size(child); + } + closedir(d); + return total; +} + +/* Add every diff_id from cfg.rootfs_diff_ids and every ChainID in the chain + * built from those diff_ids into the accumulators. Returns 0 on success, + * -1 with errno set on allocation failure inside oci_digest_set_add or + * oci_chainid_compute (caller treats this as a per-image skip). + */ +static int accumulate_chain(char *const *diff_ids, + oci_digest_set_t *diff_acc, + oci_digest_set_t *chain_acc) +{ + char prev[OCI_DIGEST_HEX_MAX + 16] = ""; + for (size_t i = 0; diff_ids[i]; i++) { + if (oci_digest_set_add(diff_acc, diff_ids[i]) < 0) + return -1; + char chain[OCI_DIGEST_HEX_MAX + 16]; + const char *prev_arg = (i == 0) ? NULL : prev; + if (oci_chainid_compute(prev_arg, diff_ids[i], chain, sizeof(chain)) < + 0) + return -1; + memcpy(prev, chain, strlen(chain) + 1); + if (oci_digest_set_add(chain_acc, chain) < 0) + return -1; + } + return 0; +} + +/* For an arbitrary manifest blob digest, locate the per-arch image-config + * digest. Walks one level of image-index indirection (linux/arm64 pick). + * Returns a heap-allocated config digest string on success (caller frees), + * or NULL with errno set when the manifest is missing, malformed, or the + * index has no linux/arm64 entry whose sub-manifest blob is on disk. + */ +static char *resolve_config_digest(oci_store_t *store, + const char *manifest_digest) +{ + oci_blob_store_t *blobs = oci_store_blobs(store); + char *body = NULL; + size_t body_len = 0; + if (slurp_blob(blobs, manifest_digest, &body, &body_len) < 0) + return NULL; + + oci_manifest_t mf = {0}; + if (oci_manifest_parse(body, body_len, &mf, NULL) == 0) { + char *cfg = strdup(mf.config.digest_str); + oci_manifest_free(&mf); + free(body); + if (!cfg) { + errno = ENOMEM; + return NULL; + } + return cfg; + } + + oci_index_t idx = {0}; + if (oci_index_parse(body, body_len, &idx, NULL) < 0) { + free(body); + errno = EINVAL; + return NULL; + } + free(body); + + const oci_index_entry_t *picked = oci_index_pick_linux_arm64(&idx); + if (!picked) { + oci_index_free(&idx); + errno = ENOENT; + return NULL; + } + char *sub_digest = strdup(picked->desc.digest_str); + oci_index_free(&idx); + if (!sub_digest) { + errno = ENOMEM; + return NULL; + } + /* Recurse on the picked sub-manifest. Only one level deep is expected, + * but resolve_config_digest handles arbitrary nesting safely because + * the recursion terminates when the body parses as an image-manifest. + */ + char *cfg = resolve_config_digest(store, sub_digest); + free(sub_digest); + return cfg; +} + +/* Read the image-config at config_digest and return a freshly allocated + * NULL-terminated copy of its rootfs_diff_ids array. Returns NULL with + * errno set on any failure; the caller treats this as a per-image skip. + * The returned strv must be released via free_strv. + */ +static char **load_diff_ids(oci_store_t *store, const char *config_digest) +{ + oci_blob_store_t *blobs = oci_store_blobs(store); + char *body = NULL; + size_t body_len = 0; + if (slurp_blob(blobs, config_digest, &body, &body_len) < 0) + return NULL; + + oci_image_config_t cfg = {0}; + if (oci_image_config_parse(body, body_len, &cfg, NULL) < 0) { + free(body); + errno = EINVAL; + return NULL; + } + free(body); + + /* Count and copy. rootfs_diff_ids is guaranteed non-NULL by the parser + * contract; an image with zero layers gives back a one-element array + * containing only the NULL terminator. + */ + size_t n = 0; + while (cfg.rootfs_diff_ids[n]) + n++; + char **copy = (char **) calloc(n + 1, sizeof(*copy)); + if (!copy) { + oci_image_config_free(&cfg); + errno = ENOMEM; + return NULL; + } + for (size_t i = 0; i < n; i++) { + copy[i] = strdup(cfg.rootfs_diff_ids[i]); + if (!copy[i]) { + free_strv(copy); + oci_image_config_free(&cfg); + errno = ENOMEM; + return NULL; + } + } + oci_image_config_free(&cfg); + return copy; +} + +/* Accumulate a pin's contribution. The pin's manifest digest is checked + * against target_manifest_digest (self-skip) and against compared_manifests + * (dedup with unpacked-tree walk). A pin whose blob is missing, malformed, + * or whose image-config cannot be parsed contributes nothing and is NOT + * counted in compared_images; the walker continues with the next pin. + * + * The accumulator pair is updated atomically per pin: if a mid-pin failure + * leaves partial entries in the sets, the affected pin still does not bump + * compared_images, which means compared_images reads "images that fully + * contributed". Partial sets only inflate shared_layers / shared_bytes; the + * overcount is bounded by the smaller of {pin's diff_id list length, target + * diff_id list length} and is preferable to dropping the pin's first N-1 + * diff_ids when entry N hits ENOMEM. + */ +static void walk_pin(oci_store_t *store, + const char *pin_manifest_digest, + const char *target_manifest_digest, + oci_digest_set_t *diff_acc, + oci_digest_set_t *chain_acc, + oci_digest_set_t *compared_manifests, + size_t *compared_images) +{ + if (strcmp(pin_manifest_digest, target_manifest_digest) == 0) + return; + + /* For an image-index pin, the picked linux/arm64 sub-manifest is the + * representative entry. Mark BOTH the pin's manifest digest and the + * resolved sub-manifest digest as compared so a later unpacked-tree + * lookup (which keys on whichever digest the operator unpacked from) + * does not double-count. + */ + char *config_digest = resolve_config_digest(store, pin_manifest_digest); + if (!config_digest) + return; + + char **diff_ids = load_diff_ids(store, config_digest); + free(config_digest); + if (!diff_ids) + return; + + if (accumulate_chain(diff_ids, diff_acc, chain_acc) < 0) { + free_strv(diff_ids); + return; + } + free_strv(diff_ids); + + (void) oci_digest_set_add(compared_manifests, pin_manifest_digest); + (*compared_images)++; +} + +/* Walk every unpacked image tree under volume_root/images/. The origin + * sidecar yields the manifest_digest + diff_ids directly, so no blob read + * is required. Trees whose origin sidecar matches a manifest already + * counted via the pin walk are skipped silently. + */ +static void walk_unpacked(oci_store_t *store, + const char *volume_root, + const char *target_manifest_digest, + oci_digest_set_t *diff_acc, + oci_digest_set_t *chain_acc, + oci_digest_set_t *compared_manifests, + size_t *compared_images) +{ + (void) store; + if (!volume_root) + return; + oci_volume_list_t trees = {0}; + if (oci_volume_list_unpacked(volume_root, &trees, NULL) < 0) + return; + for (size_t i = 0; i < trees.count; i++) { + oci_origin_t origin = {0}; + if (oci_origin_read(trees.items[i], &origin, NULL) < 0) + continue; + if (origin.manifest_digest && + strcmp(origin.manifest_digest, target_manifest_digest) == 0) { + oci_origin_free(&origin); + continue; + } + if (origin.manifest_digest && + oci_digest_set_contains(compared_manifests, + origin.manifest_digest)) { + oci_origin_free(&origin); + continue; + } + if (!origin.layer_diffids) { + oci_origin_free(&origin); + continue; + } + if (accumulate_chain(origin.layer_diffids, diff_acc, chain_acc) < 0) { + oci_origin_free(&origin); + continue; + } + if (origin.manifest_digest) + (void) oci_digest_set_add(compared_manifests, + origin.manifest_digest); + (*compared_images)++; + oci_origin_free(&origin); + } + oci_volume_list_free(&trees); +} + +int oci_dedup_metrics_compute(oci_store_t *store, + const char *target_manifest_digest, + const char *volume_root, + oci_dedup_metrics_t *out, + const char **err) +{ + static const char *dummy_err; + if (!err) + err = &dummy_err; + *err = NULL; + + if (!store || !target_manifest_digest || !out) { + *err = "dedup_metrics: NULL argument"; + errno = EINVAL; + return -1; + } + memset(out, 0, sizeof(*out)); + + /* Resolve the target's image-config and diff_ids. Hard failure here is + * surfaced to the caller: there is no useful "layer reuse" line when + * the target's own layer list is unknown. + */ + char *target_config = resolve_config_digest(store, target_manifest_digest); + if (!target_config) { + int saved = errno; + *err = "dedup_metrics: target manifest blob missing or unparseable"; + errno = saved; + return -1; + } + char **target_diff_ids = load_diff_ids(store, target_config); + free(target_config); + if (!target_diff_ids) { + int saved = errno; + *err = "dedup_metrics: target image-config missing or unparseable"; + errno = saved; + return -1; + } + + size_t n_target = 0; + while (target_diff_ids[n_target]) + n_target++; + out->total_layers = n_target; + + /* Precompute the target's ChainID chain so the longest-shared-prefix + * lookup is a per-layer set membership check rather than a recompute. + * chain_strs[i] holds ChainID(target_diff_ids[0..i]). + */ + char **chain_strs = NULL; + if (n_target > 0) { + chain_strs = (char **) calloc(n_target, sizeof(*chain_strs)); + if (!chain_strs) { + free_strv(target_diff_ids); + *err = "dedup_metrics: chain alloc failed"; + errno = ENOMEM; + return -1; + } + char prev[OCI_DIGEST_HEX_MAX + 16] = ""; + for (size_t i = 0; i < n_target; i++) { + char chain[OCI_DIGEST_HEX_MAX + 16]; + const char *prev_arg = (i == 0) ? NULL : prev; + if (oci_chainid_compute(prev_arg, target_diff_ids[i], chain, + sizeof(chain)) < 0) { + int saved = errno; + for (size_t j = 0; j < i; j++) + free(chain_strs[j]); + free((void *) chain_strs); + free_strv(target_diff_ids); + *err = "dedup_metrics: chainid compute failed for target"; + errno = saved; + return -1; + } + chain_strs[i] = strdup(chain); + if (!chain_strs[i]) { + for (size_t j = 0; j < i; j++) + free(chain_strs[j]); + free((void *) chain_strs); + free_strv(target_diff_ids); + *err = "dedup_metrics: chainid strdup failed"; + errno = ENOMEM; + return -1; + } + memcpy(prev, chain, strlen(chain) + 1); + } + } + + /* Walk pins and unpacked sysroots into the two accumulators. */ + oci_digest_set_t diff_acc = {0}; + oci_digest_set_t chain_acc = {0}; + oci_digest_set_t compared_manifests = {0}; + oci_digest_set_init(&diff_acc); + oci_digest_set_init(&chain_acc); + oci_digest_set_init(&compared_manifests); + + oci_pin_list_t pins = {0}; + if (oci_store_list_refs(store, &pins, NULL) == 0) { + for (size_t i = 0; i < pins.count; i++) { + walk_pin(store, pins.items[i].digest, target_manifest_digest, + &diff_acc, &chain_acc, &compared_manifests, + &out->compared_images); + } + oci_pin_list_free(&pins); + } + walk_unpacked(store, volume_root, target_manifest_digest, &diff_acc, + &chain_acc, &compared_manifests, &out->compared_images); + + /* Intersect target's diff_ids against the accumulated set. For every + * shared diff_id, walk the raw cache entry tree (if present) and add + * its on-disk size to shared_bytes. + */ + const char *store_root = oci_store_root(store); + for (size_t i = 0; i < n_target; i++) { + const char *diff_id = target_diff_ids[i]; + if (!oci_digest_set_contains(&diff_acc, diff_id)) + continue; + out->shared_layers++; + oci_digest_algo_t algo; + char hex[OCI_DIGEST_HEX_MAX + 1]; + if (!oci_digest_parse(diff_id, &algo, hex)) + continue; + const char *algo_name = oci_digest_algo_name(algo); + if (!algo_name) + continue; + char layer_path[DEDUP_PATH_MAX]; + int n = snprintf(layer_path, sizeof(layer_path), "%s/layers/%s/%s", + store_root, algo_name, hex); + if (n < 0 || (size_t) n >= sizeof(layer_path)) + continue; + out->shared_bytes += sum_tree_size(layer_path); + } + + /* Longest k such that ChainID(target[0..k-1]) is also reachable from + * some other image. Walk from the deepest layer backwards so the first + * hit terminates the search. + */ + for (size_t k = n_target; k > 0; k--) { + if (oci_digest_set_contains(&chain_acc, chain_strs[k - 1])) { + out->deepest_shared_prefix = k; + snprintf(out->deepest_shared_chainid, + sizeof(out->deepest_shared_chainid), "%s", + chain_strs[k - 1]); + break; + } + } + + oci_digest_set_free(&diff_acc); + oci_digest_set_free(&chain_acc); + oci_digest_set_free(&compared_manifests); + if (chain_strs) { + for (size_t i = 0; i < n_target; i++) + free(chain_strs[i]); + free((void *) chain_strs); + } + free_strv(target_diff_ids); + return 0; +} diff --git a/src/oci/dedup-metrics.h b/src/oci/dedup-metrics.h new file mode 100644 index 0000000..0c3993f --- /dev/null +++ b/src/oci/dedup-metrics.h @@ -0,0 +1,106 @@ +/* OCI cross-image layer dedup metrics + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Computes how much of one image's layer set is shared with every other image + * recorded in the local store. Two dedup angles are reported because the two + * Plan 3 layer caches dedup along different axes: + * + * - raw cache (layers/sha256//) keys per-layer payloads by their + * uncompressed-tar digest. Two images that share a layer with the same + * diff_id share that cache entry regardless of where the layer sits in + * either image's ordered layer list. shared_layers counts individual + * layers; shared_bytes accumulates the on-disk size of the raw entries + * covering those shared diff_ids (entries the raw cache has actually + * populated; absent entries still count as shared layers, just with zero + * bytes). + * + * - ChainID stack cache (layers/stacks/sha256//) keys cumulative + * stage_dir snapshots by the OCI ChainID of the terminating layer in an + * ordered prefix. Two images can share a stack prefix only when their + * first K layers (in order) have identical diff_ids; ChainID composition + * captures that. deepest_shared_prefix reports the longest such prefix + * length the target shares with at least one other image. This is the + * same shape the C3.3c-ii orchestrator short-circuits on during unpack. + * + * Output is informational, not a GC keep-set: missing or malformed image- + * config blobs on the OTHER images are skipped silently rather than failing + * the whole compute. Failure semantics are stricter for the TARGET image + * (missing config or missing manifest blob is surfaced as -1) so the caller + * can render a graceful-degrade notice instead of bogus numbers. + * + * Plan 4 oci status reuses this helper by calling it once per pin and + * aggregating the results client-side. + */ + +#pragma once + +#include +#include + +#include "digest.h" +#include "store.h" + +typedef struct { + /* Number of layers in target's image-config rootfs.diff_ids. */ + size_t total_layers; + /* |target.diff_ids ∩ union(other_image.diff_ids)|. A diff_id counted + * here may or may not have a raw cache entry on disk: dedup is about + * cross-image overlap, not cache populate state. + */ + size_t shared_layers; + /* Sum of /layers/sha256// tree sizes for diff_ids in the + * intersection that have an extant raw cache entry. Reports logical + * bytes (sum of file st_size) rather than physical disk usage; APFS + * clonefile means the on-disk footprint is typically much smaller. + */ + uint64_t shared_bytes; + /* Number of OTHER images that contributed at least one diff_id, deduped + * by manifest digest so a pin and its unpacked sysroot for the same + * manifest do not double-count. Excludes the target itself. + */ + size_t compared_images; + /* Longest K such that ChainID(target.diff_ids[0..K-1]) is also a + * ChainID reached by some other image's diff_id chain. 0 means no + * shared prefix. + */ + size_t deepest_shared_prefix; + /* The ChainID at depth deepest_shared_prefix, in ":" form. + * Empty string when deepest_shared_prefix == 0. Sized for sha512-prefixed + * output so the same buffer fits both digest algos plus the ":" + * separator. + */ + char deepest_shared_chainid[OCI_DIGEST_HEX_MAX + 16]; +} oci_dedup_metrics_t; + +/* Compute cross-image dedup metrics for one target image. + * + * target_manifest_digest: the manifest the caller is inspecting, in canonical + * ":" form. Must already be present under /blobs/. For + * an image-index pin, the caller is responsible for resolving the per- + * platform sub-manifest first; this helper does not pick arm64 itself. + * + * volume_root: optional path to the unpacked-sysroot volume (the directory + * holding images/sha256-/). NULL skips the unpacked-tree walk and + * the compute uses only pins recorded in index.json. Missing volume_root + * or empty images/ subtree is treated as the empty case, not an error. + * + * out: receives the computed metrics on success; left zeroed on failure. + * + * Failure model (-1 with errno preserved and *err populated): + * - target manifest blob missing or unparseable + * - target image-config blob missing or unparseable + * - target image-config has no rootfs.diff_ids (spec violation) + * - allocation failure during walk + * + * Other-image failures (malformed config, missing blob, malformed origin + * sidecar) are NOT surfaced: the offending image is skipped and the walk + * continues. compared_images reflects only images that contributed at + * least one usable diff_id list. + */ +int oci_dedup_metrics_compute(oci_store_t *store, + const char *target_manifest_digest, + const char *volume_root, + oci_dedup_metrics_t *out, + const char **err); diff --git a/src/oci/digest-set.c b/src/oci/digest-set.c new file mode 100644 index 0000000..0e6dc9f --- /dev/null +++ b/src/oci/digest-set.c @@ -0,0 +1,121 @@ +/* OCI digest set implementation + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Sorted-array set: add() performs a lower-bound scan with strcmp and + * keeps the array ordered so contains() can use bsearch. The capacity + * grows geometrically (x2) because Plan 1 collects up to one entry per + * manifest + config + layer across every pinned image plus every + * unpacked sysroot, and an O(n) realloc per add would dominate that. + */ + +#include "digest-set.h" + +#include +#include +#include +#include + +void oci_digest_set_init(oci_digest_set_t *s) +{ + if (!s) + return; + s->items = NULL; + s->count = 0; + s->cap = 0; +} + +void oci_digest_set_free(oci_digest_set_t *s) +{ + if (!s) + return; + if (s->items) { + for (size_t i = 0; i < s->count; i++) + free(s->items[i]); + free((void *) s->items); + } + s->items = NULL; + s->count = 0; + s->cap = 0; +} + +/* Locate the lower bound of digest in the sorted items[] range. Returns + * an index in [0, count]; *found is true when items[idx] equals digest. + */ +static size_t lower_bound(const oci_digest_set_t *s, + const char *digest, + bool *found) +{ + size_t lo = 0, hi = s->count; + while (lo < hi) { + size_t mid = lo + (hi - lo) / 2; + int cmp = strcmp(s->items[mid], digest); + if (cmp < 0) + lo = mid + 1; + else + hi = mid; + } + *found = lo < s->count && strcmp(s->items[lo], digest) == 0; + return lo; +} + +int oci_digest_set_add(oci_digest_set_t *s, const char *digest) +{ + if (!s || !digest) { + errno = EINVAL; + return -1; + } + bool found = false; + size_t pos = lower_bound(s, digest, &found); + if (found) + return 0; + + if (s->count == s->cap) { + size_t newcap = s->cap ? s->cap * 2 : 16; + void *raw = realloc((void *) s->items, newcap * sizeof(*s->items)); + if (!raw) { + errno = ENOMEM; + return -1; + } + s->items = (char **) raw; + s->cap = newcap; + } + + char *copy = strdup(digest); + if (!copy) { + errno = ENOMEM; + return -1; + } + /* Shift the tail right one slot so the new entry slides into its + * sorted position. memmove handles the overlap; with pos == count + * the move length is zero. + */ + if (pos < s->count) + memmove((void *) &s->items[pos + 1], (const void *) &s->items[pos], + (s->count - pos) * sizeof(*s->items)); + s->items[pos] = copy; + s->count++; + return 0; +} + +bool oci_digest_set_contains(const oci_digest_set_t *s, const char *digest) +{ + if (!s || !digest || s->count == 0) + return false; + bool found = false; + (void) lower_bound(s, digest, &found); + return found; +} + +size_t oci_digest_set_size(const oci_digest_set_t *s) +{ + return s ? s->count : 0; +} + +const char *oci_digest_set_at(const oci_digest_set_t *s, size_t i) +{ + if (!s || i >= s->count) + return NULL; + return s->items[i]; +} diff --git a/src/oci/digest-set.h b/src/oci/digest-set.h new file mode 100644 index 0000000..ba2f568 --- /dev/null +++ b/src/oci/digest-set.h @@ -0,0 +1,63 @@ +/* OCI digest set + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Append-only ordered set of ":" digest strings, used by the + * Plan 1 garbage collector to accumulate the reachable blob set across + * pin walks, image-config blobs, and unpacked-sysroot origin sidecars. + * + * Storage is a sorted, strdup-owned C array. Membership is checked via + * bsearch and insertion uses lower-bound + memmove. The expected + * working size is in the low hundreds (one entry per manifest, config, + * and layer blob across all pinned images), so O(n) per insertion is + * cheap. The C1.3 sweep walks blobs// and calls contains() + * once per blob, which bsearch makes O(log n). If profiling later + * proves the set hot enough to matter, swap to a hash table without + * touching the public API. + * + * The set treats digest strings as opaque bytes: the caller is + * expected to feed only validated ":" forms produced by + * oci_digest_parse so a hand-edited index.json cannot smuggle in an + * uppercase variant that defeats the dedup. + */ + +#pragma once + +#include +#include + +typedef struct { + char **items; + size_t count; + size_t cap; +} oci_digest_set_t; + +/* Zero-initialise a set in place. Safe to call on a struct that was + * declared with {0}; provided for readability at allocation sites. + */ +void oci_digest_set_init(oci_digest_set_t *s); + +/* Release every owned digest string and zero the struct. Safe on a + * zero-initialised set and on NULL. + */ +void oci_digest_set_free(oci_digest_set_t *s); + +/* Insert digest into the set. Returns 0 on success or when the digest + * is already present (idempotent), -1 with errno set on failure. A + * NULL digest is rejected with EINVAL so a caller bug does not leak + * into the sweep phase later. + */ +int oci_digest_set_add(oci_digest_set_t *s, const char *digest); + +/* True when digest is in the set. NULL inputs return false. */ +bool oci_digest_set_contains(const oci_digest_set_t *s, const char *digest); + +/* Current cardinality of the set. */ +size_t oci_digest_set_size(const oci_digest_set_t *s); + +/* Borrow the digest string at index i (lexicographically ordered). The + * returned pointer is valid until the next mutating call or free. + * Returns NULL when i is out of range. + */ +const char *oci_digest_set_at(const oci_digest_set_t *s, size_t i); diff --git a/src/oci/digest.c b/src/oci/digest.c new file mode 100644 index 0000000..9b9a060 --- /dev/null +++ b/src/oci/digest.c @@ -0,0 +1,281 @@ +/* Content digests for OCI image blobs + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "digest.h" + +#include +#include +#include +#include +#include + +/* CC_LONG is 32-bit; clamp every update call so multi-gigabyte layers cannot + * overflow the CommonCrypto length argument silently. 1 GiB is well below the + * limit and large enough that the per-call overhead is negligible. + */ +#define DIGESTER_CHUNK_MAX ((size_t) (1u << 30)) + +struct oci_digester { + oci_digest_algo_t algo; + union { + CC_SHA256_CTX sha256; + CC_SHA512_CTX sha512; + } ctx; +}; + +static const char HEX_LOWER[] = "0123456789abcdef"; + +static void bin_to_hex_lower(const uint8_t *bin, size_t bin_len, char *out) +{ + for (size_t i = 0; i < bin_len; i++) { + out[i * 2] = HEX_LOWER[(bin[i] >> 4) & 0xf]; + out[i * 2 + 1] = HEX_LOWER[bin[i] & 0xf]; + } + out[bin_len * 2] = '\0'; +} + +const char *oci_digest_algo_name(oci_digest_algo_t algo) +{ + switch (algo) { + case OCI_DIGEST_SHA256: + return "sha256"; + case OCI_DIGEST_SHA512: + return "sha512"; + } + return NULL; +} + +size_t oci_digest_hex_len(oci_digest_algo_t algo) +{ + switch (algo) { + case OCI_DIGEST_SHA256: + return OCI_DIGEST_SHA256_HEX_LEN; + case OCI_DIGEST_SHA512: + return OCI_DIGEST_SHA512_HEX_LEN; + } + return 0; +} + +bool oci_digest_algo_from_name(const char *name, oci_digest_algo_t *algo) +{ + if (!name || !algo) + return false; + if (!strcmp(name, "sha256")) { + *algo = OCI_DIGEST_SHA256; + return true; + } + if (!strcmp(name, "sha512")) { + *algo = OCI_DIGEST_SHA512; + return true; + } + return false; +} + +bool oci_digest_hex_valid(oci_digest_algo_t algo, const char *hex) +{ + if (!hex) + return false; + size_t want = oci_digest_hex_len(algo); + if (want == 0) + return false; + if (strlen(hex) != want) + return false; + for (size_t i = 0; i < want; i++) { + char c = hex[i]; + bool ok = (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'); + if (!ok) + return false; + } + return true; +} + +bool oci_digest_parse(const char *colon_form, + oci_digest_algo_t *out_algo, + char *out_hex) +{ + if (!colon_form || !out_algo || !out_hex) + return false; + + out_hex[0] = '\0'; + const char *colon = strchr(colon_form, ':'); + if (!colon || colon == colon_form) + return false; + + char name[8]; + size_t name_len = (size_t) (colon - colon_form); + if (name_len >= sizeof(name)) + return false; + memcpy(name, colon_form, name_len); + name[name_len] = '\0'; + + oci_digest_algo_t algo; + if (!oci_digest_algo_from_name(name, &algo)) + return false; + + const char *hex = colon + 1; + if (!oci_digest_hex_valid(algo, hex)) + return false; + + *out_algo = algo; + memcpy(out_hex, hex, oci_digest_hex_len(algo) + 1); + return true; +} + +oci_digester_t *oci_digester_new(oci_digest_algo_t algo) +{ + oci_digester_t *d = calloc(1, sizeof(*d)); + if (!d) + return NULL; + d->algo = algo; + switch (algo) { + case OCI_DIGEST_SHA256: + (void) CC_SHA256_Init(&d->ctx.sha256); + break; + case OCI_DIGEST_SHA512: + (void) CC_SHA512_Init(&d->ctx.sha512); + break; + default: + free(d); + return NULL; + } + return d; +} + +void oci_digester_free(oci_digester_t *d) +{ + free(d); +} + +void oci_digester_update(oci_digester_t *d, const void *buf, size_t len) +{ + if (!d || !buf || len == 0) + return; + const uint8_t *p = buf; + while (len > 0) { + size_t chunk = len > DIGESTER_CHUNK_MAX ? DIGESTER_CHUNK_MAX : len; + switch (d->algo) { + case OCI_DIGEST_SHA256: + (void) CC_SHA256_Update(&d->ctx.sha256, p, (CC_LONG) chunk); + break; + case OCI_DIGEST_SHA512: + (void) CC_SHA512_Update(&d->ctx.sha512, p, (CC_LONG) chunk); + break; + } + p += chunk; + len -= chunk; + } +} + +size_t oci_digester_finish_hex(oci_digester_t *d, char *out_hex) +{ + if (!d || !out_hex) + return 0; + uint8_t md[CC_SHA512_DIGEST_LENGTH]; + size_t bin_len = 0; + switch (d->algo) { + case OCI_DIGEST_SHA256: + (void) CC_SHA256_Final(md, &d->ctx.sha256); + bin_len = CC_SHA256_DIGEST_LENGTH; + break; + case OCI_DIGEST_SHA512: + (void) CC_SHA512_Final(md, &d->ctx.sha512); + bin_len = CC_SHA512_DIGEST_LENGTH; + break; + default: + return 0; + } + bin_to_hex_lower(md, bin_len, out_hex); + return bin_len * 2; +} + +size_t oci_digest_bytes(oci_digest_algo_t algo, + const void *buf, + size_t len, + char *out_hex) +{ + if (!out_hex) + return 0; + oci_digester_t *d = oci_digester_new(algo); + if (!d) + return 0; + oci_digester_update(d, buf, len); + size_t n = oci_digester_finish_hex(d, out_hex); + oci_digester_free(d); + return n; +} + +int oci_chainid_compute(const char *prev_chain, + const char *diff_id, + char *out, + size_t cap) +{ + if (!diff_id || !out || cap == 0) { + errno = EINVAL; + return -1; + } + + /* Validate the diff_id is well-formed before any hashing so the L0 + * passthrough path and the Li hash path agree on input validation. + */ + oci_digest_algo_t diff_algo; + char diff_hex[OCI_DIGEST_HEX_MAX + 1]; + if (!oci_digest_parse(diff_id, &diff_algo, diff_hex)) { + errno = EINVAL; + return -1; + } + + if (!prev_chain) { + /* L0 case: ChainID(L0) == DiffID(L0). Copy verbatim so a sha512 + * diff_id round-trips unchanged through the layer-0 slot. + */ + size_t diff_len = strlen(diff_id); + if (diff_len + 1 > cap) { + errno = ENAMETOOLONG; + return -1; + } + memcpy(out, diff_id, diff_len + 1); + return 0; + } + + /* Li case: validate prev_chain shape too. The spec allows any + * : digest string in the textual concatenation; in practice + * every ChainID this helper produces is sha256-prefixed, but the + * parser accepts both sha256 and sha512 so a future L0 sha512 diff_id + * still composes correctly with subsequent layers. + */ + oci_digest_algo_t prev_algo; + char prev_hex[OCI_DIGEST_HEX_MAX + 1]; + if (!oci_digest_parse(prev_chain, &prev_algo, prev_hex)) { + errno = EINVAL; + return -1; + } + + /* Output is always sha256-prefixed: ChainID composition is defined + * with SHA-256 in the OCI image-spec, regardless of the algorithm + * used for the constituent digests. + */ + static const char OUT_PREFIX[] = "sha256:"; + size_t out_need = sizeof(OUT_PREFIX) - 1 + OCI_DIGEST_SHA256_HEX_LEN + 1; + if (cap < out_need) { + errno = ENAMETOOLONG; + return -1; + } + + oci_digester_t *d = oci_digester_new(OCI_DIGEST_SHA256); + if (!d) { + errno = ENOMEM; + return -1; + } + oci_digester_update(d, prev_chain, strlen(prev_chain)); + static const char SP = ' '; + oci_digester_update(d, &SP, 1); + oci_digester_update(d, diff_id, strlen(diff_id)); + + memcpy(out, OUT_PREFIX, sizeof(OUT_PREFIX) - 1); + oci_digester_finish_hex(d, out + sizeof(OUT_PREFIX) - 1); + oci_digester_free(d); + return 0; +} diff --git a/src/oci/digest.h b/src/oci/digest.h new file mode 100644 index 0000000..06b96c5 --- /dev/null +++ b/src/oci/digest.h @@ -0,0 +1,128 @@ +/* Content digests for OCI image blobs + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Wraps macOS CommonCrypto SHA-256 and SHA-512 in a streaming API so the + * blob store and registry client can hash gigabyte-class layer downloads + * without ever buffering the full payload in memory. + * + * Hex output is always lowercase; the OCI image reference parser already + * rejects uppercase digest hex (see src/oci/ref.c), so every digest hex that + * flows between the parser, the manifest fetcher, and the local store must + * stay in the same canonical encoding to avoid silent dedup misses. + */ + +#pragma once + +#include +#include + +typedef enum { + OCI_DIGEST_SHA256, + OCI_DIGEST_SHA512, +} oci_digest_algo_t; + +/* Hex length per algorithm, excluding the trailing NUL. */ +#define OCI_DIGEST_SHA256_HEX_LEN 64 +#define OCI_DIGEST_SHA512_HEX_LEN 128 +#define OCI_DIGEST_HEX_MAX OCI_DIGEST_SHA512_HEX_LEN + +/* Opaque streaming digest. Allocated on the heap because the underlying + * CommonCrypto context is moderately sized (SHA-512 keeps an 80-word state) + * and callers tend to thread a digester pointer through several modules. + */ +typedef struct oci_digester oci_digester_t; + +/* Allocate a streaming digester for algo. Returns NULL on bad enum or oom. */ +oci_digester_t *oci_digester_new(oci_digest_algo_t algo); + +/* Release a digester. Safe on NULL. */ +void oci_digester_free(oci_digester_t *d); + +/* Append data. Splits large buffers into CC_LONG-sized chunks internally + * because CommonCrypto's update takes a uint32_t length and OCI layers can + * exceed 4 GiB. + */ +void oci_digester_update(oci_digester_t *d, const void *buf, size_t len); + +/* Finalize and write the lowercase hex string to out_hex. out_hex must hold + * at least OCI_DIGEST_HEX_MAX + 1 bytes. Returns the hex length on success + * (without trailing NUL) or 0 if d is NULL. The digester is consumed by this + * call: the only valid next operation is oci_digester_free. + */ +size_t oci_digester_finish_hex(oci_digester_t *d, char *out_hex); + +/* Lookup the algorithm name string ("sha256" / "sha512"). Returns NULL when + * algo is out of range. The returned pointer is to static storage. + */ +const char *oci_digest_algo_name(oci_digest_algo_t algo); + +/* Expected hex length for an algorithm (without trailing NUL). Returns 0 on + * bad enum. + */ +size_t oci_digest_hex_len(oci_digest_algo_t algo); + +/* Parse an algorithm name. Returns true and writes algo on match; false on + * unknown name. + */ +bool oci_digest_algo_from_name(const char *name, oci_digest_algo_t *algo); + +/* Validate that hex is exactly oci_digest_hex_len(algo) characters and that + * every character is a lowercase hex digit. Rejects NULL. + */ +bool oci_digest_hex_valid(oci_digest_algo_t algo, const char *hex); + +/* Parse ":" into algo and a canonical lowercase hex copy. The + * input hex must already be lowercase; mixed case is rejected to match the + * reference parser. out_hex must hold OCI_DIGEST_HEX_MAX + 1 bytes. On + * success returns true; otherwise returns false and out_hex is left zeroed. + */ +bool oci_digest_parse(const char *colon_form, + oci_digest_algo_t *out_algo, + char *out_hex); + +/* One-shot helper: compute algo over buf/len and emit lowercase hex into + * out_hex (which must hold OCI_DIGEST_HEX_MAX + 1 bytes). Returns the hex + * length on success or 0 on bad enum / NULL output. + */ +size_t oci_digest_bytes(oci_digest_algo_t algo, + const void *buf, + size_t len, + char *out_hex); + +/* Compute the OCI image-spec ChainID for one layer in canonical + * ":" form. ChainID is the cumulative content key used by the + * Plan 3 C3.3 stack cache: ChainID(L0) == DiffID(L0), and for any later + * layer ChainID(Li) == sha256(" ") where the input is + * the previous chain string, an ASCII space (0x20), and the current layer's + * diff_id string, both in their canonical ":" form. See OCI + * image-spec v1.0.2 section 3.4 "Layer ChainID" for the reference text. + * + * The output is always sha256-prefixed regardless of diff_id's algorithm: + * ChainID composition is defined over the textual digest representation, so + * a sha512 diff_id contributes its full "sha512:" string but the result + * is hashed with SHA-256 (the only ChainID algorithm the spec defines). + * + * Parameters: + * prev_chain NULL signals the L0 case; the helper copies diff_id into + * out verbatim and returns 0. Non-NULL must be a valid + * ":" string in canonical lowercase form. + * diff_id This layer's diff_id; must be non-NULL and ":". + * out Receives the new ChainID string (NUL-terminated, always + * "sha256:<64-hex>" when prev_chain != NULL, or a copy of + * diff_id when prev_chain == NULL). + * cap Capacity of out in bytes. Must be at least + * OCI_DIGEST_HEX_MAX + 16 so a sha512 diff_id fits in the L0 + * passthrough path. + * + * Returns 0 on success, -1 with errno set on failure: + * EINVAL diff_id NULL, prev_chain non-NULL but malformed, or + * diff_id malformed + * ENAMETOOLONG cap too small to hold the result + * ENOMEM internal SHA-256 digester allocation failed + */ +int oci_chainid_compute(const char *prev_chain, + const char *diff_id, + char *out, + size_t cap); diff --git a/src/oci/fetch.c b/src/oci/fetch.c new file mode 100644 index 0000000..1ab60b8 --- /dev/null +++ b/src/oci/fetch.c @@ -0,0 +1,1671 @@ +/* OCI registry HTTPS client + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Implements anonymous and bearer-challenge HTTPS pulls against the OCI + * distribution-spec /v2/ endpoints. Manifest fetches return body bytes plus a + * captured Content-Type and Docker-Content-Digest so the slice-3 parser and + * future tag-to-digest pinning can consume them directly. Blob fetches stream + * the response body into the slice-2 blob store, capping the running byte + * count at the descriptor's declared size and letting the writer's digest + * check reject any payload that hashes to anything other than the descriptor + * hex. + * + * The 401 retry path is "try anonymous first, then parse Www-Authenticate, + * fetch a token, retry once". A second 401 propagates as a fetch failure; the + * caller decides whether to surface authorization-failed or treat it as a + * transient network error. The cached bearer token is invalidated by any 401 + * but otherwise reused across requests on the same fetcher, so a pull of an + * image with N layers makes one token call rather than N+1. + */ + +#include "fetch.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../externals/cjson/cJSON.h" +#include "policy.h" + +/* Hard ceiling on a single manifest / index / config response. Real-world + * documents are well under 1 MiB; the limit is here so a misbehaving registry + * cannot fill memory with an unbounded body. Blob responses do not flow + * through this buffer; they stream into the blob store. + */ +#define FETCH_BODY_MAX ((size_t) 16 * 1024 * 1024) + +typedef struct { + char *realm; + char *service; + char *scope; +} bearer_challenge_t; + +struct oci_fetcher { + CURL *easy; + char *base_url_override; + char *bearer_token; + bearer_challenge_t challenge; + /* Pre-built "user:pass" string for CURLOPT_USERPWD. NULL when CLI basic + * auth is disabled. The fetcher attaches it to every easy-handle reset + * (manifest GET, blob GET, token GET) so a registry that bridges basic + * and bearer sees the basic credentials on both the manifest probe and + * the token exchange. + */ + char *user_pass; + /* PEM bundle path passed through to CURLOPT_CAINFO. NULL leaves libcurl on + * its compiled-in trust store. + */ + char *ca_file; + bool allow_insecure; + /* Caller-owned policy. NULL when the caller has not loaded a policy.json. + * Consulted by resolve_effective on every manifest/blob entry; the + * fetcher does not take a copy and does not free it. + */ + const oci_policy_t *policy; +}; + +/* Per-request merge of CLI-supplied options and the policy lookup for the + * current ref->registry. resolve_effective produces one of these and the + * request paths read it instead of f->{user_pass,ca_file,allow_insecure}. + * Strings are borrowed (point into f->* or into the policy_t entry) except + * user_pass_loaded, which holds a heap "user:pass" built from a policy + * auth_file. effective_free releases that one allocation. + */ +typedef struct { + const char *user_pass; + const char *ca_file; + bool allow_insecure; + char *user_pass_loaded; +} effective_opts_t; + +static void effective_free(effective_opts_t *eff) +{ + if (!eff) + return; + free(eff->user_pass_loaded); + eff->user_pass_loaded = NULL; + eff->user_pass = NULL; + eff->ca_file = NULL; + eff->allow_insecure = false; +} + +/* Build the per-request effective options from the fetcher's CLI defaults and + * any policy entry matching ref->registry. CLI flags win: a CLI-supplied + * user_pass / ca_file / allow_insecure shadows the policy value for the same + * field. A policy auth_file is loaded via oci_policy_load_auth, which + * enforces 0600 mode and the {username,password} JSON shape. Returns 0 on + * success, -1 with errno + *err_msg on auth_file load failure. The caller + * always invokes effective_free, including on rc != 0. + */ +static int resolve_effective(const oci_fetcher_t *f, + const oci_ref_t *ref, + effective_opts_t *eff, + const char **err_msg) +{ + memset(eff, 0, sizeof(*eff)); + eff->user_pass = f->user_pass; + eff->ca_file = f->ca_file; + eff->allow_insecure = f->allow_insecure; + + if (!f->policy || !ref || !ref->registry) + return 0; + + oci_policy_effective_t pol; + oci_policy_lookup(f->policy, ref->registry, &pol); + + if (!eff->allow_insecure && pol.insecure) + eff->allow_insecure = true; + if (!eff->ca_file && pol.ca_bundle) + eff->ca_file = pol.ca_bundle; + + /* Only consult policy auth_file when the caller did not supply CLI + * credentials. The load happens per-request; auth files are small and + * mode-checked each time, which avoids any cache-vs-disk consistency + * worry at the cost of re-parsing a sub-kilobyte JSON document. + */ + if (!eff->user_pass && pol.auth_file) { + char *user = NULL; + char *pass = NULL; + const char *aerr = NULL; + if (oci_policy_load_auth(pol.auth_file, &user, &pass, &aerr) < 0) { + int e = errno; + free(user); + free(pass); + if (err_msg) + *err_msg = aerr ? aerr : "policy auth file load failed"; + errno = e ? e : EINVAL; + return -1; + } + size_t ul = strlen(user); + size_t pl = strlen(pass); + char *up = malloc(ul + 1 + pl + 1); + if (!up) { + free(user); + free(pass); + if (err_msg) + *err_msg = "out of memory composing policy credentials"; + errno = ENOMEM; + return -1; + } + memcpy(up, user, ul); + up[ul] = ':'; + memcpy(up + ul + 1, pass, pl); + up[ul + 1 + pl] = '\0'; + free(user); + free(pass); + eff->user_pass_loaded = up; + eff->user_pass = up; + } + return 0; +} + +static pthread_once_t g_curl_init_once = PTHREAD_ONCE_INIT; +static int g_curl_init_rc = -1; + +static void curl_global_once(void) +{ + g_curl_init_rc = curl_global_init(CURL_GLOBAL_DEFAULT) == CURLE_OK ? 0 : -1; +} + +int oci_fetch_global_init(void) +{ + pthread_once(&g_curl_init_once, curl_global_once); + if (g_curl_init_rc < 0) + errno = EIO; + return g_curl_init_rc; +} + +void oci_fetch_global_cleanup(void) +{ + /* curl_global_cleanup is not safe under threading. elfuse process lives + * for the duration of one pull so leaving libcurl initialized is fine. + */ +} + +static void bearer_challenge_free(bearer_challenge_t *c) +{ + if (!c) + return; + free(c->realm); + free(c->service); + free(c->scope); + c->realm = NULL; + c->service = NULL; + c->scope = NULL; +} + +static char *build_user_pass(const char *user, const char *pass) +{ + if (!user) + return NULL; + size_t ul = strlen(user); + size_t pl = pass ? strlen(pass) : 0; + char *out = malloc(ul + 1 + pl + 1); + if (!out) + return NULL; + memcpy(out, user, ul); + out[ul] = ':'; + if (pl) + memcpy(out + ul + 1, pass, pl); + out[ul + 1 + pl] = '\0'; + return out; +} + +oci_fetcher_t *oci_fetcher_new(const oci_fetcher_options_t *opts) +{ + if (oci_fetch_global_init() < 0) + return NULL; + oci_fetcher_t *f = calloc(1, sizeof(*f)); + if (!f) { + errno = ENOMEM; + return NULL; + } + f->easy = curl_easy_init(); + if (!f->easy) { + free(f); + errno = EIO; + return NULL; + } + if (opts && opts->base_url_override) { + f->base_url_override = strdup(opts->base_url_override); + if (!f->base_url_override) { + curl_easy_cleanup(f->easy); + free(f); + errno = ENOMEM; + return NULL; + } + } + if (opts && opts->username) { + f->user_pass = build_user_pass(opts->username, opts->password); + if (!f->user_pass) { + curl_easy_cleanup(f->easy); + free(f->base_url_override); + free(f); + errno = ENOMEM; + return NULL; + } + } + if (opts && opts->ca_file) { + f->ca_file = strdup(opts->ca_file); + if (!f->ca_file) { + curl_easy_cleanup(f->easy); + free(f->base_url_override); + free(f->user_pass); + free(f); + errno = ENOMEM; + return NULL; + } + } + if (opts) + f->allow_insecure = opts->allow_insecure; + if (opts) + f->policy = opts->policy; + return f; +} + +void oci_fetcher_free(oci_fetcher_t *f) +{ + if (!f) + return; + if (f->easy) + curl_easy_cleanup(f->easy); + free(f->base_url_override); + free(f->bearer_token); + bearer_challenge_free(&f->challenge); + free(f->user_pass); + free(f->ca_file); + free(f); +} + +void oci_fetch_response_free(oci_fetch_response_t *r) +{ + if (!r) + return; + free(r->body); + free(r->content_type); + free(r->docker_content_digest); + free(r->etag); + r->body = NULL; + r->content_type = NULL; + r->docker_content_digest = NULL; + r->etag = NULL; + r->body_len = 0; + r->http_status = 0; +} + +/* Strip the [bracketed] form of an IPv6 literal and any trailing :port from a + * registry-shaped string ("127.0.0.1:fake", "ghcr.io", "[::1]:5000", + * "registry.example.com"). Writes the bare host into out and returns true on + * success; returns false when out is too small to fit the result. + * + * Bracketed IPv6 forms have a colon inside the address, so port-stripping + * keys off the closing ']'; for non-bracketed registries the rightmost ':' + * is the port delimiter. + */ +static bool extract_host_from_registry(const char *reg, char *out, size_t cap) +{ + if (!reg || !out || cap == 0) + return false; + if (reg[0] == '[') { + const char *close = strchr(reg, ']'); + if (!close) + return false; + size_t n = (size_t) (close - reg - 1); + if (n + 1 > cap) + return false; + memcpy(out, reg + 1, n); + out[n] = '\0'; + return true; + } + const char *colon = strrchr(reg, ':'); + size_t n = colon ? (size_t) (colon - reg) : strlen(reg); + if (n + 1 > cap) + return false; + memcpy(out, reg, n); + out[n] = '\0'; + return true; +} + +static bool is_loopback_host(const char *host) +{ + if (!host) + return false; + if (!strcasecmp(host, "127.0.0.1")) + return true; + if (!strcasecmp(host, "localhost")) + return true; + if (!strcasecmp(host, "::1")) + return true; + return false; +} + +/* Reject allow_insecure when the registry host is not on the loopback + * whitelist. Honors ref->registry as the authoritative target even when a + * test passes base_url_override, so that policy reflects the production + * surface ("which host am I pulling from?") rather than where the bytes + * happen to flow during a unit test. The decision is made on the effective + * opts (CLI || policy), so a policy insecure=true on a non-loopback host + * fails the same way a CLI --insecure on a non-loopback host fails. + */ +static int check_insecure_policy(const effective_opts_t *eff, + const oci_ref_t *ref, + const char **err_msg) +{ + if (!eff->allow_insecure) + return 0; + char host[256]; + if (!extract_host_from_registry(ref->registry, host, sizeof(host))) { + if (err_msg) + *err_msg = "registry host is malformed"; + errno = EINVAL; + return -1; + } + if (!is_loopback_host(host)) { + if (err_msg) + *err_msg = "allow_insecure is restricted to loopback registries"; + errno = EPERM; + return -1; + } + return 0; +} + +/* Apply the per-request effective security options to the easy handle in its + * post-reset state. Called from every GET path (manifest, blob, token) after + * curl_easy_reset so the option set survives the reset. + */ +static void apply_security_opts(CURL *easy, const effective_opts_t *eff) +{ + if (eff->user_pass) { + curl_easy_setopt(easy, CURLOPT_USERPWD, eff->user_pass); + curl_easy_setopt(easy, CURLOPT_HTTPAUTH, (long) CURLAUTH_BASIC); + } + if (eff->ca_file) + curl_easy_setopt(easy, CURLOPT_CAINFO, eff->ca_file); + if (eff->allow_insecure) { + curl_easy_setopt(easy, CURLOPT_SSL_VERIFYPEER, 0L); + curl_easy_setopt(easy, CURLOPT_SSL_VERIFYHOST, 0L); + } +} + +/* docker.io is the canonical registry name from the reference parser; the + * actual API host is registry-1.docker.io. Every other registry (ghcr.io, + * quay.io, public.ecr.aws, mirrors) uses its own host directly. + */ +static const char *api_host_for_registry(const char *reg) +{ + if (reg && !strcmp(reg, "docker.io")) + return "registry-1.docker.io"; + return reg; +} + +static char *build_base_url(const oci_fetcher_t *f, const oci_ref_t *ref) +{ + if (f->base_url_override) + return strdup(f->base_url_override); + const char *host = api_host_for_registry(ref->registry); + if (!host) + return NULL; + size_t n = strlen(host) + sizeof("https://"); + char *url = malloc(n); + if (!url) + return NULL; + snprintf(url, n, "https://%s", host); + return url; +} + +static char *build_manifest_url(const oci_fetcher_t *f, + const oci_ref_t *ref, + const char *selector) +{ + char *base = build_base_url(f, ref); + if (!base) + return NULL; + size_t n = strlen(base) + strlen(ref->repository) + strlen(selector) + + sizeof("/v2//manifests/"); + char *url = malloc(n); + if (!url) { + free(base); + return NULL; + } + snprintf(url, n, "%s/v2/%s/manifests/%s", base, ref->repository, selector); + free(base); + return url; +} + +static char *build_blob_url(const oci_fetcher_t *f, + const oci_ref_t *ref, + const char *digest_str) +{ + char *base = build_base_url(f, ref); + if (!base) + return NULL; + size_t n = strlen(base) + strlen(ref->repository) + strlen(digest_str) + + sizeof("/v2//blobs/"); + char *url = malloc(n); + if (!url) { + free(base); + return NULL; + } + snprintf(url, n, "%s/v2/%s/blobs/%s", base, ref->repository, digest_str); + free(base); + return url; +} + +typedef struct { + char *buf; + size_t len; + size_t cap; + size_t max; + bool overflow; +} body_buf_t; + +static size_t body_write_cb(char *ptr, + size_t size, + size_t nmemb, + void *userdata) +{ + body_buf_t *b = userdata; + size_t n = size * nmemb; + if (b->overflow) + return 0; + if (b->len + n + 1 > b->max) { + b->overflow = true; + return 0; + } + if (b->len + n + 1 > b->cap) { + size_t newcap = b->cap ? b->cap : 4096; + while (newcap < b->len + n + 1) + newcap *= 2; + if (newcap > b->max + 1) + newcap = b->max + 1; + char *r = realloc(b->buf, newcap); + if (!r) { + b->overflow = true; + return 0; + } + b->buf = r; + b->cap = newcap; + } + memcpy(b->buf + b->len, ptr, n); + b->len += n; + b->buf[b->len] = '\0'; + return n; +} + +static char *trim_inplace(char *s) +{ + if (!s) + return NULL; + while (*s && isspace((unsigned char) *s)) + s++; + size_t n = strlen(s); + while (n > 0 && isspace((unsigned char) s[n - 1])) { + s[n - 1] = '\0'; + n--; + } + return s; +} + +static char *match_header(char *line, const char *key) +{ + size_t klen = strlen(key); + if (strncasecmp(line, key, klen) != 0) + return NULL; + if (line[klen] != ':') + return NULL; + char *v = line + klen + 1; + while (*v == ' ' || *v == '\t') + v++; + return v; +} + +static char *strdup_range(const char *s, const char *end) +{ + size_t n = (size_t) (end - s); + char *r = malloc(n + 1); + if (!r) + return NULL; + memcpy(r, s, n); + r[n] = '\0'; + return r; +} + +/* Parse a Bearer challenge value into realm/service/scope. Accepts unquoted + * values too (some test fixtures and a few private registries skip the + * quotes). Returns 0 on success or -1 on malformed input. On success *out is + * fully owned by the caller; any prior contents are freed. + */ +static int parse_bearer_challenge(const char *value, bearer_challenge_t *out) +{ + bearer_challenge_t tmp = {0}; + const char *p = value; + while (*p == ' ' || *p == '\t') + p++; + if (strncasecmp(p, "Bearer", 6) != 0) + return -1; + p += 6; + while (*p == ' ' || *p == '\t') + p++; + while (*p) { + const char *key_start = p; + while (*p && *p != '=' && *p != ',') + p++; + if (*p != '=') { + bearer_challenge_free(&tmp); + return -1; + } + const char *key_end = p; + p++; + char *value_str; + if (*p == '"') { + p++; + const char *vstart = p; + while (*p && *p != '"') + p++; + if (*p != '"') { + bearer_challenge_free(&tmp); + return -1; + } + value_str = strdup_range(vstart, p); + p++; + } else { + const char *vstart = p; + while (*p && *p != ',') + p++; + value_str = strdup_range(vstart, p); + } + if (!value_str) { + bearer_challenge_free(&tmp); + return -1; + } + size_t klen = (size_t) (key_end - key_start); + char **target = NULL; + if (klen == 5 && !strncasecmp(key_start, "realm", 5)) + target = &tmp.realm; + else if (klen == 7 && !strncasecmp(key_start, "service", 7)) + target = &tmp.service; + else if (klen == 5 && !strncasecmp(key_start, "scope", 5)) + target = &tmp.scope; + if (target) { + free(*target); + *target = value_str; + } else { + free(value_str); + } + while (*p == ',' || *p == ' ' || *p == '\t') + p++; + } + if (!tmp.realm) { + bearer_challenge_free(&tmp); + return -1; + } + bearer_challenge_free(out); + *out = tmp; + return 0; +} + +typedef struct { + char *content_type; + char *docker_content_digest; + char *etag; + bearer_challenge_t *challenge_out; +} headers_ctx_t; + +static size_t header_cb(char *buffer, + size_t size, + size_t nitems, + void *userdata) +{ + headers_ctx_t *ctx = userdata; + size_t n = size * nitems; + size_t total = n; + if (n == 0 || n >= 4096) + return total; + char line[4096]; + memcpy(line, buffer, n); + line[n] = '\0'; + while (n > 0 && (line[n - 1] == '\r' || line[n - 1] == '\n')) + line[--n] = '\0'; + if (n == 0) + return total; + + char *v = match_header(line, "Content-Type"); + if (v) { + v = trim_inplace(v); + char *semi = strchr(v, ';'); + if (semi) + *semi = '\0'; + v = trim_inplace(v); + free(ctx->content_type); + ctx->content_type = strdup(v); + return total; + } + v = match_header(line, "Docker-Content-Digest"); + if (v) { + v = trim_inplace(v); + free(ctx->docker_content_digest); + ctx->docker_content_digest = strdup(v); + return total; + } + v = match_header(line, "ETag"); + if (v) { + v = trim_inplace(v); + free(ctx->etag); + ctx->etag = strdup(v); + return total; + } + if (ctx->challenge_out) { + v = match_header(line, "Www-Authenticate"); + if (v) { + v = trim_inplace(v); + (void) parse_bearer_challenge(v, ctx->challenge_out); + } + } + return total; +} + +static struct curl_slist *build_request_headers(const oci_fetcher_t *f, + const char *const *accept_types, + const char *if_none_match) +{ + struct curl_slist *hdrs = NULL; + if (accept_types) { + for (const char *const *p = accept_types; *p; p++) { + char hdr[256]; + snprintf(hdr, sizeof(hdr), "Accept: %s", *p); + hdrs = curl_slist_append(hdrs, hdr); + } + } + if (f->bearer_token) { + size_t n = strlen(f->bearer_token) + sizeof("Authorization: Bearer "); + char *hdr = malloc(n); + if (hdr) { + snprintf(hdr, n, "Authorization: Bearer %s", f->bearer_token); + hdrs = curl_slist_append(hdrs, hdr); + free(hdr); + } + } + if (if_none_match) { + size_t n = strlen(if_none_match) + sizeof("If-None-Match: "); + char *hdr = malloc(n); + if (hdr) { + snprintf(hdr, n, "If-None-Match: %s", if_none_match); + hdrs = curl_slist_append(hdrs, hdr); + free(hdr); + } + } + return hdrs; +} + +static int fetch_token(oci_fetcher_t *f, + const effective_opts_t *eff, + const char **err_msg) +{ + if (!f->challenge.realm) { + if (err_msg) + *err_msg = "no bearer realm to fetch token from"; + errno = EINVAL; + return -1; + } + + char *enc_service = f->challenge.service + ? curl_easy_escape(f->easy, f->challenge.service, 0) + : NULL; + char *enc_scope = f->challenge.scope + ? curl_easy_escape(f->easy, f->challenge.scope, 0) + : NULL; + size_t n = strlen(f->challenge.realm) + + (enc_service ? strlen(enc_service) + 16 : 0) + + (enc_scope ? strlen(enc_scope) + 16 : 0) + 2; + char *url = malloc(n); + if (!url) { + curl_free(enc_service); + curl_free(enc_scope); + if (err_msg) + *err_msg = "out of memory"; + errno = ENOMEM; + return -1; + } + int len = snprintf(url, n, "%s", f->challenge.realm); + char sep = strchr(f->challenge.realm, '?') ? '&' : '?'; + if (enc_service) { + len += snprintf(url + len, n - (size_t) len, "%cservice=%s", sep, + enc_service); + sep = '&'; + } + if (enc_scope) { + snprintf(url + len, n - (size_t) len, "%cscope=%s", sep, enc_scope); + } + curl_free(enc_service); + curl_free(enc_scope); + + body_buf_t body = {.max = FETCH_BODY_MAX}; + headers_ctx_t hctx = {0}; + curl_easy_reset(f->easy); + apply_security_opts(f->easy, eff); + curl_easy_setopt(f->easy, CURLOPT_URL, url); + curl_easy_setopt(f->easy, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(f->easy, CURLOPT_MAXREDIRS, 5L); + curl_easy_setopt(f->easy, CURLOPT_USERAGENT, "elfuse-oci/1"); + curl_easy_setopt(f->easy, CURLOPT_WRITEFUNCTION, body_write_cb); + curl_easy_setopt(f->easy, CURLOPT_WRITEDATA, &body); + curl_easy_setopt(f->easy, CURLOPT_HEADERFUNCTION, header_cb); + curl_easy_setopt(f->easy, CURLOPT_HEADERDATA, &hctx); + + CURLcode rc = curl_easy_perform(f->easy); + long status = 0; + curl_easy_getinfo(f->easy, CURLINFO_RESPONSE_CODE, &status); + free(url); + free(hctx.content_type); + free(hctx.docker_content_digest); + + if (rc != CURLE_OK) { + free(body.buf); + if (err_msg) + *err_msg = curl_easy_strerror(rc); + errno = EIO; + return -1; + } + if (status < 200 || status >= 300) { + free(body.buf); + if (err_msg) + *err_msg = "token endpoint returned non-2xx status"; + errno = EPROTO; + return -1; + } + if (!body.buf || body.len == 0) { + free(body.buf); + if (err_msg) + *err_msg = "token endpoint returned empty body"; + errno = EPROTO; + return -1; + } + + cJSON *json = cJSON_ParseWithLength(body.buf, body.len); + free(body.buf); + if (!json) { + if (err_msg) + *err_msg = "token endpoint returned invalid JSON"; + errno = EPROTO; + return -1; + } + cJSON *t = cJSON_GetObjectItemCaseSensitive(json, "token"); + if (!cJSON_IsString(t) || !t->valuestring) + t = cJSON_GetObjectItemCaseSensitive(json, "access_token"); + if (!cJSON_IsString(t) || !t->valuestring) { + cJSON_Delete(json); + if (err_msg) + *err_msg = "token endpoint response lacks 'token' field"; + errno = EPROTO; + return -1; + } + free(f->bearer_token); + f->bearer_token = strdup(t->valuestring); + cJSON_Delete(json); + if (!f->bearer_token) { + if (err_msg) + *err_msg = "out of memory caching token"; + errno = ENOMEM; + return -1; + } + return 0; +} + +static int perform_manifest_get(oci_fetcher_t *f, + const effective_opts_t *eff, + const char *url, + const char *const *accept_types, + const char *if_none_match, + oci_fetch_response_t *out, + bearer_challenge_t *challenge_out, + const char **err_msg) +{ + body_buf_t body = {.max = FETCH_BODY_MAX}; + headers_ctx_t hctx = {.challenge_out = challenge_out}; + if (challenge_out) + bearer_challenge_free(challenge_out); + + curl_easy_reset(f->easy); + apply_security_opts(f->easy, eff); + curl_easy_setopt(f->easy, CURLOPT_URL, url); + curl_easy_setopt(f->easy, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(f->easy, CURLOPT_MAXREDIRS, 5L); + curl_easy_setopt(f->easy, CURLOPT_USERAGENT, "elfuse-oci/1"); + curl_easy_setopt(f->easy, CURLOPT_WRITEFUNCTION, body_write_cb); + curl_easy_setopt(f->easy, CURLOPT_WRITEDATA, &body); + curl_easy_setopt(f->easy, CURLOPT_HEADERFUNCTION, header_cb); + curl_easy_setopt(f->easy, CURLOPT_HEADERDATA, &hctx); + struct curl_slist *hdrs = + build_request_headers(f, accept_types, if_none_match); + if (hdrs) + curl_easy_setopt(f->easy, CURLOPT_HTTPHEADER, hdrs); + + CURLcode rc = curl_easy_perform(f->easy); + long status = 0; + curl_easy_getinfo(f->easy, CURLINFO_RESPONSE_CODE, &status); + if (hdrs) + curl_slist_free_all(hdrs); + + out->http_status = status; + if (rc != CURLE_OK) { + free(body.buf); + free(hctx.content_type); + free(hctx.docker_content_digest); + free(hctx.etag); + if (err_msg) + *err_msg = curl_easy_strerror(rc); + errno = EIO; + return -1; + } + if (body.overflow) { + free(body.buf); + free(hctx.content_type); + free(hctx.docker_content_digest); + free(hctx.etag); + if (err_msg) + *err_msg = "response body exceeded max size"; + errno = EFBIG; + return -1; + } + out->body = body.buf; + out->body_len = body.len; + out->content_type = hctx.content_type; + out->docker_content_digest = hctx.docker_content_digest; + out->etag = hctx.etag; + return 0; +} + +int oci_fetch_manifest(oci_fetcher_t *f, + const oci_ref_t *ref, + const char *digest_or_tag, + const char *const *accept_types, + const char *if_none_match, + oci_fetch_response_t *out, + const char **err_msg) +{ + if (!f || !ref || !out) { + if (err_msg) + *err_msg = "invalid arguments"; + errno = EINVAL; + return -1; + } + memset(out, 0, sizeof(*out)); + effective_opts_t eff; + if (resolve_effective(f, ref, &eff, err_msg) < 0) + return -1; + if (check_insecure_policy(&eff, ref, err_msg) < 0) { + effective_free(&eff); + return -1; + } + const char *selector = digest_or_tag; + if (!selector) + selector = ref->digest; + if (!selector) + selector = ref->tag; + if (!selector) { + if (err_msg) + *err_msg = "reference has no tag or digest"; + errno = EINVAL; + effective_free(&eff); + return -1; + } + char *url = build_manifest_url(f, ref, selector); + if (!url) { + if (err_msg) + *err_msg = "out of memory"; + errno = ENOMEM; + effective_free(&eff); + return -1; + } + + bearer_challenge_t challenge = {0}; + /* Always capture the Bearer challenge, even when a token is already + * cached: registry tokens are short-lived (Docker Hub expires them in + * ~300s), so a long multi-blob pull can outlive the cached token. If the + * stale token 401s, the captured challenge lets the block below refresh + * it and retry instead of failing the pull. + */ + int rc = perform_manifest_get(f, &eff, url, accept_types, if_none_match, + out, &challenge, err_msg); + if (rc < 0) { + free(url); + bearer_challenge_free(&challenge); + effective_free(&eff); + return -1; + } + + if (out->http_status == 401 && challenge.realm) { + bearer_challenge_free(&f->challenge); + f->challenge = challenge; + memset(&challenge, 0, sizeof(challenge)); + oci_fetch_response_free(out); + memset(out, 0, sizeof(*out)); + if (fetch_token(f, &eff, err_msg) < 0) { + free(url); + effective_free(&eff); + return -1; + } + rc = perform_manifest_get(f, &eff, url, accept_types, if_none_match, + out, NULL, err_msg); + if (rc < 0) { + free(url); + effective_free(&eff); + return -1; + } + } else { + bearer_challenge_free(&challenge); + } + + free(url); + effective_free(&eff); + + /* 304 Not Modified is a success path for conditional revalidation: the + * caller asked the registry whether the pinned digest still matches and + * the answer is yes. The body is intentionally empty; the etag (when the + * server emitted one) stays attached for caller diagnostics. + */ + if (out->http_status == 304) + return 0; + if (out->http_status < 200 || out->http_status >= 300) { + if (err_msg) + *err_msg = "manifest fetch returned non-2xx status"; + errno = EPROTO; + return -1; + } + return 0; +} + +typedef struct { + oci_blob_writer_t *w; + /* The easy handle this stream feeds. Needed so the body callback can + * peek CURLINFO_RESPONSE_CODE on the first chunk and notice when a + * server ignored the Range header (200 instead of 206) before any + * bytes get committed to the writer. + */ + CURL *easy; + int64_t bytes_seen; + int64_t bytes_expected; + /* Bytes already present on disk in the writer's partial. Zero on a + * fresh fetch. Drives the body-callback's status peek. + */ + int64_t resume_offset; + bool overflow; + bool write_failed; + /* Set when the body callback observes a non-206 status while the + * request carried a Range header. Triggers BH_NEEDS_RESTART in the + * score path; the writer's polluted digester state is discarded + * along with the partial when the restart re-arms a fresh writer. + */ + bool range_rejected; +} blob_stream_ctx_t; + +static size_t blob_stream_cb(char *ptr, + size_t size, + size_t nmemb, + void *userdata) +{ + blob_stream_ctx_t *ctx = userdata; + size_t n = size * nmemb; + if (ctx->overflow || ctx->write_failed || ctx->range_rejected) + return 0; + /* First chunk on a resumed transfer: if the server replied with + * anything other than 206 Partial Content, the Range header was + * ignored or rejected. Surface the restart signal here rather than + * letting the size cap trip on the full-body retransmission. + */ + if (ctx->resume_offset > 0 && ctx->bytes_seen == ctx->resume_offset && + ctx->easy) { + long status = 0; + curl_easy_getinfo(ctx->easy, CURLINFO_RESPONSE_CODE, &status); + if (status != 206) { + ctx->range_rejected = true; + return 0; + } + } + int64_t projected = ctx->bytes_seen + (int64_t) n; + if (projected > ctx->bytes_expected) { + ctx->overflow = true; + return 0; + } + if (!oci_blob_writer_write(ctx->w, ptr, n)) { + ctx->write_failed = true; + return 0; + } + ctx->bytes_seen = projected; + return n; +} + +/* Per-handle state for a batch transfer. The handle owns its easy handle, + * staging writer, URL string, request-header slist, and any captured bearer + * challenge / response headers. batch_handle_free is safe to call on a + * zero-initialised slot, and safe to call multiple times. + */ +typedef enum { + BH_ACTIVE, /* enqueueable: not yet completed this round */ + BH_NEEDS_RETRY, /* first round hit 401 + Bearer challenge */ + BH_NEEDS_RESTART, /* server ignored Range or replied 416; refetch fresh */ + BH_DONE_OK, /* transfer completed; writer holds verified bytes */ + BH_FAILED, /* transport / status / size error; err_msg populated */ +} batch_state_t; + +typedef struct { + const oci_descriptor_t *desc; + oci_blob_writer_t *w; + char *url; + CURL *easy; + blob_stream_ctx_t bctx; + bearer_challenge_t challenge; + headers_ctx_t hctx; + struct curl_slist *hdrs; + long http_status; + CURLcode last_curl_rc; + batch_state_t state; + bool added; + /* Bytes already present on disk from a prior interrupted fetch. Zero on + * a fresh start; positive when oci_blob_writer_resume_named picked up a + * partial. Drives the per-handle Range header and the score-side detection + * of a server that ignored the Range request. + */ + int64_t resume_offset; + /* Per-blob progress callback. Borrowed from the batch entry's argument; + * NULL when the caller did not request progress. The xferinfo wrapper + * forwards into this callback with bytes_dl adjusted to total-blob + * progress (libcurl's dlnow + resume_offset) so the renderer can pair + * bytes_dl with desc->size as a true completion ratio. + */ + oci_fetch_blob_batch_progress_cb_t progress_cb; + void *progress_user; + const char *err_msg; +} batch_handle_t; + +/* libcurl xferinfo wrapper. clientp is the owning batch_handle_t so the + * callback can look up the descriptor and the resume offset without a + * separate context struct. dltotal is ignored because resumed transfers + * report dltotal == remaining bytes, not the full blob size -- desc->size + * is the authoritative total. The return value propagates from the + * caller's progress_cb so a future renderer can abort a transfer by + * returning non-zero, matching libcurl's xferinfo contract. + */ +static int batch_xferinfo_cb(void *clientp, + curl_off_t dltotal, + curl_off_t dlnow, + curl_off_t ultotal, + curl_off_t ulnow) +{ + (void) dltotal; + (void) ultotal; + (void) ulnow; + batch_handle_t *h = clientp; + if (!h || !h->progress_cb) + return 0; + int64_t bytes_dl = (int64_t) dlnow + h->resume_offset; + return h->progress_cb(h->desc, bytes_dl, h->desc->size, h->progress_user); +} + +static int batch_max_concurrent(void) +{ + const char *e = getenv("OCI_FETCH_MAX_CONCURRENT"); + if (!e || !*e) + return 4; + long n = strtol(e, NULL, 10); + if (n < 1) + n = 1; + if (n > 16) + n = 16; + return (int) n; +} + +static void batch_handle_free(batch_handle_t *h) +{ + if (h->w) { + oci_blob_writer_abort(h->w); + h->w = NULL; + } + if (h->easy) { + curl_easy_cleanup(h->easy); + h->easy = NULL; + } + if (h->hdrs) { + curl_slist_free_all(h->hdrs); + h->hdrs = NULL; + } + free(h->url); + h->url = NULL; + bearer_challenge_free(&h->challenge); + free(h->hctx.content_type); + h->hctx.content_type = NULL; + free(h->hctx.docker_content_digest); + h->hctx.docker_content_digest = NULL; + free(h->hctx.etag); + h->hctx.etag = NULL; +} + +/* Configure an easy handle for a blob fetch. Used both at initial prepare + * time and (after a writer + slist reset) during the post-401 retry round. + * The challenge capture slot is wired only on round 0 since the existing + * single-blob path only attempts one refresh. + */ +static void batch_configure_easy(oci_fetcher_t *f, + const effective_opts_t *eff, + batch_handle_t *h, + bool capture_challenge) +{ + h->bctx.w = h->w; + h->bctx.easy = h->easy; + /* Seed bytes_seen with the partial bytes the writer already absorbed so + * the streaming overflow gate measures total-blob progress against + * desc->size, not just the bytes the server returned on this leg. + */ + h->bctx.bytes_seen = h->resume_offset; + h->bctx.bytes_expected = h->desc->size; + h->bctx.resume_offset = h->resume_offset; + h->bctx.overflow = false; + h->bctx.write_failed = false; + h->bctx.range_rejected = false; + h->hctx.challenge_out = capture_challenge ? &h->challenge : NULL; + + apply_security_opts(h->easy, eff); + curl_easy_setopt(h->easy, CURLOPT_URL, h->url); + curl_easy_setopt(h->easy, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(h->easy, CURLOPT_MAXREDIRS, 5L); + curl_easy_setopt(h->easy, CURLOPT_USERAGENT, "elfuse-oci/1"); + curl_easy_setopt(h->easy, CURLOPT_WRITEFUNCTION, blob_stream_cb); + curl_easy_setopt(h->easy, CURLOPT_WRITEDATA, &h->bctx); + curl_easy_setopt(h->easy, CURLOPT_HEADERFUNCTION, header_cb); + curl_easy_setopt(h->easy, CURLOPT_HEADERDATA, &h->hctx); + if (h->resume_offset > 0) { + char range[64]; + snprintf(range, sizeof(range), "%lld-", (long long) h->resume_offset); + curl_easy_setopt(h->easy, CURLOPT_RANGE, range); + } + if (h->progress_cb) { + curl_easy_setopt(h->easy, CURLOPT_NOPROGRESS, 0L); + curl_easy_setopt(h->easy, CURLOPT_XFERINFOFUNCTION, batch_xferinfo_cb); + curl_easy_setopt(h->easy, CURLOPT_XFERINFODATA, h); + } + h->hdrs = build_request_headers(f, NULL, NULL); + if (h->hdrs) + curl_easy_setopt(h->easy, CURLOPT_HTTPHEADER, h->hdrs); +} + +static int batch_prepare_handle(oci_fetcher_t *f, + const effective_opts_t *eff, + const oci_ref_t *ref, + batch_handle_t *h, + oci_blob_store_t *store, + const char **err_msg) +{ + h->w = oci_blob_writer_resume_named(store, h->desc->algo, h->desc->hex, + h->desc->size, &h->resume_offset); + if (!h->w) { + if (err_msg) + *err_msg = "failed to start blob writer"; + return -1; + } + h->url = build_blob_url(f, ref, h->desc->digest_str); + if (!h->url) { + if (err_msg) + *err_msg = "out of memory"; + errno = ENOMEM; + return -1; + } + h->easy = curl_easy_init(); + if (!h->easy) { + if (err_msg) + *err_msg = "curl_easy_init failed"; + errno = EIO; + return -1; + } + h->state = BH_ACTIVE; + h->added = false; + h->http_status = 0; + h->last_curl_rc = CURLE_OK; + h->err_msg = NULL; + /* Capture the Bearer challenge on round 0 even with a cached token: a + * short-lived token can expire mid-pull, and the round-0 401 retry can + * only refresh it if the challenge was parsed off the response. + */ + batch_configure_easy(f, eff, h, true); + return 0; +} + +/* Re-arm a handle for a fresh transfer attempt: token-refresh retry after a + * 401 + Bearer challenge, or restart-from-zero after a server ignored the + * Range header (200 instead of 206) or replied 416. The original writer is + * aborted (its staging file gets unlinked) and a brand-new one starts at + * byte zero, with resume_offset reset so batch_configure_easy emits no + * Range header on the next attempt. The easy handle is reset and re-wired + * with the current bearer token. Challenge capture is disabled so any + * second-round 401 falls straight through to FAILED, and a second-round + * 200-after-Range cannot reoccur because resume_offset is now zero. + */ +static int batch_reset_handle_fresh(oci_fetcher_t *f, + const effective_opts_t *eff, + batch_handle_t *h, + oci_blob_store_t *store) +{ + oci_blob_writer_abort(h->w); + h->w = NULL; + if (h->hdrs) { + curl_slist_free_all(h->hdrs); + h->hdrs = NULL; + } + free(h->hctx.content_type); + h->hctx.content_type = NULL; + free(h->hctx.docker_content_digest); + h->hctx.docker_content_digest = NULL; + free(h->hctx.etag); + h->hctx.etag = NULL; + bearer_challenge_free(&h->challenge); + + h->w = oci_blob_writer_begin_named(store, h->desc->algo, h->desc->hex); + if (!h->w) + return -1; + h->resume_offset = 0; + curl_easy_reset(h->easy); + h->state = BH_ACTIVE; + h->added = false; + h->http_status = 0; + h->last_curl_rc = CURLE_OK; + h->err_msg = NULL; + batch_configure_easy(f, eff, h, false); + return 0; +} + +/* Score a completed CURLMSG_DONE entry. Translates a curl + HTTP status pair + * into a batch_state_t transition, mirroring the diagnostic strings the + * single-blob path historically produced so the test suite's err_msg + * assertions stay byte-identical. + */ +static void batch_score_done(batch_handle_t *h, + CURLcode crc, + long status, + int round) +{ + h->http_status = status; + h->last_curl_rc = crc; + /* The body callback flagged a non-206 response to a Range request. + * Surface the restart intent before any size / digest / curl-error + * diagnostics: the discarded bytes would otherwise look like an + * overflow, and a 416 with a small error body would look like a + * payload write failure. + */ + if (h->bctx.range_rejected) { + h->state = BH_NEEDS_RESTART; + return; + } + if (crc != CURLE_OK) { + if (h->bctx.overflow) { + h->err_msg = "blob exceeded declared size"; + errno = EPROTO; + } else if (h->bctx.write_failed) { + h->err_msg = "blob writer rejected payload"; + errno = EIO; + } else { + h->err_msg = curl_easy_strerror(crc); + errno = EIO; + } + h->state = BH_FAILED; + return; + } + if (status == 401 && h->challenge.realm && round == 0) { + h->state = BH_NEEDS_RETRY; + return; + } + /* A 416 with an empty body never reached blob_stream_cb, so the + * range_rejected flag above did not fire. Catch that path here. + * status == 200 with resume_offset > 0 also belongs to this restart + * arm, though in practice the body callback catches it first. + */ + if (h->resume_offset > 0 && (status == 200 || status == 416)) { + h->state = BH_NEEDS_RESTART; + return; + } + if (status < 200 || status >= 300) { + h->err_msg = "blob fetch returned non-2xx status"; + errno = EPROTO; + h->state = BH_FAILED; + return; + } + if (h->bctx.bytes_seen != h->desc->size) { + h->err_msg = "blob size mismatch"; + errno = EPROTO; + h->state = BH_FAILED; + return; + } + h->state = BH_DONE_OK; + /* libcurl's xferinfo does not guarantee a final dlnow == dltotal tick, + * so the renderer would otherwise stall one update short of "done". + * One explicit invocation at the score boundary normalises the + * sequence the user-side callback sees, regardless of socket pacing. + */ + if (h->progress_cb) + (void) h->progress_cb(h->desc, h->desc->size, h->desc->size, + h->progress_user); +} + +int oci_fetch_blob_batch(oci_fetcher_t *f, + const oci_ref_t *ref, + const oci_descriptor_t *const *descs, + size_t n_descs, + oci_blob_store_t *store, + oci_fetch_blob_batch_progress_cb_t progress_cb, + void *cb_user_data, + const char **err_msg) +{ + if (!f || !ref || !descs || !store) { + if (err_msg) + *err_msg = "invalid arguments"; + errno = EINVAL; + return -1; + } + if (n_descs == 0) + return 0; + + effective_opts_t eff; + if (resolve_effective(f, ref, &eff, err_msg) < 0) + return -1; + if (check_insecure_policy(&eff, ref, err_msg) < 0) { + effective_free(&eff); + return -1; + } + + /* Drop stale tmp partials that no surviving batch can resume from. A + * week is long enough to let an interrupted multi-day pull finish on + * the next attempt while still keeping the staging area bounded for + * caches that see frequent unique blobs. The blob-store guards the + * tmp/ namespace, so the wide blob-* prefix cannot touch unrelated + * files. + */ + oci_blob_store_sweep_partials(store, 7L * 86400); + + int rc = -1; + int max_concurrent = batch_max_concurrent(); + bool any_failed = false; + CURLM *multi = NULL; + + batch_handle_t *handles = calloc(n_descs, sizeof(*handles)); + if (!handles) { + if (err_msg) + *err_msg = "out of memory"; + errno = ENOMEM; + goto cleanup; + } + + /* Dedup pass: drop blobs already in the store, collapse same-digest + * entries (the layers array can repeat a digest legitimately). nh is + * the number of handles that actually need a transfer. + */ + size_t nh = 0; + for (size_t i = 0; i < n_descs; i++) { + const oci_descriptor_t *d = descs[i]; + if (!d || d->size < 0) { + if (err_msg) + *err_msg = "descriptor size is negative"; + errno = EINVAL; + goto cleanup; + } + if (oci_blob_store_has(store, d->algo, d->hex)) + continue; + bool dup = false; + for (size_t j = 0; j < nh; j++) { + if (handles[j].desc->algo == d->algo && + strcmp(handles[j].desc->hex, d->hex) == 0) { + dup = true; + break; + } + } + if (dup) + continue; + handles[nh].desc = d; + handles[nh].progress_cb = progress_cb; + handles[nh].progress_user = cb_user_data; + nh++; + } + if (nh == 0) { + rc = 0; + goto cleanup; + } + + for (size_t i = 0; i < nh; i++) { + if (batch_prepare_handle(f, &eff, ref, &handles[i], store, err_msg) < 0) + goto cleanup; + } + + multi = curl_multi_init(); + if (!multi) { + if (err_msg) + *err_msg = "curl_multi_init failed"; + errno = EIO; + goto cleanup; + } + + int round = 0; + /* Outer loop: each iteration tops up the multi up to max_concurrent + * ACTIVE handles and drains them until still_running hits zero. When no + * ACTIVE remain the loop checks for NEEDS_RETRY (single token refresh + * per batch) and either restarts those handles or exits. + */ + while (1) { + size_t added_count = 0; + for (size_t i = 0; i < nh; i++) { + if (handles[i].added) + added_count++; + } + for (size_t i = 0; i < nh && added_count < (size_t) max_concurrent; + i++) { + if (handles[i].state == BH_ACTIVE && !handles[i].added) { + if (curl_multi_add_handle(multi, handles[i].easy) == CURLM_OK) { + handles[i].added = true; + added_count++; + } + } + } + if (added_count == 0) + break; + + int still_running = 0; + do { + int num_fds = 0; + CURLMcode mrc = curl_multi_poll(multi, NULL, 0, 1000, &num_fds); + if (mrc != CURLM_OK) { + if (err_msg) + *err_msg = curl_multi_strerror(mrc); + errno = EIO; + any_failed = true; + goto drained; + } + curl_multi_perform(multi, &still_running); + CURLMsg *msg; + int n_msgs = 0; + while ((msg = curl_multi_info_read(multi, &n_msgs)) != NULL) { + if (msg->msg != CURLMSG_DONE) + continue; + batch_handle_t *h = NULL; + for (size_t i = 0; i < nh; i++) { + if (handles[i].easy == msg->easy_handle) { + h = &handles[i]; + break; + } + } + if (!h) + continue; + CURLcode crc = msg->data.result; + long status = 0; + curl_easy_getinfo(h->easy, CURLINFO_RESPONSE_CODE, &status); + curl_multi_remove_handle(multi, h->easy); + h->added = false; + batch_score_done(h, crc, status, round); + } + } while (still_running > 0); + drained:; + /* If there are still ACTIVE slots not yet enqueued, fall back into + * the outer loop to add them; otherwise check for retries. + */ + bool more_active = false; + for (size_t i = 0; i < nh; i++) + if (handles[i].state == BH_ACTIVE) { + more_active = true; + break; + } + if (more_active) + continue; + + bool any_retry = false; + bool any_restart = false; + for (size_t i = 0; i < nh; i++) { + if (handles[i].state == BH_NEEDS_RETRY) + any_retry = true; + else if (handles[i].state == BH_NEEDS_RESTART) + any_restart = true; + } + if (!any_retry && !any_restart) + break; + if (any_failed) + break; + + if (any_retry) { + /* Single token refresh per batch. Steal one retry handle's + * challenge onto f->challenge so fetch_token sees the + * realm/service/scope, then re-arm every NEEDS_RETRY handle + * with the new bearer. + */ + for (size_t i = 0; i < nh; i++) { + if (handles[i].state == BH_NEEDS_RETRY) { + bearer_challenge_free(&f->challenge); + f->challenge = handles[i].challenge; + memset(&handles[i].challenge, 0, + sizeof(handles[i].challenge)); + break; + } + } + if (fetch_token(f, &eff, err_msg) < 0) { + for (size_t i = 0; i < nh; i++) { + if (handles[i].state == BH_NEEDS_RETRY) { + handles[i].state = BH_FAILED; + handles[i].err_msg = "token refresh failed"; + } + } + any_failed = true; + break; + } + round++; + for (size_t i = 0; i < nh; i++) { + if (handles[i].state == BH_NEEDS_RETRY) { + if (batch_reset_handle_fresh(f, &eff, &handles[i], store) < + 0) { + handles[i].state = BH_FAILED; + handles[i].err_msg = "failed to reset writer for retry"; + any_failed = true; + } + } + } + if (any_failed) + break; + } + + if (any_restart) { + /* Range-resume retry: no token refresh, just a fresh writer + * with resume_offset cleared so the next attempt fetches the + * full blob without a Range header. The reset itself zeroes + * resume_offset, so a second 200-after-Range / 416 cannot + * pick the restart branch again -- the handle either + * succeeds or falls into BH_FAILED on the next score. + */ + for (size_t i = 0; i < nh; i++) { + if (handles[i].state == BH_NEEDS_RESTART) { + if (batch_reset_handle_fresh(f, &eff, &handles[i], store) < + 0) { + handles[i].state = BH_FAILED; + handles[i].err_msg = + "failed to reset writer for restart"; + any_failed = true; + } + } + } + if (any_failed) + break; + } + } + + for (size_t i = 0; i < nh; i++) { + if (handles[i].state == BH_FAILED) { + any_failed = true; + if (err_msg && !*err_msg && handles[i].err_msg) + *err_msg = handles[i].err_msg; + } else if (handles[i].state == BH_ACTIVE || + handles[i].state == BH_NEEDS_RETRY || + handles[i].state == BH_NEEDS_RESTART) { + /* Should be unreachable: the loop only exits when nothing is + * still queued. Defensive: treat as failure rather than + * silently dropping the slot. + */ + any_failed = true; + handles[i].state = BH_FAILED; + if (err_msg && !*err_msg) + *err_msg = "batch left a handle in a non-terminal state"; + } + } + if (any_failed) { + if (err_msg && !*err_msg) + *err_msg = "batch blob fetch failed"; + goto cleanup; + } + + /* Commit only after every transfer succeeded. Commit consumes the writer + * (frees on success), so clear h->w to suppress the batch_handle_free + * abort. A digest mismatch here aborts any remaining unflushed writers + * and surfaces the historical "blob digest mismatch on commit" string. + */ + for (size_t i = 0; i < nh; i++) { + if (handles[i].state != BH_DONE_OK) + continue; + if (oci_blob_writer_commit(handles[i].w) < 0) { + handles[i].w = NULL; + if (err_msg) + *err_msg = "blob digest mismatch on commit"; + for (size_t j = i + 1; j < nh; j++) { + if (handles[j].state == BH_DONE_OK && handles[j].w) { + oci_blob_writer_abort(handles[j].w); + handles[j].w = NULL; + } + } + goto cleanup; + } + handles[i].w = NULL; + } + rc = 0; + +cleanup: + if (multi) + curl_multi_cleanup(multi); + if (handles) { + for (size_t i = 0; i < n_descs; i++) + batch_handle_free(&handles[i]); + free(handles); + } + effective_free(&eff); + return rc; +} + +int oci_fetch_blob(oci_fetcher_t *f, + const oci_ref_t *ref, + const oci_descriptor_t *desc, + oci_blob_store_t *store, + const char **err_msg) +{ + if (!desc) { + if (err_msg) + *err_msg = "invalid arguments"; + errno = EINVAL; + return -1; + } + return oci_fetch_blob_batch(f, ref, &desc, 1, store, NULL, NULL, err_msg); +} diff --git a/src/oci/fetch.h b/src/oci/fetch.h new file mode 100644 index 0000000..c91009f --- /dev/null +++ b/src/oci/fetch.h @@ -0,0 +1,228 @@ +/* OCI registry HTTPS client + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Wraps libcurl for the subset of the OCI distribution-spec that elfuse needs + * to pull an image: + * + * - Anonymous GET against /v2//manifests/ and + * /v2//blobs/ + * - 401 + Www-Authenticate: Bearer challenge: fetch a token from the realm + * advertised by the registry, then retry the original request with + * Authorization: Bearer + * - Blob streaming: pipe the response body into the slice-2 blob store with + * digest and declared-size verification, so a hostile or truncated layer + * never produces a visible-complete blob + * + * Future slices extend the options struct with basic auth credentials, + * custom CA bundle, and a loopback-gated TLS verify-off path + * (oci-roadmap.md Q7 ship list). The public entry points stay stable. + * + * Thread safety: oci_fetch_global_init must run once before any fetcher is + * created. Each oci_fetcher_t holds its own libcurl easy handle and is not + * safe to share across threads; create one per worker. + */ + +#pragma once + +#include +#include + +#include "blob-store.h" +#include "manifest.h" +#include "ref.h" + +typedef struct oci_policy oci_policy_t; + +typedef struct { + /* Optional override of the registry base URL. When non-NULL, the fetcher + * uses this prefix for every /v2/... request instead of computing one + * from ref->registry. Test scaffolding sets this to a local mock + * (https://127.0.0.1:); production callers leave it NULL. + */ + const char *base_url_override; + + /* HTTP Basic authentication. When username is non-NULL, libcurl produces + * Authorization: Basic on every request the fetcher + * issues, including the token endpoint when the registry also requires a + * Bearer flow. password may be NULL for an empty secret. CLI-supplied + * credentials override anything a policy auth_file points at for the + * same registry. + */ + const char *username; + const char *password; + + /* Path to a PEM-encoded CA bundle. When non-NULL the fetcher passes it to + * libcurl as CURLOPT_CAINFO, replacing the system trust store for that + * connection. Effective only with an OpenSSL-style SSL backend (the + * default macOS Secure Transport backend ignores CAINFO). CLI-supplied + * ca_file overrides any policy ca_bundle for the same registry. + */ + const char *ca_file; + + /* Disable TLS verification. Honored only when the resolved registry host + * is on the loopback whitelist (127.0.0.1, localhost, ::1). Any other + * host with allow_insecure=true causes oci_fetch_manifest / + * oci_fetch_blob to fail with errno=EPERM before a single byte is sent. + * A policy insecure=true for the resolved host has the same effect and + * goes through the same loopback gate; CLI allow_insecure=true is an + * override that wins when the policy declares insecure=false. + */ + bool allow_insecure; + + /* Optional reference to a loaded oci_policy_t. When non-NULL the fetcher + * consults the policy on every manifest/blob request using ref->registry + * as the lookup key and merges the per-host effective view with the CLI + * options above (CLI wins). Lifetime is caller-owned; the policy must + * outlive the fetcher. + */ + const oci_policy_t *policy; +} oci_fetcher_options_t; + +typedef struct oci_fetcher oci_fetcher_t; + +/* Per-process libcurl global init. Safe to call multiple times; only the + * first call performs work. Returns 0 on success or -1 with errno=EIO if + * libcurl rejects the initialization. + */ +int oci_fetch_global_init(void); + +/* Counterpart of oci_fetch_global_init. The caller may invoke it on shutdown + * but elfuse runs short enough that leaving libcurl initialized until process + * exit is acceptable. + */ +void oci_fetch_global_cleanup(void); + +/* Allocate a fetcher. opts may be NULL for defaults. Returns NULL on + * allocation failure with errno preserved. + */ +oci_fetcher_t *oci_fetcher_new(const oci_fetcher_options_t *opts); + +/* Release the fetcher. Safe on NULL. */ +void oci_fetcher_free(oci_fetcher_t *f); + +typedef struct { + /* Heap-allocated response body. NUL-terminated so callers can pass it + * directly to JSON parsers that expect a C string, while body_len is the + * authoritative byte count. + */ + char *body; + size_t body_len; + /* Content-Type header value with parameters stripped (everything before + * the first ';'). NULL if the server omitted the header. + */ + char *content_type; + /* Docker-Content-Digest header value verbatim, e.g. "sha256:abc...". + * NULL if the server omitted it. Useful for tag-to-digest pinning. + */ + char *docker_content_digest; + /* ETag header verbatim, including any surrounding quotes or weak prefix + * (e.g. "sha256:abc..." or W/"..."). NULL if the server omitted it. + * Captured so conditional-GET callers can echo it back without parsing. + */ + char *etag; + long http_status; +} oci_fetch_response_t; + +/* Release any heap fields. Safe on a zero-initialised struct. */ +void oci_fetch_response_free(oci_fetch_response_t *r); + +/* Fetch a manifest, image index, or image config blob by reference. + * + * ref registry/repository, plus optional default tag/digest + * digest_or_tag the actual GET selector ("sha256:..." or a tag string). + * NULL means: use ref->digest if set, otherwise ref->tag. + * accept_types NULL-terminated list of media types to advertise in the + * Accept header. Pass NULL to suppress the Accept header. + * if_none_match optional If-None-Match value sent verbatim. Pass the + * registry-style strong quoted form ("sha256:...") to ask + * the registry for 304 Not Modified when the upstream + * manifest still hashes to the pinned digest. NULL skips + * the conditional header entirely. + * + * On success returns 0 and fills *out (caller frees via + * oci_fetch_response_free). A 304 response is success: out->http_status is + * 304, out->body is NULL, out->body_len is 0, and out->etag may still be + * populated. On HTTP error (other non-2xx) returns -1 with out->http_status + * populated and errno=EPROTO; the body may still be present for + * diagnostics. On transport / auth failure returns -1 with errno preserved + * and *err_msg (when non-NULL) pointing at a static description. + */ +int oci_fetch_manifest(oci_fetcher_t *f, + const oci_ref_t *ref, + const char *digest_or_tag, + const char *const *accept_types, + const char *if_none_match, + oci_fetch_response_t *out, + const char **err_msg); + +/* Fetch a blob into the local store. The descriptor's algo, hex, and size + * fields drive verification: incoming bytes feed an oci_blob_writer keyed by + * the digest, the running byte count is capped at desc->size so a hostile + * server cannot stream forever, and the writer's own digest check at commit + * rejects any payload that hashes to anything other than desc->hex. + * + * Returns 0 on success, -1 with errno set on failure. err_msg points at a + * static description for the common diagnostic modes (digest mismatch, + * size mismatch, transport error, HTTP status). + * + * Already-present blobs are an immediate success (store-side has() check) + * with no network call. + * + * Implementation is a one-element forwarder onto oci_fetch_blob_batch; the + * separate entry point exists so callers that only need a single blob need + * not allocate a descriptor array. + */ +int oci_fetch_blob(oci_fetcher_t *f, + const oci_ref_t *ref, + const oci_descriptor_t *desc, + oci_blob_store_t *store, + const char **err_msg); + +/* Per-blob progress callback. Invoked (potentially repeatedly) by the batch + * fetcher as bytes accrue. bytes_dl is the number of bytes already streamed + * for desc; bytes_total mirrors desc->size. Return value is reserved (C5.3 + * may use a non-zero return to signal abort); C5.1 callers should return 0. + * + * Reserved typedef -- the callback parameter to oci_fetch_blob_batch is + * accepted but not yet invoked. C5.3 wires it through CURLOPT_XFERINFOFUNCTION + * for in-flight progress; the typedef is committed at C5.1 so the batch API + * surface does not move between Plan 5 commits. + */ +typedef int (*oci_fetch_blob_batch_progress_cb_t)(const oci_descriptor_t *desc, + int64_t bytes_dl, + int64_t bytes_total, + void *user_data); + +/* Fetch multiple blobs in parallel via libcurl's multi interface. + * + * descs / n_descs array of descriptor pointers to fetch. Already-present + * blobs (store-side has() hit) are skipped before any + * handle is allocated. Duplicate digests within the batch + * are collapsed to one transfer. + * progress_cb optional per-blob progress callback (C5.3); pass NULL + * cb_user_data opaque pointer forwarded to progress_cb + * + * Concurrency cap reads OCI_FETCH_MAX_CONCURRENT (default 4, clamped to + * [1, 16]). The batch is atomic: any single-blob failure (network error, + * HTTP non-2xx after one auth retry, size or digest mismatch) aborts every + * in-flight writer and the function returns -1 with err_msg set. On success + * every blob is committed to the store before the function returns; commits + * happen sequentially after all transfers succeed. + * + * A single shared effective_opts_t is resolved at entry (policy + CLI merge + * for ref->registry). Token refresh is handled inline: when any first-round + * handle returns 401 with a Bearer challenge, the batch drains all + * in-flight handles, refreshes the fetcher's bearer token once, and restarts + * every 401 handle with the new bearer header. A second 401 fails the + * batch. + */ +int oci_fetch_blob_batch(oci_fetcher_t *f, + const oci_ref_t *ref, + const oci_descriptor_t *const *descs, + size_t n_descs, + oci_blob_store_t *store, + oci_fetch_blob_batch_progress_cb_t progress_cb, + void *cb_user_data, + const char **err_msg); diff --git a/src/oci/inspect.c b/src/oci/inspect.c new file mode 100644 index 0000000..f2a4391 --- /dev/null +++ b/src/oci/inspect.c @@ -0,0 +1,573 @@ +/* Offline manifest tree renderer for elfuse oci inspect + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Reads the blob the local pin points at, classifies it as an image index or + * image manifest, and prints a tree. No network, no fetcher. The manifest + * model from slice 3 enforces every digest is lowercase and every descriptor + * size is non-negative, so the renderer can trust its inputs once the parse + * returns 0. + * + * Detection between index and manifest is structural: oci_index_parse refuses + * a body that has no "manifests" array, oci_manifest_parse refuses a body + * that has no "config" + "layers" pair. The two parsers therefore reject + * disjoint shapes, and trying one then the other is unambiguous. Image + * configs never reach this code path because pins point at manifest-shaped + * blobs (slice 5a stores the manifest body it received from the registry). + */ + +#include "inspect.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "blob-store.h" +#include "dedup-metrics.h" +#include "digest.h" +#include "manifest.h" +#include "media-type.h" + +/* Upper bound on a manifest/index body. Real manifests are well under 1 MiB; + * a 64 MiB cap is generous and prevents a corrupted store from forcing a + * pathological malloc. + */ +#define INSPECT_BODY_MAX ((size_t) 64 * 1024 * 1024) + +/* Render a digest in two compact forms: + * + * - short_digest("sha256:abcdef0123456789...") + * -> "sha256:abcdef012345..." (first 19 chars + "...") + * + * Matches the slice 5a pull progress line so the two surfaces stay visually + * consistent. The caller-supplied buffer keeps the function reentrant; using + * one static buffer would clobber on the second %s in a single printf. + */ +static void short_digest(const char *full, char out[24]) +{ + if (!full) { + snprintf(out, 24, "(null)"); + return; + } + size_t len = strlen(full); + if (len <= 22) { + snprintf(out, 24, "%s", full); + return; + } + snprintf(out, 24, "%.19s...", full); +} + +/* Compose a "linux/arm64/v8" string from a parsed platform descriptor. The + * variant suffix is omitted when the variant field is empty so a platform + * with no variant prints as "linux/amd64" rather than "linux/amd64/". + */ +static void render_platform(const oci_platform_t *p, char out[64]) +{ + const char *os = p->os && *p->os ? p->os : "?"; + const char *arch = + p->architecture && *p->architecture ? p->architecture : "?"; + if (p->variant && *p->variant) { + snprintf(out, 64, "%s/%s/%s", os, arch, p->variant); + } else { + snprintf(out, 64, "%s/%s", os, arch); + } +} + +/* Open /blobs// and slurp the contents into a fresh + * heap buffer. NUL-terminates the buffer so the slice 3 parsers (which accept + * exact-length bytes) can also be fed as C strings if a caller wants. On + * miss returns -1 with errno=ENOENT; on read failure returns -1 with errno + * preserved or set to EIO. + */ +static int read_blob_file(oci_blob_store_t *blobs, + oci_digest_algo_t algo, + const char *hex, + char **out_body, + size_t *out_len) +{ + char path[4096]; + int n = oci_blob_store_path(blobs, algo, hex, path, sizeof(path)); + if (n < 0 || (size_t) n >= sizeof(path)) { + errno = ENAMETOOLONG; + return -1; + } + int fd = open(path, O_RDONLY); + if (fd < 0) + return -1; + struct stat st; + if (fstat(fd, &st) < 0) { + int saved = errno; + close(fd); + errno = saved; + return -1; + } + if (st.st_size < 0 || (uintmax_t) st.st_size > INSPECT_BODY_MAX) { + close(fd); + errno = EFBIG; + return -1; + } + size_t want = (size_t) st.st_size; + char *buf = malloc(want + 1); + if (!buf) { + close(fd); + errno = ENOMEM; + return -1; + } + size_t off = 0; + while (off < want) { + ssize_t r = read(fd, buf + off, want - off); + if (r < 0) { + int saved = errno; + free(buf); + close(fd); + errno = saved; + return -1; + } + if (r == 0) + break; + off += (size_t) r; + } + close(fd); + if (off != want) { + free(buf); + errno = EIO; + return -1; + } + buf[want] = '\0'; + *out_body = buf; + *out_len = want; + return 0; +} + +/* Emit one entry of a JSON-style string array with backslash and double-quote + * escaping. Control characters pass through verbatim: image-config Entrypoint + * and Cmd entries are container argv strings, which in practice never carry + * raw control bytes, and a partial JSON escape table would mislead a reader + * who expects strict RFC 8259 conformance. Callers downstream of inspect + * (commit 2 onward) reparse the array via the cJSON-backed image-config + * loader, not by scanning the human-readable inspect output. + */ +static void print_quoted_token(FILE *out, const char *s) +{ + fputc('"', out); + for (const char *p = s; *p; p++) { + if (*p == '"' || *p == '\\') + fputc('\\', out); + fputc(*p, out); + } + fputc('"', out); +} + +/* Render a NULL-terminated string array as ["a", "b", "c"]. Empty array + * (arr[0] == NULL) renders as []. Caller has already pre-checked arr != NULL. + */ +static void print_json_string_array(FILE *out, char *const *arr) +{ + fputc('[', out); + for (size_t i = 0; arr[i] != NULL; i++) { + if (i > 0) + fputs(", ", out); + print_quoted_token(out, arr[i]); + } + fputc(']', out); +} + +/* Render the image-config runtime block (User, WorkingDir, Entrypoint, Cmd, + * Env). Absent fields (NULL pointer in the parsed model) skip the bullet + * entirely; empty arrays still print "[]" because that is the + * spec-defined "explicit empty" shape and silently hiding it would + * misrepresent the image. Label column width is 13 so values align with the + * preceding " config: " column from render_manifest. + * + * Multi-line Env: the first var sits on the "env:" line; remaining vars + * indent to the value column on continuation lines. Output stays grep-friendly + * (each VAR=value on its own line) without sacrificing the leading section + * header. + */ +static void render_runtime(FILE *out, const oci_image_runtime_t *rt) +{ + fprintf(out, "runtime:\n"); + if (rt->user) + fprintf(out, " user: %s\n", rt->user); + if (rt->working_dir) + fprintf(out, " workingdir: %s\n", rt->working_dir); + if (rt->entrypoint) { + fprintf(out, " entrypoint: "); + print_json_string_array(out, rt->entrypoint); + fputc('\n', out); + } + if (rt->cmd) { + fprintf(out, " cmd: "); + print_json_string_array(out, rt->cmd); + fputc('\n', out); + } + if (rt->env) { + if (rt->env[0] == NULL) { + fprintf(out, " env: []\n"); + } else { + for (size_t i = 0; rt->env[i] != NULL; i++) { + fprintf(out, "%s%s\n", + i == 0 ? " env: " : " ", + rt->env[i]); + } + } + } +} + +/* Best-effort read+parse of the image-config blob referenced by a manifest's + * config descriptor; on success, emits the runtime block. Failure (blob + * missing, parse rejects the body) is silent: the surrounding inspect output + * already names the config digest in the layer table, so a reader can chase + * it via 'elfuse oci pull' or by inspecting the store directly. Inspect's + * primary contract is to render the manifest tree, not to fail when a + * pulled image is missing the auxiliary config blob. + */ +static void try_render_runtime(FILE *out, + oci_blob_store_t *blobs, + const oci_descriptor_t *config_desc) +{ + char *body = NULL; + size_t body_len = 0; + if (read_blob_file(blobs, config_desc->algo, config_desc->hex, &body, + &body_len) < 0) + return; + oci_image_config_t cfg = {0}; + if (oci_image_config_parse(body, body_len, &cfg, NULL) == 0) { + render_runtime(out, &cfg.config); + oci_image_config_free(&cfg); + } + free(body); +} + +/* Print the config + layer table for a parsed manifest. When manifest_digest + * is non-NULL, a "manifest: ()" header line goes + * first; the direct-manifest path passes NULL so it does not duplicate the + * already-printed pin line. After the layer table, attempt to render the + * image-config runtime block (User, Env, Entrypoint, Cmd, WorkingDir) when + * the config blob can be loaded; absent/unreadable config blobs leave the + * runtime section out, since the manifest tree itself is the primary signal. + */ +static void render_manifest(FILE *out, + oci_blob_store_t *blobs, + const oci_manifest_t *mf, + const char *manifest_digest) +{ + if (manifest_digest) { + const char *mt = oci_media_type_name(mf->media_type); + fprintf(out, "manifest: %s (%s)\n", manifest_digest, + mt ? mt : "unknown"); + } + char buf[24]; + short_digest(mf->config.digest_str, buf); + const char *config_mt = oci_media_type_name(mf->config.media_type); + fprintf(out, " config: %-22s %12" PRId64 "B %s\n", buf, mf->config.size, + config_mt ? config_mt : "unknown"); + fprintf(out, " layers:\n"); + for (size_t i = 0; i < mf->nlayers; i++) { + const oci_descriptor_t *l = &mf->layers[i]; + short_digest(l->digest_str, buf); + const char *lmt = oci_media_type_name(l->media_type); + fprintf(out, " [%zu] %-22s %12" PRId64 "B %s\n", i, buf, + l->size, lmt ? lmt : "unknown"); + } + try_render_runtime(out, blobs, &mf->config); +} + +/* Render the C3.4 "layer reuse:" section. Compares the target manifest's + * diff_id list and ChainID chain against every other image recorded in the + * store (pins plus, when volume_root is set, unpacked sysroots) and prints + * a two-line summary: + * + * layer reuse: + * raw cache: N/M layers shared with K other image(s)[, X on cache] + * stack cache: deepest shared prefix reaches layer P/M (sha256:...) + * + * Failure modes are intentionally soft: a missing or malformed image-config + * for the target prints "layer reuse: (image-config unavailable)" without + * disturbing the surrounding manifest tree output. An empty store (no other + * images to compare against) prints "(no other images to compare)" so the + * operator can tell "0 shared because nothing to share with" apart from + * "0 shared because nothing overlaps". + * + * Bytes formatting: values >= 1 MiB render as "~X.Y MiB on cache"; smaller + * non-zero values render in bytes; zero bytes are omitted (still print the + * layer count, just without a bytes clause) because a 0 B clause would imply + * the raw cache is populated when it isn't. + */ +static void render_layer_reuse(FILE *out, + oci_store_t *store, + const char *manifest_digest, + const char *volume_root) +{ + oci_dedup_metrics_t m = {0}; + const char *err = NULL; + if (oci_dedup_metrics_compute(store, manifest_digest, volume_root, &m, + &err) < 0) { + fprintf(out, "layer reuse: (image-config unavailable)\n"); + return; + } + if (m.compared_images == 0) { + fprintf(out, "layer reuse: (no other images to compare)\n"); + return; + } + fprintf(out, "layer reuse:\n"); + fprintf(out, " raw cache: %zu/%zu layers shared with %zu other image(s)", + m.shared_layers, m.total_layers, m.compared_images); + if (m.shared_bytes >= (uint64_t) 1024 * 1024) { + double mib = (double) m.shared_bytes / (1024.0 * 1024.0); + fprintf(out, ", ~%.1f MiB on cache", mib); + } else if (m.shared_bytes > 0) { + fprintf(out, ", %" PRIu64 " B on cache", m.shared_bytes); + } + fputc('\n', out); + if (m.deepest_shared_prefix > 0) { + char short_chain[24]; + short_digest(m.deepest_shared_chainid, short_chain); + fprintf(out, + " stack cache: deepest shared prefix reaches layer %zu/%zu" + " (%s)\n", + m.deepest_shared_prefix, m.total_layers, short_chain); + } else { + fprintf(out, " stack cache: no shared prefix\n"); + } +} + +/* Render the index entry table. Default mode prints only the picked + * linux/arm64 entry (with a "[arm64]" tag); --all-platforms prints every + * entry, tagging the picked one so users still see which one elfuse will + * resolve. + */ +static void render_index_platforms(FILE *out, + const oci_index_t *idx, + const oci_index_entry_t *picked, + bool show_all) +{ + fprintf(out, "platforms:\n"); + for (size_t i = 0; i < idx->nentries; i++) { + const oci_index_entry_t *e = &idx->entries[i]; + bool is_picked = (e == picked); + if (!show_all && !is_picked) + continue; + char digest_buf[24]; + short_digest(e->desc.digest_str, digest_buf); + char platform_buf[64]; + render_platform(&e->platform, platform_buf); + const char *mt = oci_media_type_name(e->desc.media_type); + fprintf(out, " %-9s %-22s %-22s %12" PRId64 "B %s\n", + is_picked ? "[arm64]" : "", platform_buf, digest_buf, + e->desc.size, mt ? mt : "unknown"); + } + fprintf(out, "\n"); +} + +int oci_inspect(oci_store_t *store, + const oci_ref_t *ref, + const oci_inspect_options_t *opts, + const char **err_msg) +{ + if (!store || !ref || !ref->registry || !ref->repository) { + if (err_msg) + *err_msg = "invalid arguments"; + errno = EINVAL; + return -1; + } + FILE *out = opts && opts->out ? opts->out : stdout; + bool show_all = opts && opts->show_all_platforms; + + /* 1. Resolve manifest digest from ref. */ + char *pinned = NULL; + bool from_pin = false; + if (ref->digest) { + pinned = strdup(ref->digest); + if (!pinned) { + errno = ENOMEM; + if (err_msg) + *err_msg = "out of memory"; + return -1; + } + } else if (ref->tag) { + const char *get_err = NULL; + int gr = oci_store_get_ref(store, ref, &pinned, &get_err); + if (gr < 0) { + if (errno == ENOENT) { + fprintf(out, + "pinned: (no local manifest; run 'elfuse oci " + "pull' first)\n"); + return 0; + } + if (err_msg) + *err_msg = get_err ? get_err : "failed to read pin"; + return -1; + } + from_pin = true; + } else { + /* The slice 1 ref parser defaults tag to "latest" when no digest is + * given, so this branch is structurally unreachable through the CLI. + * Guard it anyway so a hand-constructed ref does not segfault. + */ + if (err_msg) + *err_msg = "ref has neither tag nor digest"; + errno = EINVAL; + return -1; + } + + /* 2. Print the pin line. The digest reference annotation tells the user + * this came from ref->digest rather than the local pin file. + */ + if (from_pin) { + fprintf(out, "pinned: %s\n", pinned); + } else { + fprintf(out, "pinned: %s (digest reference)\n", pinned); + } + + /* 3. Validate the digest and read the blob. */ + oci_digest_algo_t algo; + char hex[OCI_DIGEST_HEX_MAX + 1]; + if (!oci_digest_parse(pinned, &algo, hex)) { + if (err_msg) + *err_msg = "pinned digest is malformed"; + errno = EINVAL; + free(pinned); + return -1; + } + + char *body = NULL; + size_t body_len = 0; + if (read_blob_file(oci_store_blobs(store), algo, hex, &body, &body_len) < + 0) { + if (errno == ENOENT) { + fprintf(out, "error: manifest blob %s not found in local store\n", + pinned); + if (err_msg) + *err_msg = "manifest blob missing from local store"; + free(pinned); + errno = ENOENT; + return -1; + } + int saved = errno; + if (err_msg) + *err_msg = "failed to read manifest blob"; + free(pinned); + errno = saved; + return -1; + } + + /* 4. Classify: try index first, then manifest. The two parsers reject + * disjoint shapes (one requires "manifests", the other requires "config" + * + "layers"), so a successful parse is unambiguous. + */ + oci_index_t idx = {0}; + oci_manifest_t mf = {0}; + bool is_index = false; + bool is_manifest = false; + if (oci_index_parse(body, body_len, &idx, NULL) == 0) { + is_index = true; + } else if (oci_manifest_parse(body, body_len, &mf, NULL) == 0) { + is_manifest = true; + } else { + if (err_msg) + *err_msg = "manifest blob is neither a valid index nor manifest"; + errno = EPROTO; + free(body); + free(pinned); + return -1; + } + + /* 5. Render. */ + int rc = 0; + if (is_index) { + const char *imt = oci_media_type_name(idx.media_type); + fprintf(out, "type: image index (%s)\n\n", imt ? imt : "unknown"); + + const oci_index_entry_t *picked = oci_index_pick_linux_arm64(&idx); + render_index_platforms(out, &idx, picked, show_all); + + /* Default mode drills into the picked linux/arm64 sub-manifest. The + * --all-platforms request is "show me the cover", not "drill"; skip + * the sub-manifest read entirely. + */ + if (!show_all) { + if (!picked) { + fprintf(out, "error: index has no linux/arm64 entry\n"); + if (err_msg) + *err_msg = "index has no linux/arm64 entry"; + errno = ENOENT; + rc = -1; + } else { + char *sub_body = NULL; + size_t sub_len = 0; + if (read_blob_file(oci_store_blobs(store), picked->desc.algo, + picked->desc.hex, &sub_body, &sub_len) < 0) { + if (errno == ENOENT) { + fprintf(stderr, + "warning: linux/arm64 manifest blob %s not " + "in local store\n", + picked->desc.digest_str); + if (err_msg) + *err_msg = + "indexed manifest blob missing from local " + "store"; + errno = ENOENT; + rc = -1; + } else { + int saved = errno; + if (err_msg) + *err_msg = "failed to read sub-manifest blob"; + errno = saved; + rc = -1; + } + } else { + oci_manifest_t sub_mf = {0}; + if (oci_manifest_parse(sub_body, sub_len, &sub_mf, NULL) == + 0) { + render_manifest(out, oci_store_blobs(store), &sub_mf, + picked->desc.digest_str); + if (!opts || !opts->suppress_layer_reuse) { + render_layer_reuse(out, store, + picked->desc.digest_str, + opts ? opts->volume_root : NULL); + } + oci_manifest_free(&sub_mf); + } else { + fprintf(out, + "error: sub-manifest blob %s is malformed\n", + picked->desc.digest_str); + if (err_msg) + *err_msg = "sub-manifest is malformed"; + errno = EPROTO; + rc = -1; + } + free(sub_body); + } + } + } + } else if (is_manifest) { + const char *mmt = oci_media_type_name(mf.media_type); + fprintf(out, "type: image manifest (%s)\n\n", + mmt ? mmt : "unknown"); + render_manifest(out, oci_store_blobs(store), &mf, NULL); + if (!opts || !opts->suppress_layer_reuse) { + render_layer_reuse(out, store, pinned, + opts ? opts->volume_root : NULL); + } + } + + /* errno preserved across cleanup, like slice 5a oci_pull. */ + int saved_errno = errno; + oci_index_free(&idx); + oci_manifest_free(&mf); + free(body); + free(pinned); + if (rc != 0) + errno = saved_errno; + return rc; +} diff --git a/src/oci/inspect.h b/src/oci/inspect.h new file mode 100644 index 0000000..d282920 --- /dev/null +++ b/src/oci/inspect.h @@ -0,0 +1,74 @@ +/* Offline manifest tree renderer for elfuse oci inspect + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Reads the local store the slice 5a pull pipeline populated and prints the + * resolved manifest graph without touching the network. The function does not + * print the canonical reference header (registry / repository / tag / digest); + * that piece is owned by src/oci/cli.c so the slice-1 inspect smoke output + * stays exactly the same when the store has no record for a ref. + * + * Manifest digest resolution order: + * 1. ref->digest, when set (digest-pinned reference) + * 2. Pin descriptor in /index.json whose ref.name annotation matches + * the canonical "/:" form + * 3. Neither: print "(no local manifest...)" and return 0 (informational) + * + * Render policy: + * - The blob is parsed as an index or a manifest based on the canonical + * mediaType embedded in the JSON. Unknown media types abort with EPROTO. + * - For an image index: prints a platform table. Default mode shows only + * the linux/arm64 entry and then drills into its sub-manifest to print + * the config descriptor and layer table. --all-platforms (opts-> + * show_all_platforms) lists every entry and skips the drill -- it is + * "what platforms does this image cover", not "what is inside the arm64 + * variant". + * - For an image manifest: prints config + layers directly. + * + * Failure mode for partial stores: when the index loads but the linux/arm64 + * sub-manifest blob is missing from the store, the platform table is still + * printed (stdout), a warning lands on stderr, and the call returns -1 with + * errno=ENOENT. That preserves the informational view while letting scripts + * detect the inconsistency through the exit code. + */ + +#pragma once + +#include +#include + +#include "ref.h" +#include "store.h" + +typedef struct { + /* Destination for the rendered tree. NULL defaults to stdout. */ + FILE *out; + /* List every platform entry of an image index instead of only the picked + * linux/arm64 entry. In this mode oci_inspect does not drill into any + * sub-manifest and skips the layer reuse section. + */ + bool show_all_platforms; + /* Optional volume root for the unpacked-sysroot walk in the layer reuse + * section. NULL means pin-only dedup, matching the C1.2 GC walker + * convention. Pure information: dedup metrics never write to disk. + */ + const char *volume_root; + /* When true, suppress the "layer reuse:" section that is otherwise + * rendered after the manifest layer table. The default (false, which is + * also the NULL-opts case) renders the section; tests that only want to + * verify the renderer baseline without the dedup compute side-effects set + * this true to skip it. The CLI never sets this true. + */ + bool suppress_layer_reuse; +} oci_inspect_options_t; + +/* Render the manifest tree the store holds for ref. opts may be NULL for the + * defaults (out=stdout, show_all_platforms=false). Returns 0 on success or + * pin miss; -1 with errno preserved and *err_msg (when non-NULL) pointing at + * a static description on failure (malformed blob, blob missing, IO error). + */ +int oci_inspect(oci_store_t *store, + const oci_ref_t *ref, + const oci_inspect_options_t *opts, + const char **err_msg); diff --git a/src/oci/layer-apply.c b/src/oci/layer-apply.c new file mode 100644 index 0000000..7fc66eb --- /dev/null +++ b/src/oci/layer-apply.c @@ -0,0 +1,616 @@ +/* OCI layer applier implementation + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "oci/layer-apply.h" + +#define LA_PATH_MAX 4096 +#define LA_CHUNK 65536 + +/* strchrnul is gated behind macOS 15.4 deployment target; emulate it + * inline so the applier builds against older SDKs without an + * availability check. + */ +static const char *la_strchrnul(const char *s, int c) +{ + const char *p = strchr(s, c); + return p ? p : s + strlen(s); +} + +static int set_err(const char **err, const char *msg, int err_no) +{ + if (err) + *err = msg; + errno = err_no; + return -1; +} + +static const char *basename_of(const char *p) +{ + const char *slash = strrchr(p, '/'); + return slash ? slash + 1 : p; +} + +/* Normalize the parent prefix of guest_path into parent_out (without + * trailing slash). parent_out may be the same buffer as guest_path's + * substring source, but for safety the caller passes a fresh buffer. + */ +static void parent_of(const char *guest_path, char *parent_out, size_t cap) +{ + const char *slash = strrchr(guest_path, '/'); + if (!slash) { + parent_out[0] = '\0'; + return; + } + size_t n = (size_t) (slash - guest_path); + if (n >= cap) + n = cap - 1; + memcpy(parent_out, guest_path, n); + parent_out[n] = '\0'; +} + +int oci_path_join_safe(const char *root_dir, + const char *guest_path, + char *out, + size_t cap, + const char **err) +{ + if (!root_dir || !guest_path || !out || cap == 0) + return set_err(err, "path join: NULL argument", EINVAL); + if (guest_path[0] == '/') + return set_err(err, "path join: absolute guest path", EINVAL); + if (guest_path[0] == '\0' || strcmp(guest_path, ".") == 0) + return set_err(err, "path join: empty path", EINVAL); + + /* Walk segments and reject `..` outright. Mirrors the no-follow + * basename rule from src/syscall/path.h::path_translate_at. + */ + const char *p = guest_path; + while (*p) { + const char *next = la_strchrnul(p, '/'); + size_t seglen = (size_t) (next - p); + if (seglen == 2 && p[0] == '.' && p[1] == '.') + return set_err(err, "path join: segment is ..", EINVAL); + p = *next ? next + 1 : next; + } + + size_t rl = strlen(root_dir); + size_t gl = strlen(guest_path); + if (rl + 1 + gl + 1 > cap) + return set_err(err, "path join: assembled length overflow", + ENAMETOOLONG); + memcpy(out, root_dir, rl); + out[rl] = '/'; + memcpy(out + rl + 1, guest_path, gl + 1); + return 0; +} + +int oci_symlink_target_check(const char *link_dir, const char *target) +{ + if (!target) + return set_err(NULL, NULL, EINVAL); + + /* Compose the conceptual destination relative to the unpack root. + * Absolute targets treat '/' as the unpack root (the symlink is + * unpacked into the layer sysroot, so absolute means root-relative); + * relative targets start from link_dir. + */ + char buf[LA_PATH_MAX]; + if (target[0] == '/') { + if (strlcpy(buf, target + 1, sizeof(buf)) >= sizeof(buf)) + return set_err(NULL, NULL, ENAMETOOLONG); + } else { + if (link_dir && link_dir[0]) { + if ((size_t) snprintf(buf, sizeof(buf), "%s/%s", link_dir, + target) >= sizeof(buf)) + return set_err(NULL, NULL, ENAMETOOLONG); + } else { + if (strlcpy(buf, target, sizeof(buf)) >= sizeof(buf)) + return set_err(NULL, NULL, ENAMETOOLONG); + } + } + + /* Track depth as we walk segments. `..` decrements; `.` and empty + * stay. Any drop below zero means a follower would step above the + * unpack root, which is rejected. + */ + int depth = 0; + char *save = NULL; + char *seg = strtok_r(buf, "/", &save); + while (seg) { + if (strcmp(seg, "..") == 0) { + if (--depth < 0) + return set_err(NULL, NULL, ELOOP); + } else if (strcmp(seg, ".") != 0 && seg[0] != '\0') { + depth++; + } + seg = strtok_r(NULL, "/", &save); + } + return 0; +} + +/* Recursive rm-rf rooted at path. Used by whiteout and opaque dir. */ +static int rm_recursive(const char *path) +{ + struct stat st; + if (lstat(path, &st) < 0) { + if (errno == ENOENT) + return 0; + return -1; + } + if (!S_ISDIR(st.st_mode)) { + return unlink(path); + } + DIR *d = opendir(path); + if (!d) + return -1; + struct dirent *de; + int rc = 0; + while ((de = readdir(d))) { + if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) + continue; + char child[LA_PATH_MAX]; + if ((size_t) snprintf(child, sizeof(child), "%s/%s", path, + de->d_name) >= sizeof(child)) { + rc = -1; + errno = ENAMETOOLONG; + break; + } + if (rm_recursive(child) < 0) { + rc = -1; + break; + } + } + closedir(d); + if (rc == 0) + rc = rmdir(path); + return rc; +} + +/* mkdir -p for the directory containing 'host_path'. Mode 0755 for + * implicit parents; the entry's own mode is applied later when the + * tar provides it. + */ +static int mkdir_parents(const char *host_path, const char **err) +{ + char buf[LA_PATH_MAX]; + if (strlcpy(buf, host_path, sizeof(buf)) >= sizeof(buf)) + return set_err(err, "mkdir parents: path overflow", ENAMETOOLONG); + char *slash = strrchr(buf, '/'); + if (!slash || slash == buf) + return 0; + *slash = '\0'; + /* Walk components and mkdir each. */ + for (char *p = buf + 1; *p; p++) { + if (*p != '/') + continue; + *p = '\0'; + if (mkdir(buf, 0755) < 0 && errno != EEXIST) + return set_err(err, "mkdir parents: mkdir failed", errno); + *p = '/'; + } + if (mkdir(buf, 0755) < 0 && errno != EEXIST) + return set_err(err, "mkdir parents: mkdir failed", errno); + return 0; +} + +static int copy_payload_to_fd(oci_tar_reader_t *r, + int fd, + uint64_t want, + const char **err) +{ + uint8_t buf[LA_CHUNK]; + while (want > 0) { + size_t got = 0; + size_t take = want > sizeof(buf) ? sizeof(buf) : (size_t) want; + const char *terr = NULL; + if (oci_tar_read_payload(r, buf, take, &got, &terr) < 0) + return set_err(err, terr ? terr : "tar payload read failed", EIO); + if (got == 0) + return set_err(err, "tar payload truncated", EIO); + const uint8_t *p = buf; + size_t remaining = got; + while (remaining > 0) { + ssize_t n = write(fd, p, remaining); + if (n < 0) { + if (errno == EINTR) + continue; + return set_err(err, "layer apply: file write failed", errno); + } + p += n; + remaining -= (size_t) n; + } + want -= got; + } + return 0; +} + +static int apply_regular(oci_tar_reader_t *r, + const oci_tar_entry_t *e, + const char *host_path, + oci_layer_apply_stats_t *stats, + const char **err) +{ + if (mkdir_parents(host_path, err) < 0) + return -1; + /* Remove any existing entry first so we never accidentally write + * through a symlink left by a lower layer. + */ + if (unlink(host_path) < 0 && errno != ENOENT) { + if (errno == EISDIR && rm_recursive(host_path) < 0) + return set_err(err, "layer apply: cannot remove existing dir", + errno); + else if (errno != EISDIR) + return set_err(err, "layer apply: cannot remove existing entry", + errno); + } + int fd = open(host_path, O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, 0644); + if (fd < 0) + return set_err(err, "layer apply: open file failed", errno); + if (copy_payload_to_fd(r, fd, e->size, err) < 0) { + close(fd); + unlink(host_path); + return -1; + } + /* fchmod to the requested mode bits (low 12). The host inode may + * silently drop bits the running user cannot set, but the sidecar + * carries the authoritative value regardless. + */ + if (fchmod(fd, (mode_t) (e->mode & 07777)) < 0 && errno != EPERM) + return set_err(err, "layer apply: fchmod failed", errno); + close(fd); + if (stats) + stats->files++; + return 0; +} + +static int apply_dir(const oci_tar_entry_t *e, + const char *host_path, + oci_layer_apply_stats_t *stats, + const char **err) +{ + if (mkdir_parents(host_path, err) < 0) + return -1; + if (mkdir(host_path, 0755) < 0 && errno != EEXIST) + return set_err(err, "layer apply: mkdir failed", errno); + /* Apply mode bits; reuse fchmod via opening the dir so trailing + * symlinks are not followed (open with O_NOFOLLOW + O_DIRECTORY). + */ + int fd = open(host_path, O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); + if (fd >= 0) { + (void) fchmod(fd, (mode_t) (e->mode & 07777)); + close(fd); + } + if (stats) + stats->dirs++; + return 0; +} + +static int apply_symlink(const oci_tar_entry_t *e, + const char *host_path, + const char *guest_path, + oci_layer_apply_stats_t *stats, + const char **err) +{ + if (!e->linkname || !e->linkname[0]) + return set_err(err, "layer apply: empty symlink target", EINVAL); + + char parent[LA_PATH_MAX]; + parent_of(guest_path, parent, sizeof(parent)); + if (oci_symlink_target_check(parent, e->linkname) < 0) + return set_err(err, "layer apply: symlink target escapes root", ELOOP); + + if (mkdir_parents(host_path, err) < 0) + return -1; + if (unlink(host_path) < 0 && errno != ENOENT) { + if (errno == EISDIR && rm_recursive(host_path) < 0) + return set_err(err, "layer apply: cannot remove existing dir", + errno); + else if (errno != EISDIR) + return set_err(err, "layer apply: cannot remove existing entry", + errno); + } + if (symlink(e->linkname, host_path) < 0) + return set_err(err, "layer apply: symlink failed", errno); + if (stats) + stats->symlinks++; + return 0; +} + +static int apply_hardlink(const oci_tar_entry_t *e, + const char *root_dir, + const char *host_path, + oci_layer_apply_stats_t *stats, + const char **err) +{ + if (!e->linkname || !e->linkname[0]) + return set_err(err, "layer apply: empty hardlink target", EINVAL); + + char target_host[LA_PATH_MAX]; + /* The hardlink target is an intra-archive guest path. Validate it + * the same way as a regular entry's path. + */ + if (oci_path_join_safe(root_dir, e->linkname, target_host, + sizeof(target_host), err) < 0) + return -1; + /* The target must exist already; OCI mandates entries in apply + * order, and a hardlink that names a missing file is a malformed + * archive (or a forward reference, which we explicitly reject). + */ + struct stat st; + if (lstat(target_host, &st) < 0) + return set_err(err, "layer apply: hardlink target missing", ENOLINK); + + if (mkdir_parents(host_path, err) < 0) + return -1; + if (unlink(host_path) < 0 && errno != ENOENT) + return set_err(err, "layer apply: cannot remove existing entry", errno); + if (link(target_host, host_path) < 0) + return set_err(err, "layer apply: link failed", errno); + if (stats) + stats->hardlinks++; + return 0; +} + +static int apply_whiteout(const oci_tar_entry_t *e, + const char *root_dir, + oci_meta_table_t *meta, + oci_layer_apply_stats_t *stats, + const char **err) +{ + /* Path is "...dir/.wh."; the entry being whited out is + * "...dir/". + */ + const char *base = basename_of(e->path); + if (strncmp(base, ".wh.", 4) != 0 || base[4] == '\0') + return set_err(err, "layer apply: malformed whiteout entry", EINVAL); + + size_t parent_len = (size_t) (base - e->path); + char target[LA_PATH_MAX]; + if (parent_len + strlen(base + 4) + 1 > sizeof(target)) + return set_err(err, "layer apply: whiteout path overflow", + ENAMETOOLONG); + memcpy(target, e->path, parent_len); + snprintf(target + parent_len, sizeof(target) - parent_len, "%s", base + 4); + + char host_path[LA_PATH_MAX]; + if (oci_path_join_safe(root_dir, target, host_path, sizeof(host_path), + err) < 0) + return -1; + if (rm_recursive(host_path) < 0 && errno != ENOENT) + return set_err(err, "layer apply: whiteout removal failed", errno); + if (meta) + oci_meta_remove(meta, target); + if (stats) + stats->whiteouts++; + return 0; +} + +static int apply_opaque_whiteout(const oci_tar_entry_t *e, + const char *root_dir, + oci_meta_table_t *meta, + oci_layer_apply_stats_t *stats, + const char **err) +{ + /* Path is "...dir/.wh..wh..opq"; clear all CHILDREN of "...dir/" + * but leave the directory itself. + */ + const char *base = basename_of(e->path); + if (strcmp(base, ".wh..wh..opq") != 0) + return set_err(err, "layer apply: malformed opaque marker", EINVAL); + size_t parent_len = (size_t) (base - e->path); + char dir[LA_PATH_MAX]; + if (parent_len == 0) { + /* Opaque at layer root: clear top-level lower-layer entries. */ + dir[0] = '\0'; + } else { + /* Drop the trailing slash before the marker. */ + size_t copy = parent_len - 1; + if (copy + 1 > sizeof(dir)) + return set_err(err, "layer apply: opaque path overflow", + ENAMETOOLONG); + memcpy(dir, e->path, copy); + dir[copy] = '\0'; + } + + char host_dir[LA_PATH_MAX]; + if (dir[0] == '\0') { + if ((size_t) snprintf(host_dir, sizeof(host_dir), "%s", root_dir) >= + sizeof(host_dir)) + return set_err(err, "layer apply: opaque host path overflow", + ENAMETOOLONG); + } else { + if (oci_path_join_safe(root_dir, dir, host_dir, sizeof(host_dir), err) < + 0) + return -1; + } + + DIR *d = opendir(host_dir); + if (!d) { + if (errno == ENOENT) { + if (stats) + stats->opaques++; + return 0; + } + return set_err(err, "layer apply: opaque opendir failed", errno); + } + struct dirent *de; + int rc = 0; + while ((de = readdir(d))) { + if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) + continue; + if (strncmp(de->d_name, ".wh.", 4) == 0) + continue; /* let the marker entry itself stay for the iter */ + char child_host[LA_PATH_MAX]; + if ((size_t) snprintf(child_host, sizeof(child_host), "%s/%s", host_dir, + de->d_name) >= sizeof(child_host)) { + rc = -1; + errno = ENAMETOOLONG; + break; + } + if (rm_recursive(child_host) < 0) { + rc = -1; + break; + } + if (meta) { + char child_guest[LA_PATH_MAX]; + if (dir[0]) + snprintf(child_guest, sizeof(child_guest), "%s/%s", dir, + de->d_name); + else + snprintf(child_guest, sizeof(child_guest), "%s", de->d_name); + oci_meta_remove(meta, child_guest); + } + } + closedir(d); + if (rc < 0) + return set_err(err, "layer apply: opaque clear failed", errno); + if (stats) + stats->opaques++; + return 0; +} + +/* Whiteout-handling discipline shared by oci_layer_apply (overlay) and + * oci_layer_apply_raw_tar (Plan 3 C3.3 raw per-layer cache populate). + * Overlay mode interprets .wh. and .wh..wh..opq tar entries as + * delete / clear directives against root_dir; raw mode leaves them as + * regular 0-byte files at their tar path so the assembler can replay + * the whiteout intent against the running work_dir later. + */ +typedef enum { + APPLY_MODE_OVERLAY, + APPLY_MODE_RAW_TAR, +} apply_mode_t; + +static int layer_apply_impl(oci_tar_reader_t *r, + const char *root_dir, + apply_mode_t mode, + oci_layer_apply_stats_t *stats, + oci_meta_table_t *meta, + const char **err) +{ + static const char *dummy_err; + if (!err) + err = &dummy_err; + *err = NULL; + if (!r || !root_dir) + return set_err(err, "layer apply: NULL argument", EINVAL); + + for (;;) { + oci_tar_entry_t e; + const char *terr = NULL; + int rc = oci_tar_next(r, &e, &terr); + if (rc < 0) + return set_err(err, terr ? terr : "tar next failed", + errno ? errno : EIO); + if (rc == 0) + return 0; + + /* Strip a leading slash if present; OCI layer paths are + * relative, but some encoders ship them with a leading slash. + */ + const char *gp = e.path; + while (gp[0] == '/') + gp++; + + /* Root-directory tar entry: docker/buildkit emit "./" as the + * first entry of a layer; the DIR-type trailing-slash strip + * upstream collapses it to ".". The unpack root already + * exists by the time the assembler enters this loop, so the + * root entry has no work to drive. Skip empty paths the same + * way for archives that record a zero-length root name. + */ + if (gp[0] == '\0' || (gp[0] == '.' && gp[1] == '\0')) + continue; + + if (mode == APPLY_MODE_OVERLAY) { + if (e.is_opaque_whiteout) { + oci_tar_entry_t e2 = e; + e2.path = gp; + if (apply_opaque_whiteout(&e2, root_dir, meta, stats, err) < 0) + return -1; + continue; + } + if (e.is_whiteout) { + oci_tar_entry_t e2 = e; + e2.path = gp; + if (apply_whiteout(&e2, root_dir, meta, stats, err) < 0) + return -1; + continue; + } + } + /* Raw-tar mode falls through: .wh. and .wh..wh..opq are + * typeflag '0' regular tar entries with zero payload, and the + * regular-file dispatch below writes them on disk as 0-byte + * files at their tar path. The assembler consumes the markers + * later. + */ + + if (e.type == OCI_TAR_UNSUPPORTED) + return set_err(err, "layer apply: unsupported entry type", ENOTSUP); + + char host_path[LA_PATH_MAX]; + if (oci_path_join_safe(root_dir, gp, host_path, sizeof(host_path), + err) < 0) + return -1; + + oci_tar_entry_t e2 = e; + e2.path = gp; + switch (e.type) { + case OCI_TAR_REG: + if (apply_regular(r, &e2, host_path, stats, err) < 0) + return -1; + break; + case OCI_TAR_DIR: + if (apply_dir(&e2, host_path, stats, err) < 0) + return -1; + break; + case OCI_TAR_SYMLINK: + if (apply_symlink(&e2, host_path, gp, stats, err) < 0) + return -1; + break; + case OCI_TAR_HARDLINK: + if (apply_hardlink(&e2, root_dir, host_path, stats, err) < 0) + return -1; + break; + case OCI_TAR_UNSUPPORTED: + /* Already handled above; here for switch completeness. */ + return set_err(err, "layer apply: unsupported entry type", ENOTSUP); + } + + if (meta && e.type != OCI_TAR_HARDLINK) + (void) oci_meta_record(meta, gp, e.uid, e.gid, e.mode); + } +} + +int oci_layer_apply(oci_tar_reader_t *r, + const char *root_dir, + oci_layer_apply_stats_t *stats, + oci_meta_table_t *meta, + const char **err) +{ + return layer_apply_impl(r, root_dir, APPLY_MODE_OVERLAY, stats, meta, err); +} + +int oci_layer_apply_raw_tar(oci_tar_reader_t *r, + const char *root_dir, + oci_layer_apply_stats_t *stats, + oci_meta_table_t *meta, + const char **err) +{ + return layer_apply_impl(r, root_dir, APPLY_MODE_RAW_TAR, stats, meta, err); +} diff --git a/src/oci/layer-apply.h b/src/oci/layer-apply.h new file mode 100644 index 0000000..6ebcf1a --- /dev/null +++ b/src/oci/layer-apply.h @@ -0,0 +1,131 @@ +/* OCI layer applier: drive a tar stream into an unpack root + * + * Consumes oci_tar_entry_t records from oci_tar_reader_t (fed by the + * decompression dispatch) and applies them under root_dir. Walks + * entries in strict order so whiteouts and opaque markers interact + * with prior upper-layer state correctly. + * + * Path containment rules mirror src/syscall/path.h::path_translate_at + * from PR #33: reject `..` traversal in any path component, reject + * absolute components past the layer root, and reject symlinks whose + * resolved target would escape the unpack root. The unpack-time check + * uses oci_path_join_safe + oci_symlink_target_check so a malicious + * tar that smuggles a relative `..` through the prefix slot cannot + * touch host paths. + * + * Whiteouts: + * .wh. remove the upper-layer entry from this dir + * .wh..wh..opq clear the containing dir's lower-layer contents + * + * Mode bits and uid/gid go into the running oci_meta_table_t; the host + * inode is created with the running user's identity, and the sidecar + * lookup at runtime restores the Linux view. Block, char, fifo, and + * socket entries are rejected with ENOTSUP, matching the asymmetric + * subset in oci-roadmap.md Q3. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include + +#include "oci/layer-meta.h" +#include "oci/tar.h" + +typedef struct { + size_t files; + size_t dirs; + size_t symlinks; + size_t hardlinks; + size_t whiteouts; + size_t opaques; +} oci_layer_apply_stats_t; + +/* Apply every entry from r into root_dir. root_dir must already exist + * and be writable; the applier creates parent directories on demand. + * stats and meta may be NULL; passing both lets the caller drive a + * dry-run for fixture validation. + * + * Whiteout entries (.wh.) and opaque markers (.wh..wh..opq) are + * processed against root_dir: .wh. rm-rfs the named upper-layer + * entry, and .wh..wh..opq clears the marker's directory of lower-layer + * contents. This is the canonical overlay-merge behaviour used by the + * top-level oci_unpack assembly path. For populating a Plan 3 C3.3 raw + * per-layer cache where whiteout markers must remain on disk as files, + * use oci_layer_apply_raw_tar instead. + * + * Returns 0 on success, -1 on error with *err set and errno carrying + * one of: + * ENOTSUP - tar entry type rejected (block/char/fifo/socket) + * ELOOP - symlink target escapes the unpack root + * ENOLINK - hardlink target was not seen earlier in the same + * layer or as a previously-unpacked file under root + * ENAMETOOLONG - assembled host path overflows PATH_MAX + * EINVAL - tar entry path contains `..` or is absolute + * ENAMETOOLONG / EIO - tar reader / payload write surface + */ +int oci_layer_apply(oci_tar_reader_t *r, + const char *root_dir, + oci_layer_apply_stats_t *stats, + oci_meta_table_t *meta, + const char **err); + +/* Plan 3 C3.3: raw per-layer extract for the cross-image dedup cache. + * + * Identical to oci_layer_apply except .wh. and .wh..wh..opq tar + * entries are NOT interpreted as delete / clear directives: they fall + * through to the regular-file dispatch and land on disk as 0-byte + * regular files at their tar path. The on-disk shape of root_dir then + * preserves the layer's whiteout intent for the later assembly pass + * (the assembler walks the raw directory, applies whiteouts against + * the running work_dir, then copies non-whiteout entries on top). + * + * Consequences vs overlay mode: + * - stats->whiteouts and stats->opaques stay at zero; the markers + * are counted in stats->files because they exist on disk as + * regular zero-length files + * - the per-layer oci_meta_table_t records the markers like any + * other regular file (uid/gid/mode tuples from the tar header) + * - root_dir is treated as a fresh extraction target: there is no + * pre-existing upper-layer state to delete or clear, so the + * whiteout branches would have been no-ops even if executed + * + * Returns 0 on success, -1 on error with the same errno surface as + * oci_layer_apply (the dispatch and path validation logic is shared). + */ +int oci_layer_apply_raw_tar(oci_tar_reader_t *r, + const char *root_dir, + oci_layer_apply_stats_t *stats, + oci_meta_table_t *meta, + const char **err); + +/* Compose root_dir + '/' + guest_path into out. Rejects: + * - leading '/' in guest_path (absolute is not allowed for tar entries) + * - any path component equal to `..` (escape) + * - empty path or `.` (no-op write target) + * - assembled length >= cap + * Exposed for unit tests; the applier consumes it internally. + */ +int oci_path_join_safe(const char *root_dir, + const char *guest_path, + char *out, + size_t cap, + const char **err); + +/* Validate that a symlink at link_dir pointing to target stays under + * the unpack root if a follower started at link_dir. Pure string-level + * analysis: parses absolute vs relative target, normalizes + * `.`/`..`/empty segments, and asserts the running depth never drops + * below zero relative to the unpack root. + * + * link_dir is the symlink's containing directory expressed as a + * sysroot-relative path (without leading `/`), e.g. "etc/links" for a + * symlink at "etc/links/foo". An empty link_dir means the symlink + * lives directly under the unpack root. + * + * Returns 0 when the target is safe, -1 with errno=ELOOP otherwise. + */ +int oci_symlink_target_check(const char *link_dir, const char *target); diff --git a/src/oci/layer-meta.c b/src/oci/layer-meta.c new file mode 100644 index 0000000..9cf2d86 --- /dev/null +++ b/src/oci/layer-meta.c @@ -0,0 +1,448 @@ +/* OCI sidecar metadata implementation + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../externals/cjson/cJSON.h" +#include "oci/layer-meta.h" + +#define OCI_META_FILE ".elfuse-meta.json" +#define OCI_META_VERSION 1 + +typedef struct { + char *path; + uint64_t uid; + uint64_t gid; + uint32_t mode; +} oci_meta_entry_t; + +struct oci_meta_table { + oci_meta_entry_t *entries; + size_t len; + size_t cap; +}; + +oci_meta_table_t *oci_meta_table_new(void) +{ + return calloc(1, sizeof(oci_meta_table_t)); +} + +static void entry_clear(oci_meta_entry_t *e) +{ + free(e->path); + e->path = NULL; +} + +void oci_meta_table_free(oci_meta_table_t *t) +{ + if (!t) + return; + for (size_t i = 0; i < t->len; i++) + entry_clear(&t->entries[i]); + free(t->entries); + free(t); +} + +static int grow_if_needed(oci_meta_table_t *t) +{ + if (t->len < t->cap) + return 0; + size_t nc = t->cap == 0 ? 16 : t->cap * 2; + oci_meta_entry_t *grown = realloc(t->entries, nc * sizeof(*grown)); + if (!grown) + return -1; + t->entries = grown; + t->cap = nc; + return 0; +} + +static ssize_t find_index(const oci_meta_table_t *t, const char *path) +{ + /* Linear scan. OCI layers typically hold a few hundred to a few + * thousand entries; the wall cost is dwarfed by the IO each entry + * already triggers. If profiling later shows this is hot, swap in + * an open-addressing hash keyed by FNV-1a of the path string. + */ + for (size_t i = 0; i < t->len; i++) + if (strcmp(t->entries[i].path, path) == 0) + return (ssize_t) i; + return -1; +} + +int oci_meta_record(oci_meta_table_t *t, + const char *guest_path, + uint64_t uid, + uint64_t gid, + uint32_t mode) +{ + if (!t || !guest_path) { + errno = EINVAL; + return -1; + } + ssize_t idx = find_index(t, guest_path); + if (idx >= 0) { + t->entries[idx].uid = uid; + t->entries[idx].gid = gid; + t->entries[idx].mode = mode; + return 0; + } + if (grow_if_needed(t) < 0) { + errno = ENOMEM; + return -1; + } + char *dup = strdup(guest_path); + if (!dup) { + errno = ENOMEM; + return -1; + } + t->entries[t->len].path = dup; + t->entries[t->len].uid = uid; + t->entries[t->len].gid = gid; + t->entries[t->len].mode = mode; + t->len++; + return 0; +} + +void oci_meta_remove(oci_meta_table_t *t, const char *guest_path) +{ + if (!t || !guest_path) + return; + ssize_t idx = find_index(t, guest_path); + if (idx < 0) + return; + entry_clear(&t->entries[idx]); + /* Move last entry into the freed slot to keep the array compact. */ + if ((size_t) idx != t->len - 1) + t->entries[idx] = t->entries[t->len - 1]; + t->len--; +} + +int oci_meta_lookup(const oci_meta_table_t *t, + const char *guest_path, + uint64_t *out_uid, + uint64_t *out_gid, + uint32_t *out_mode) +{ + if (!t || !guest_path) { + errno = EINVAL; + return -1; + } + ssize_t idx = find_index(t, guest_path); + if (idx < 0) { + errno = ENOENT; + return -1; + } + if (out_uid) + *out_uid = t->entries[idx].uid; + if (out_gid) + *out_gid = t->entries[idx].gid; + if (out_mode) + *out_mode = t->entries[idx].mode; + return 0; +} + +size_t oci_meta_count(const oci_meta_table_t *t) +{ + return t ? t->len : 0; +} + +static char *build_path(const char *root_dir, const char *name, bool tmp) +{ + size_t want = strlen(root_dir) + 1 + strlen(name) + (tmp ? 4 : 0) + 1; + char *p = malloc(want); + if (!p) + return NULL; + snprintf(p, want, "%s/%s%s", root_dir, name, tmp ? ".tmp" : ""); + return p; +} + +static bool valid_basename(const char *filename) +{ + if (!filename || !*filename) + return false; + for (const char *p = filename; *p; p++) + if (*p == '/') + return false; + return true; +} + +int oci_meta_write_named(const oci_meta_table_t *t, + const char *root_dir, + const char *filename, + const char **err) +{ + static const char *dummy_err; + if (!err) + err = &dummy_err; + *err = NULL; + if (!root_dir) { + *err = "meta write: NULL root"; + errno = EINVAL; + return -1; + } + if (!valid_basename(filename)) { + *err = "meta write: filename must be a non-empty basename"; + errno = EINVAL; + return -1; + } + + cJSON *root = cJSON_CreateObject(); + if (!root) { + *err = "meta write: cJSON_CreateObject failed"; + errno = ENOMEM; + return -1; + } + if (!cJSON_AddNumberToObject(root, "version", OCI_META_VERSION)) { + cJSON_Delete(root); + *err = "meta write: version add failed"; + errno = ENOMEM; + return -1; + } + cJSON *arr = cJSON_AddArrayToObject(root, "entries"); + if (!arr) { + cJSON_Delete(root); + *err = "meta write: entries add failed"; + errno = ENOMEM; + return -1; + } + for (size_t i = 0; t && i < t->len; i++) { + cJSON *e = cJSON_CreateObject(); + if (!e) { + cJSON_Delete(root); + *err = "meta write: entry object failed"; + errno = ENOMEM; + return -1; + } + cJSON_AddItemToArray(arr, e); + if (!cJSON_AddStringToObject(e, "p", t->entries[i].path) || + !cJSON_AddNumberToObject(e, "u", (double) t->entries[i].uid) || + !cJSON_AddNumberToObject(e, "g", (double) t->entries[i].gid) || + !cJSON_AddNumberToObject(e, "m", (double) t->entries[i].mode)) { + cJSON_Delete(root); + *err = "meta write: entry field failed"; + errno = ENOMEM; + return -1; + } + } + + char *json = cJSON_PrintUnformatted(root); + cJSON_Delete(root); + if (!json) { + *err = "meta write: cJSON_Print failed"; + errno = ENOMEM; + return -1; + } + size_t jlen = strlen(json); + + char *tmp_path = build_path(root_dir, filename, true); + char *final_path = build_path(root_dir, filename, false); + if (!tmp_path || !final_path) { + free(tmp_path); + free(final_path); + free(json); + *err = "meta write: path allocation failed"; + errno = ENOMEM; + return -1; + } + + int fd = open(tmp_path, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644); + if (fd < 0) { + *err = "meta write: open tmp failed"; + goto fail_paths; + } + if (write(fd, json, jlen) != (ssize_t) jlen) { + *err = "meta write: write failed"; + close(fd); + unlink(tmp_path); + goto fail_paths; + } + if (fsync(fd) < 0) { + *err = "meta write: fsync failed"; + close(fd); + unlink(tmp_path); + goto fail_paths; + } + close(fd); + if (rename(tmp_path, final_path) < 0) { + *err = "meta write: rename failed"; + unlink(tmp_path); + goto fail_paths; + } + + free(tmp_path); + free(final_path); + free(json); + return 0; + +fail_paths: + free(tmp_path); + free(final_path); + free(json); + return -1; +} + +int oci_meta_read_named(const char *root_dir, + const char *filename, + oci_meta_table_t **out, + const char **err) +{ + static const char *dummy_err; + if (!err) + err = &dummy_err; + *err = NULL; + if (!root_dir || !out) { + *err = "meta read: NULL argument"; + errno = EINVAL; + return -1; + } + *out = NULL; + if (!valid_basename(filename)) { + *err = "meta read: filename must be a non-empty basename"; + errno = EINVAL; + return -1; + } + + char *path = build_path(root_dir, filename, false); + if (!path) { + *err = "meta read: path allocation failed"; + errno = ENOMEM; + return -1; + } + int fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) { + *err = "meta read: open failed"; + free(path); + return -1; /* errno already ENOENT or similar */ + } + free(path); + + struct stat st; + if (fstat(fd, &st) < 0) { + close(fd); + *err = "meta read: fstat failed"; + return -1; + } + /* Cap at 64 MiB so a corrupt sidecar cannot drag the host into + * swap; real-world tables are under a few hundred KiB. + */ + if (st.st_size <= 0 || st.st_size > (off_t) (64 * 1024 * 1024)) { + close(fd); + *err = "meta read: file size out of bounds"; + errno = EINVAL; + return -1; + } + char *buf = malloc((size_t) st.st_size + 1); + if (!buf) { + close(fd); + *err = "meta read: buffer allocation failed"; + errno = ENOMEM; + return -1; + } + ssize_t got = read(fd, buf, (size_t) st.st_size); + close(fd); + if (got != st.st_size) { + free(buf); + *err = "meta read: short read"; + errno = EIO; + return -1; + } + buf[got] = '\0'; + + cJSON *root = cJSON_Parse(buf); + free(buf); + if (!root) { + *err = "meta read: malformed JSON"; + errno = EINVAL; + return -1; + } + cJSON *version = cJSON_GetObjectItemCaseSensitive(root, "version"); + if (!cJSON_IsNumber(version) || version->valueint != OCI_META_VERSION) { + cJSON_Delete(root); + *err = "meta read: unsupported version"; + errno = EINVAL; + return -1; + } + cJSON *arr = cJSON_GetObjectItemCaseSensitive(root, "entries"); + if (!cJSON_IsArray(arr)) { + cJSON_Delete(root); + *err = "meta read: missing entries array"; + errno = EINVAL; + return -1; + } + + oci_meta_table_t *table = oci_meta_table_new(); + if (!table) { + cJSON_Delete(root); + *err = "meta read: table allocation failed"; + errno = ENOMEM; + return -1; + } + + cJSON *e; + cJSON_ArrayForEach(e, arr) + { + cJSON *p = cJSON_GetObjectItemCaseSensitive(e, "p"); + cJSON *u = cJSON_GetObjectItemCaseSensitive(e, "u"); + cJSON *g = cJSON_GetObjectItemCaseSensitive(e, "g"); + cJSON *m = cJSON_GetObjectItemCaseSensitive(e, "m"); + if (!cJSON_IsString(p) || !cJSON_IsNumber(u) || !cJSON_IsNumber(g) || + !cJSON_IsNumber(m)) { + oci_meta_table_free(table); + cJSON_Delete(root); + *err = "meta read: entry shape invalid"; + errno = EINVAL; + return -1; + } + if (oci_meta_record(table, p->valuestring, (uint64_t) u->valuedouble, + (uint64_t) g->valuedouble, + (uint32_t) m->valuedouble) < 0) { + oci_meta_table_free(table); + cJSON_Delete(root); + *err = "meta read: record failed"; + return -1; + } + } + + cJSON_Delete(root); + *out = table; + return 0; +} + +int oci_meta_write(const oci_meta_table_t *t, + const char *root_dir, + const char **err) +{ + return oci_meta_write_named(t, root_dir, OCI_META_FILE, err); +} + +int oci_meta_read(const char *root_dir, + oci_meta_table_t **out, + const char **err) +{ + return oci_meta_read_named(root_dir, OCI_META_FILE, out, err); +} + +int oci_meta_merge(oci_meta_table_t *dst, const oci_meta_table_t *src) +{ + if (!dst || !src) { + errno = EINVAL; + return -1; + } + for (size_t i = 0; i < src->len; i++) { + const oci_meta_entry_t *e = &src->entries[i]; + if (oci_meta_record(dst, e->path, e->uid, e->gid, e->mode) < 0) + return -1; + } + return 0; +} diff --git a/src/oci/layer-meta.h b/src/oci/layer-meta.h new file mode 100644 index 0000000..2337288 --- /dev/null +++ b/src/oci/layer-meta.h @@ -0,0 +1,114 @@ +/* OCI sidecar metadata for unpacked layers + * + * elfuse unpacks layers as the invoking macOS user; it cannot chown to + * arbitrary uids/gids, and the host inode mode cannot always carry the + * full set of permission bits Linux expects. The sidecar records the + * authoritative uid/gid/mode per guest path so Phase 3's syscall layer + * can present the guest with the original Linux view. + * + * Serialization format: /.elfuse-meta.json with the shape + * { "version": 1, + * "entries": [ { "p": "/path", "u": NNN, "g": NNN, "m": NNN } ] } + * Mode bits are stored decimal (cJSON has no native octal). Setuid, + * setgid, and sticky bits are encoded in the bottom 12 bits along with + * the rwx triplets, per oci-roadmap.md Q3. + * + * xattrs are intentionally absent: Q3 commits Phase 2 to ignore-with- + * warning on xattr entries rather than fabricate a half-supported + * namespace mapping between Linux user/security/system xattrs and the + * macOS extended-attribute domain. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include + +typedef struct oci_meta_table oci_meta_table_t; + +/* Allocate an empty table. Returns NULL on OOM. */ +oci_meta_table_t *oci_meta_table_new(void); + +/* Release the table and all owned strings. Safe on NULL. */ +void oci_meta_table_free(oci_meta_table_t *t); + +/* Insert or update an entry. Idempotent: re-recording the same path + * overwrites the previous tuple. Returns 0 on success, -1 with errno + * set on allocation failure. + */ +int oci_meta_record(oci_meta_table_t *t, + const char *guest_path, + uint64_t uid, + uint64_t gid, + uint32_t mode); + +/* Remove a path from the table. No-op if the path is not recorded. + * Whiteouts and tar-overwrites both rely on this so the persisted + * sidecar does not accumulate stale tuples for files that no longer + * exist in the unpacked tree. + */ +void oci_meta_remove(oci_meta_table_t *t, const char *guest_path); + +/* Look up a path. Returns 0 with out-params filled, or -1 with + * errno=ENOENT if the path was never recorded. Any out param may be + * NULL to discard that field. + */ +int oci_meta_lookup(const oci_meta_table_t *t, + const char *guest_path, + uint64_t *out_uid, + uint64_t *out_gid, + uint32_t *out_mode); + +/* Number of live entries. */ +size_t oci_meta_count(const oci_meta_table_t *t); + +/* Serialize the table to / via atomic rename. + * Returns 0 on success, -1 on failure with errno and *err set. Passing + * an empty table writes a valid file containing an empty entries array. + * + * filename must be a relative basename (no embedded '/'); EINVAL is + * returned otherwise. The Plan 3 C3.3c raw per-layer cache writes + * ".elfuse-meta.layer.json" so that the assembled stack snapshot can + * keep the default ".elfuse-meta.json" name for the cumulative table + * without collisions during clonefile-stacked assembly. + */ +int oci_meta_write_named(const oci_meta_table_t *t, + const char *root_dir, + const char *filename, + const char **err); + +/* Parse / and populate a fresh table. Caller takes + * ownership via *out (freed with oci_meta_table_free). Missing file + * returns -1 with errno=ENOENT; malformed JSON or version mismatch + * returns -1 with errno=EINVAL. filename constraints match + * oci_meta_write_named (relative basename, no embedded '/'). + */ +int oci_meta_read_named(const char *root_dir, + const char *filename, + oci_meta_table_t **out, + const char **err); + +/* Thin wrappers passing the default ".elfuse-meta.json" filename. Used + * by the cumulative-sidecar paths (stack snapshot, final unpack tree). + */ +int oci_meta_write(const oci_meta_table_t *t, + const char *root_dir, + const char **err); + +int oci_meta_read(const char *root_dir, + oci_meta_table_t **out, + const char **err); + +/* Copy every entry from src into dst via oci_meta_record. Existing dst + * entries with the same guest path are overwritten (record's idempotent + * upsert semantics). Used by the Plan 3 C3.2 unpack layer cache hit path + * so a clonefile-restored layer's persisted sidecar repopulates the in- + * memory meta table that subsequent layer applies extend. Returns 0 on + * success or -1 with errno set (EINVAL on NULL inputs, ENOMEM on record + * allocation failure). Partial merges may leave dst with a subset of src + * already applied; the caller treats this as a fatal error. + */ +int oci_meta_merge(oci_meta_table_t *dst, const oci_meta_table_t *src); diff --git a/src/oci/manifest.c b/src/oci/manifest.c new file mode 100644 index 0000000..778d9b0 --- /dev/null +++ b/src/oci/manifest.c @@ -0,0 +1,727 @@ +/* OCI image manifest, image index, and image config parsers + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "manifest.h" + +#include +#include +#include +#include +#include + +#include "../../externals/cjson/cJSON.h" + +/* Maximum representable size that can survive a double round-trip without + * silent precision loss. JSON numbers parse through double in cJSON, so any + * size beyond 2^53 - 1 would already be off by ones; sizes well below that + * cover every realistic OCI layer. + */ +#define SIZE_MAX_SAFE INT64_C(0x1fffffffffffff) + +/* Optional string helper. JSON value may be absent (NULL item) or string + * type. Returns 1 on accepted (out_dup may be empty on success when allow + * is true), 0 on absent (out_dup left as default), -1 on type error. The + * caller owns the returned string. + */ +static int dup_optional_string(const cJSON *parent, + const char *key, + char **out_dup, + const char **err_msg, + const char *type_err) +{ + const cJSON *item = cJSON_GetObjectItemCaseSensitive(parent, key); + if (!item) + return 0; + if (!cJSON_IsString(item) || !item->valuestring) { + if (err_msg) + *err_msg = type_err; + return -1; + } + char *dup = strdup(item->valuestring); + if (!dup) { + if (err_msg) + *err_msg = "out of memory copying string field"; + return -1; + } + free(*out_dup); + *out_dup = dup; + return 1; +} + +static int require_string(const cJSON *parent, + const char *key, + char **out_dup, + const char **err_msg, + const char *missing_msg, + const char *type_msg) +{ + const cJSON *item = cJSON_GetObjectItemCaseSensitive(parent, key); + if (!item) { + if (err_msg) + *err_msg = missing_msg; + return -1; + } + if (!cJSON_IsString(item) || !item->valuestring) { + if (err_msg) + *err_msg = type_msg; + return -1; + } + char *dup = strdup(item->valuestring); + if (!dup) { + if (err_msg) + *err_msg = "out of memory copying required string"; + return -1; + } + free(*out_dup); + *out_dup = dup; + return 0; +} + +/* Convert a JSON string array into a NULL-terminated char** array. Returns + * 0 on success, -1 on type error or allocation failure. On absent field + * the function returns 1 and leaves *out_array untouched. + */ +static int dup_string_array(const cJSON *parent, + const char *key, + char ***out_array, + const char **err_msg, + const char *type_msg, + bool required) +{ + const cJSON *item = cJSON_GetObjectItemCaseSensitive(parent, key); + if (!item) { + if (required) { + if (err_msg) + *err_msg = type_msg; + return -1; + } + return 1; + } + if (!cJSON_IsArray(item)) { + if (err_msg) + *err_msg = type_msg; + return -1; + } + int n = cJSON_GetArraySize(item); + if (n < 0) + n = 0; + char **arr = calloc((size_t) n + 1, sizeof(*arr)); + if (!arr) { + if (err_msg) + *err_msg = "out of memory allocating string array"; + return -1; + } + for (int i = 0; i < n; i++) { + const cJSON *elem = cJSON_GetArrayItem(item, i); + if (!cJSON_IsString(elem) || !elem->valuestring) { + if (err_msg) + *err_msg = type_msg; + goto fail; + } + arr[i] = strdup(elem->valuestring); + if (!arr[i]) { + if (err_msg) + *err_msg = "out of memory copying string-array element"; + goto fail; + } + } + arr[n] = NULL; + /* Free any prior value before publishing the new one. */ + if (*out_array) { + for (char **p = *out_array; *p; p++) + free(*p); + free(*out_array); + } + *out_array = arr; + return 0; +fail: + for (int i = 0; i < n; i++) + free(arr[i]); + free(arr); + return -1; +} + +/* Parse a non-negative integer-valued JSON number. cJSON keeps numbers in + * a double so the practical upper bound is 2^53 - 1; OCI layer sizes are + * well below that. + */ +static int parse_size_field(const cJSON *parent, + const char *key, + int64_t *out, + const char **err_msg) +{ + const cJSON *item = cJSON_GetObjectItemCaseSensitive(parent, key); + if (!item) { + if (err_msg) + *err_msg = "descriptor missing size field"; + return -1; + } + if (!cJSON_IsNumber(item)) { + if (err_msg) + *err_msg = "descriptor size field is not a number"; + return -1; + } + double v = item->valuedouble; + if (!(v >= 0.0) || v > (double) SIZE_MAX_SAFE) { + if (err_msg) + *err_msg = "descriptor size out of representable range"; + return -1; + } + /* Round-trip check: the JSON number must already be an integer. The + * double-to-int64 cast truncates; reject anything with a fractional part + * before truncation hides the divergence. + */ + int64_t as_int = (int64_t) v; + if ((double) as_int != v) { + if (err_msg) + *err_msg = "descriptor size field is not an integer"; + return -1; + } + *out = as_int; + return 0; +} + +static int parse_descriptor(const cJSON *obj, + oci_descriptor_t *out, + const char **err_msg) +{ + memset(out, 0, sizeof(*out)); + + /* mediaType: optional per OCI image-spec (some legacy responses omit it + * on the implicit root), but every descriptor that lives inside another + * document does carry it. Treat it as required at parse time and let + * the caller relax it for the top-level document if needed. + */ + char *raw_mt = NULL; + if (require_string(obj, "mediaType", &raw_mt, err_msg, + "descriptor missing mediaType", + "descriptor mediaType must be a string") < 0) + goto fail; + out->raw_media_type = raw_mt; + out->media_type = oci_media_type_parse(raw_mt); + + if (require_string(obj, "digest", &out->digest_str, err_msg, + "descriptor missing digest", + "descriptor digest must be a string") < 0) + goto fail; + if (!oci_digest_parse(out->digest_str, &out->algo, out->hex)) { + if (err_msg) + *err_msg = "descriptor digest is malformed or not lowercase"; + goto fail; + } + + if (parse_size_field(obj, "size", &out->size, err_msg) < 0) + goto fail; + return 0; +fail: + oci_descriptor_free(out); + return -1; +} + +static int parse_platform(const cJSON *obj, + oci_platform_t *out, + const char **err_msg) +{ + memset(out, 0, sizeof(*out)); + if (!obj || !cJSON_IsObject(obj)) { + if (err_msg) + *err_msg = "platform field missing or not an object"; + return -1; + } + if (require_string(obj, "architecture", &out->architecture, err_msg, + "platform missing architecture", + "platform architecture must be a string") < 0) + goto fail; + if (require_string(obj, "os", &out->os, err_msg, "platform missing os", + "platform os must be a string") < 0) + goto fail; + + /* variant and os.version default to "" so callers can compare without + * NULL checks. dup_optional_string sets the field only when present. + */ + if (dup_optional_string(obj, "variant", &out->variant, err_msg, + "platform variant must be a string") < 0) + goto fail; + if (!out->variant) { + out->variant = strdup(""); + if (!out->variant) { + if (err_msg) + *err_msg = "out of memory defaulting variant"; + goto fail; + } + } + if (dup_optional_string(obj, "os.version", &out->os_version, err_msg, + "platform os.version must be a string") < 0) + goto fail; + if (!out->os_version) { + out->os_version = strdup(""); + if (!out->os_version) { + if (err_msg) + *err_msg = "out of memory defaulting os.version"; + goto fail; + } + } + return 0; +fail: + oci_platform_free(out); + return -1; +} + +static int parse_int_field(const cJSON *parent, + const char *key, + int *out, + bool required, + const char **err_msg, + const char *missing_msg, + const char *type_msg) +{ + const cJSON *item = cJSON_GetObjectItemCaseSensitive(parent, key); + if (!item) { + if (required) { + if (err_msg) + *err_msg = missing_msg; + return -1; + } + return 1; + } + if (!cJSON_IsNumber(item)) { + if (err_msg) + *err_msg = type_msg; + return -1; + } + /* Round-trip check: cJSON's valueint truncates the JSON number, so a + * fractional value like "schemaVersion": 2.7 would otherwise pass an + * == 2 comparison downstream. Reject anything that is out of int range + * or carries a fractional part before the truncation hides it. + */ + double v = item->valuedouble; + if (v < (double) INT_MIN || v > (double) INT_MAX) { + if (err_msg) + *err_msg = type_msg; + return -1; + } + int as_int = (int) v; + if ((double) as_int != v) { + if (err_msg) + *err_msg = type_msg; + return -1; + } + *out = as_int; + return 0; +} + +/* Convert a cJSON parse failure into our diagnostic message space. cJSON's + * cJSON_GetErrorPtr is process-global; the message we set is static and the + * caller never frees it. + */ +static void set_parse_err(const char **err_msg, const char *fallback) +{ + if (err_msg) + *err_msg = fallback; + errno = EINVAL; +} + +int oci_manifest_parse(const char *json, + size_t len, + oci_manifest_t *out, + const char **err_msg) +{ + if (!json || !out) { + set_parse_err(err_msg, "oci_manifest_parse: NULL input"); + return -1; + } + memset(out, 0, sizeof(*out)); + + cJSON *root = cJSON_ParseWithLength(json, len); + if (!root) { + set_parse_err(err_msg, "manifest JSON is malformed"); + return -1; + } + if (!cJSON_IsObject(root)) { + set_parse_err(err_msg, "manifest JSON root is not an object"); + goto fail; + } + + if (parse_int_field(root, "schemaVersion", &out->schema_version, true, + err_msg, "manifest missing schemaVersion", + "manifest schemaVersion must be a number") < 0) + goto fail; + if (out->schema_version != 2) { + set_parse_err(err_msg, "manifest schemaVersion must be 2"); + goto fail; + } + + /* mediaType on the manifest itself is optional in some Docker responses + * (the Content-Type header is canonical there); record raw and parsed + * forms but do not reject on absence. + */ + if (dup_optional_string(root, "mediaType", &out->raw_media_type, err_msg, + "manifest mediaType must be a string") < 0) + goto fail; + out->media_type = out->raw_media_type + ? oci_media_type_parse(out->raw_media_type) + : OCI_MT_UNKNOWN; + + const cJSON *cfg = cJSON_GetObjectItemCaseSensitive(root, "config"); + if (!cfg || !cJSON_IsObject(cfg)) { + set_parse_err(err_msg, "manifest config descriptor missing"); + goto fail; + } + if (parse_descriptor(cfg, &out->config, err_msg) < 0) + goto fail; + if (!oci_media_type_is_config(out->config.media_type)) { + set_parse_err(err_msg, "manifest config has non-config media type"); + goto fail; + } + + const cJSON *layers = cJSON_GetObjectItemCaseSensitive(root, "layers"); + if (!layers || !cJSON_IsArray(layers)) { + set_parse_err(err_msg, "manifest layers array missing"); + goto fail; + } + int nlayers = cJSON_GetArraySize(layers); + if (nlayers < 0) + nlayers = 0; + if (nlayers > 0) { + out->layers = calloc((size_t) nlayers, sizeof(*out->layers)); + if (!out->layers) { + set_parse_err(err_msg, "out of memory allocating layer array"); + errno = ENOMEM; + goto fail; + } + } + for (int i = 0; i < nlayers; i++) { + const cJSON *desc = cJSON_GetArrayItem(layers, i); + if (!cJSON_IsObject(desc)) { + set_parse_err(err_msg, "manifest layer entry is not an object"); + goto fail; + } + if (parse_descriptor(desc, &out->layers[out->nlayers], err_msg) < 0) + goto fail; + oci_media_type_t lmt = out->layers[out->nlayers].media_type; + /* Count the slot now that parse_descriptor has populated (and + * possibly heap-allocated) it. The validation below can still + * goto fail; counting first lets oci_manifest_free reclaim it. + */ + out->nlayers++; + if (!oci_media_type_is_layer(lmt)) { + set_parse_err(err_msg, "manifest layer has non-layer media type"); + goto fail; + } + if (oci_media_type_is_foreign(lmt)) { + set_parse_err(err_msg, + "manifest references foreign (nondistributable) " + "layer; not supported"); + goto fail; + } + if (!oci_media_type_is_layer_supported(lmt)) { + set_parse_err(err_msg, + "manifest layer media type is not supported " + "(only tar / tar+gzip / tar+zstd)"); + goto fail; + } + } + + cJSON_Delete(root); + return 0; +fail: + cJSON_Delete(root); + oci_manifest_free(out); + return -1; +} + +int oci_index_parse(const char *json, + size_t len, + oci_index_t *out, + const char **err_msg) +{ + if (!json || !out) { + set_parse_err(err_msg, "oci_index_parse: NULL input"); + return -1; + } + memset(out, 0, sizeof(*out)); + + cJSON *root = cJSON_ParseWithLength(json, len); + if (!root) { + set_parse_err(err_msg, "index JSON is malformed"); + return -1; + } + if (!cJSON_IsObject(root)) { + set_parse_err(err_msg, "index JSON root is not an object"); + goto fail; + } + + if (parse_int_field(root, "schemaVersion", &out->schema_version, true, + err_msg, "index missing schemaVersion", + "index schemaVersion must be a number") < 0) + goto fail; + if (out->schema_version != 2) { + set_parse_err(err_msg, "index schemaVersion must be 2"); + goto fail; + } + + if (dup_optional_string(root, "mediaType", &out->raw_media_type, err_msg, + "index mediaType must be a string") < 0) + goto fail; + out->media_type = out->raw_media_type + ? oci_media_type_parse(out->raw_media_type) + : OCI_MT_UNKNOWN; + + const cJSON *manifests = + cJSON_GetObjectItemCaseSensitive(root, "manifests"); + if (!manifests || !cJSON_IsArray(manifests)) { + set_parse_err(err_msg, "index manifests array missing"); + goto fail; + } + int n = cJSON_GetArraySize(manifests); + if (n < 0) + n = 0; + if (n > 0) { + out->entries = calloc((size_t) n, sizeof(*out->entries)); + if (!out->entries) { + set_parse_err(err_msg, "out of memory allocating index entries"); + errno = ENOMEM; + goto fail; + } + } + for (int i = 0; i < n; i++) { + const cJSON *entry = cJSON_GetArrayItem(manifests, i); + if (!cJSON_IsObject(entry)) { + set_parse_err(err_msg, "index manifest entry is not an object"); + goto fail; + } + oci_index_entry_t *slot = &out->entries[out->nentries]; + if (parse_descriptor(entry, &slot->desc, err_msg) < 0) + goto fail; + /* Count the entry now that slot->desc holds allocated strings; the + * platform parse below can still goto fail, where oci_index_free + * must see this slot to reclaim both desc and platform. + */ + out->nentries++; + const cJSON *plat = cJSON_GetObjectItemCaseSensitive(entry, "platform"); + if (parse_platform(plat, &slot->platform, err_msg) < 0) + goto fail; + } + + cJSON_Delete(root); + return 0; +fail: + cJSON_Delete(root); + oci_index_free(out); + return -1; +} + +int oci_image_config_parse(const char *json, + size_t len, + oci_image_config_t *out, + const char **err_msg) +{ + if (!json || !out) { + set_parse_err(err_msg, "oci_image_config_parse: NULL input"); + return -1; + } + memset(out, 0, sizeof(*out)); + + cJSON *root = cJSON_ParseWithLength(json, len); + if (!root) { + set_parse_err(err_msg, "image config JSON is malformed"); + return -1; + } + if (!cJSON_IsObject(root)) { + set_parse_err(err_msg, "image config JSON root is not an object"); + goto fail; + } + + if (require_string(root, "architecture", &out->architecture, err_msg, + "image config missing architecture", + "image config architecture must be a string") < 0) + goto fail; + if (require_string(root, "os", &out->os, err_msg, "image config missing os", + "image config os must be a string") < 0) + goto fail; + if (dup_optional_string(root, "variant", &out->variant, err_msg, + "image config variant must be a string") < 0) + goto fail; + + const cJSON *cfg = cJSON_GetObjectItemCaseSensitive(root, "config"); + if (cfg) { + if (!cJSON_IsObject(cfg)) { + set_parse_err(err_msg, "image config.config must be an object"); + goto fail; + } + if (dup_optional_string(cfg, "User", &out->config.user, err_msg, + "image config User must be a string") < 0) + goto fail; + if (dup_optional_string(cfg, "WorkingDir", &out->config.working_dir, + err_msg, + "image config WorkingDir must be a string") < 0) + goto fail; + if (dup_string_array(cfg, "Env", &out->config.env, err_msg, + "image config Env must be a string array", + false) < 0) + goto fail; + if (dup_string_array( + cfg, "Entrypoint", &out->config.entrypoint, err_msg, + "image config Entrypoint must be a string array", false) < 0) + goto fail; + if (dup_string_array(cfg, "Cmd", &out->config.cmd, err_msg, + "image config Cmd must be a string array", + false) < 0) + goto fail; + } + + const cJSON *rootfs = cJSON_GetObjectItemCaseSensitive(root, "rootfs"); + if (!rootfs || !cJSON_IsObject(rootfs)) { + set_parse_err(err_msg, "image config rootfs object missing"); + goto fail; + } + const cJSON *type = cJSON_GetObjectItemCaseSensitive(rootfs, "type"); + if (!type || !cJSON_IsString(type) || !type->valuestring || + strcmp(type->valuestring, "layers") != 0) { + set_parse_err(err_msg, "image config rootfs.type must be \"layers\""); + goto fail; + } + if (dup_string_array(rootfs, "diff_ids", &out->rootfs_diff_ids, err_msg, + "image config rootfs.diff_ids must be a string " + "array", + true) < 0) + goto fail; + /* Validate every diff_id is a recognized digest. */ + for (char **p = out->rootfs_diff_ids; p && *p; p++) { + oci_digest_algo_t algo; + char hex[OCI_DIGEST_HEX_MAX + 1]; + if (!oci_digest_parse(*p, &algo, hex)) { + set_parse_err(err_msg, + "image config rootfs.diff_ids entry is malformed " + "or not lowercase"); + goto fail; + } + } + + cJSON_Delete(root); + return 0; +fail: + cJSON_Delete(root); + oci_image_config_free(out); + return -1; +} + +void oci_descriptor_free(oci_descriptor_t *d) +{ + if (!d) + return; + free(d->digest_str); + free(d->raw_media_type); + memset(d, 0, sizeof(*d)); +} + +void oci_platform_free(oci_platform_t *p) +{ + if (!p) + return; + free(p->architecture); + free(p->os); + free(p->variant); + free(p->os_version); + memset(p, 0, sizeof(*p)); +} + +static void runtime_free(oci_image_runtime_t *r) +{ + if (!r) + return; + free(r->user); + free(r->working_dir); + if (r->env) { + for (char **p = r->env; *p; p++) + free(*p); + free(r->env); + } + if (r->entrypoint) { + for (char **p = r->entrypoint; *p; p++) + free(*p); + free(r->entrypoint); + } + if (r->cmd) { + for (char **p = r->cmd; *p; p++) + free(*p); + free(r->cmd); + } + memset(r, 0, sizeof(*r)); +} + +void oci_manifest_free(oci_manifest_t *m) +{ + if (!m) + return; + free(m->raw_media_type); + oci_descriptor_free(&m->config); + for (size_t i = 0; i < m->nlayers; i++) + oci_descriptor_free(&m->layers[i]); + free(m->layers); + memset(m, 0, sizeof(*m)); +} + +void oci_index_free(oci_index_t *idx) +{ + if (!idx) + return; + free(idx->raw_media_type); + for (size_t i = 0; i < idx->nentries; i++) { + oci_descriptor_free(&idx->entries[i].desc); + oci_platform_free(&idx->entries[i].platform); + } + free(idx->entries); + memset(idx, 0, sizeof(*idx)); +} + +void oci_image_config_free(oci_image_config_t *c) +{ + if (!c) + return; + free(c->architecture); + free(c->os); + free(c->variant); + runtime_free(&c->config); + if (c->rootfs_diff_ids) { + for (char **p = c->rootfs_diff_ids; *p; p++) + free(*p); + free(c->rootfs_diff_ids); + } + memset(c, 0, sizeof(*c)); +} + +const oci_index_entry_t *oci_index_pick_linux_arm64(const oci_index_t *idx) +{ + if (!idx || !idx->entries) + return NULL; + + const oci_index_entry_t *fallback_empty = NULL; + const oci_index_entry_t *fallback_any = NULL; + + for (size_t i = 0; i < idx->nentries; i++) { + const oci_index_entry_t *e = &idx->entries[i]; + if (strcmp(e->platform.os, "linux") != 0) + continue; + if (strcmp(e->platform.architecture, "arm64") != 0) + continue; + /* Skip foreign or unrecognized manifest media types: the registry + * fetch path cannot consume them anyway, so they are not viable + * even when the platform matches. + */ + if (!oci_media_type_is_manifest(e->desc.media_type)) + continue; + if (strcmp(e->platform.variant, "v8") == 0) + return e; + if (e->platform.variant[0] == '\0') { + if (!fallback_empty) + fallback_empty = e; + } else if (!fallback_any) { + fallback_any = e; + } + } + return fallback_empty ? fallback_empty : fallback_any; +} diff --git a/src/oci/manifest.h b/src/oci/manifest.h new file mode 100644 index 0000000..66ff14d --- /dev/null +++ b/src/oci/manifest.h @@ -0,0 +1,160 @@ +/* OCI image manifest, image index, and image config parsers + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Parses the three JSON document types served by an OCI / Docker registry: + * + * - image manifest: config descriptor + ordered layer descriptors + * - image index: platform-tagged manifest descriptors (multi-arch) + * - image config: architecture/os + runtime fields + rootfs diff_ids + * + * Phase 1 keeps the model offline: parsers operate on in-memory JSON bytes + * the caller already obtained from a registry fetch or disk fixture. The + * registry client lives in a later slice; the manifest model exists now so + * the fetch path can deserialize responses, and so the blob store can + * persist the parsed graph without round-tripping through opaque JSON. + * + * Every descriptor digest is validated up-front with oci/digest.c, so a + * parsed oci_descriptor_t is guaranteed to have a lowercase + * : form and a populated (algo, hex[]) pair the blob store can + * consume directly. + * + * Unknown / extension media types do not fail the parse; they are recorded + * with raw_media_type set and media_type == OCI_MT_UNKNOWN so callers can + * decide whether to ignore or reject. The selection helper for + * linux/arm64 manifests intentionally skips any entry that already failed + * media-type recognition because the registry fetch path cannot resolve + * it anyway. + */ + +#pragma once + +#include +#include + +#include "digest.h" +#include "media-type.h" + +typedef struct { + /* Original ":" string, lowercase, never NULL after parse. */ + char *digest_str; + /* Parsed digest algorithm. */ + oci_digest_algo_t algo; + /* Parsed lowercase hex (NUL-terminated). */ + char hex[OCI_DIGEST_HEX_MAX + 1]; + /* Declared size in bytes. Negative values are rejected at parse. */ + int64_t size; + /* Canonical media-type enum, OCI_MT_UNKNOWN if not in the recognized + * table. + */ + oci_media_type_t media_type; + /* Original media-type string for diagnostics. NULL if absent. */ + char *raw_media_type; +} oci_descriptor_t; + +typedef struct { + /* "arm64", "amd64", "ppc64le", ... Never NULL after parse. */ + char *architecture; + /* "linux", "windows", ... Never NULL after parse. */ + char *os; + /* "v8", "v7", "" (empty string when absent in JSON). */ + char *variant; + /* "10.0.14393.1066" for Windows builds, "" otherwise. */ + char *os_version; +} oci_platform_t; + +typedef struct { + oci_descriptor_t desc; + /* Empty platform fields ("" strings, not NULL) when JSON omits them so + * predicates can compare unconditionally. + */ + oci_platform_t platform; +} oci_index_entry_t; + +typedef struct { + int schema_version; + /* Top-level mediaType field. OCI manifests carry an explicit mediaType; + * Docker manifests historically rely on the descriptor or HTTP + * Content-Type. The parser falls back to OCI_MT_UNKNOWN if the JSON + * field is missing and lets the caller cross-check against the + * registry's Content-Type. + */ + oci_media_type_t media_type; + /* Original mediaType string, NULL if absent. */ + char *raw_media_type; + oci_index_entry_t *entries; + size_t nentries; +} oci_index_t; + +typedef struct { + int schema_version; + oci_media_type_t media_type; + char *raw_media_type; + oci_descriptor_t config; + oci_descriptor_t *layers; + size_t nlayers; +} oci_manifest_t; + +/* Image config runtime block (the inner "config" object). Phase 3 of the + * OCI roadmap consumes these fields; the model exists in Phase 1 to support + * elfuse oci inspect rendering. NULL-terminated string arrays are NULL when + * the JSON omits the field; empty arrays are represented as an allocated + * one-element array containing only the NULL terminator. + */ +typedef struct { + char *user; + char *working_dir; + char **env; + char **entrypoint; + char **cmd; +} oci_image_runtime_t; + +typedef struct { + char *architecture; + char *os; + char *variant; + oci_image_runtime_t config; + /* rootfs.diff_ids, NULL-terminated. Always populated (the OCI image-spec + * requires "rootfs"); a parse without this field returns -1. + */ + char **rootfs_diff_ids; +} oci_image_config_t; + +/* Parsers. Each takes raw JSON bytes (need not be NUL-terminated; pass the + * exact length). On success returns 0 and populates out. On failure returns + * -1 with errno preserved when set (ENOMEM, EINVAL) and writes a static + * diagnostic message into *err_msg (when err_msg != NULL). + */ +int oci_manifest_parse(const char *json, + size_t len, + oci_manifest_t *out, + const char **err_msg); + +int oci_index_parse(const char *json, + size_t len, + oci_index_t *out, + const char **err_msg); + +int oci_image_config_parse(const char *json, + size_t len, + oci_image_config_t *out, + const char **err_msg); + +/* Release any heap fields. Safe on zero-initialised structs and on NULL. */ +void oci_manifest_free(oci_manifest_t *m); +void oci_index_free(oci_index_t *idx); +void oci_image_config_free(oci_image_config_t *c); +void oci_descriptor_free(oci_descriptor_t *d); +void oci_platform_free(oci_platform_t *p); + +/* Select the linux/arm64 manifest from an index. Returns a pointer into + * idx->entries on success (caller does not free) or NULL when no acceptable + * platform is present. Preference order, highest first: + * 1. os=="linux" && arch=="arm64" && variant=="v8" + * 2. os=="linux" && arch=="arm64" && variant=="" + * 3. os=="linux" && arch=="arm64" (any other variant; first wins) + * Foreign / unsupported media types are skipped: even if a foreign-layer + * manifest claims linux/arm64, the registry fetch path cannot consume it. + */ +const oci_index_entry_t *oci_index_pick_linux_arm64(const oci_index_t *idx); diff --git a/src/oci/media-type.c b/src/oci/media-type.c new file mode 100644 index 0000000..0c6f9a7 --- /dev/null +++ b/src/oci/media-type.c @@ -0,0 +1,194 @@ +/* OCI / Docker media-type canonicalization + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "media-type.h" + +#include +#include +#include +#include + +struct mt_entry { + const char *name; + oci_media_type_t kind; +}; + +/* All recognized OCI and Docker media types in a single table. Order has no + * semantic meaning; the lookup is linear because the table is small (~16 + * entries) and runs at most once per descriptor parse. + */ +static const struct mt_entry MEDIA_TYPES[] = { + /* Manifest documents. */ + {"application/vnd.oci.image.manifest.v1+json", OCI_MT_MANIFEST_OCI}, + {"application/vnd.docker.distribution.manifest.v2+json", + OCI_MT_MANIFEST_DOCKER}, + + /* Image indexes / manifest lists. */ + {"application/vnd.oci.image.index.v1+json", OCI_MT_INDEX_OCI}, + {"application/vnd.docker.distribution.manifest.list.v2+json", + OCI_MT_INDEX_DOCKER}, + + /* Image config. */ + {"application/vnd.oci.image.config.v1+json", OCI_MT_CONFIG_OCI}, + {"application/vnd.docker.container.image.v1+json", OCI_MT_CONFIG_DOCKER}, + + /* Supported layer payloads. */ + {"application/vnd.oci.image.layer.v1.tar", OCI_MT_LAYER_OCI_TAR}, + {"application/vnd.oci.image.layer.v1.tar+gzip", OCI_MT_LAYER_OCI_TAR_GZIP}, + {"application/vnd.oci.image.layer.v1.tar+zstd", OCI_MT_LAYER_OCI_TAR_ZSTD}, + {"application/vnd.docker.image.rootfs.diff.tar.gzip", + OCI_MT_LAYER_DOCKER_TAR_GZIP}, + {"application/vnd.docker.image.rootfs.diff.tar.zstd", + OCI_MT_LAYER_DOCKER_TAR_ZSTD}, + + /* Foreign (nondistributable) layers. Recognized so the parser can produce + * a precise rejection message instead of falling through to UNKNOWN. + */ + {"application/vnd.oci.image.layer.nondistributable.v1.tar", + OCI_MT_LAYER_FOREIGN_OCI}, + {"application/vnd.oci.image.layer.nondistributable.v1.tar+gzip", + OCI_MT_LAYER_FOREIGN_OCI_GZIP}, + {"application/vnd.docker.image.rootfs.foreign.diff.tar", + OCI_MT_LAYER_FOREIGN_DOCKER}, + {"application/vnd.docker.image.rootfs.foreign.diff.tar.gzip", + OCI_MT_LAYER_FOREIGN_DOCKER_GZIP}, +}; + +#define MEDIA_TYPE_COUNT (sizeof(MEDIA_TYPES) / sizeof(MEDIA_TYPES[0])) + +/* Strip surrounding whitespace and any parameters after ';'. Writes the + * canonical span into out. Returns the canonical length or 0 if the input + * collapses to empty. + */ +static size_t canonicalize(const char *s, char *out, size_t out_size) +{ + if (!s || out_size == 0) + return 0; + + while (*s == ' ' || *s == '\t') + s++; + + const char *end = s; + while (*end && *end != ';') + end++; + while (end > s && (end[-1] == ' ' || end[-1] == '\t')) + end--; + + size_t len = (size_t) (end - s); + if (len == 0 || len >= out_size) + return 0; + memcpy(out, s, len); + out[len] = '\0'; + return len; +} + +oci_media_type_t oci_media_type_parse(const char *s) +{ + if (!s) + return OCI_MT_UNKNOWN; + + /* Media-type values in OCI manifests are short; 192 bytes covers every + * canonical name in the table with room for adversarial whitespace. + */ + char buf[192]; + if (canonicalize(s, buf, sizeof(buf)) == 0) + return OCI_MT_UNKNOWN; + + /* RFC 6838: media type and subtype tokens are case-insensitive. The + * parameter span (after ';') is already stripped by canonicalize, so the + * whole of buf is type/subtype and can be matched case-insensitively. + */ + for (size_t i = 0; i < MEDIA_TYPE_COUNT; i++) { + if (!strcasecmp(MEDIA_TYPES[i].name, buf)) + return MEDIA_TYPES[i].kind; + } + return OCI_MT_UNKNOWN; +} + +const char *oci_media_type_name(oci_media_type_t mt) +{ + for (size_t i = 0; i < MEDIA_TYPE_COUNT; i++) { + if (MEDIA_TYPES[i].kind == mt) + return MEDIA_TYPES[i].name; + } + return NULL; +} + +bool oci_media_type_is_manifest(oci_media_type_t mt) +{ + return mt == OCI_MT_MANIFEST_OCI || mt == OCI_MT_MANIFEST_DOCKER; +} + +bool oci_media_type_is_index(oci_media_type_t mt) +{ + return mt == OCI_MT_INDEX_OCI || mt == OCI_MT_INDEX_DOCKER; +} + +bool oci_media_type_is_config(oci_media_type_t mt) +{ + return mt == OCI_MT_CONFIG_OCI || mt == OCI_MT_CONFIG_DOCKER; +} + +bool oci_media_type_is_layer(oci_media_type_t mt) +{ + switch (mt) { + case OCI_MT_LAYER_OCI_TAR: + case OCI_MT_LAYER_OCI_TAR_GZIP: + case OCI_MT_LAYER_OCI_TAR_ZSTD: + case OCI_MT_LAYER_DOCKER_TAR_GZIP: + case OCI_MT_LAYER_DOCKER_TAR_ZSTD: + case OCI_MT_LAYER_FOREIGN_OCI: + case OCI_MT_LAYER_FOREIGN_OCI_GZIP: + case OCI_MT_LAYER_FOREIGN_DOCKER: + case OCI_MT_LAYER_FOREIGN_DOCKER_GZIP: + return true; + default: + return false; + } +} + +bool oci_media_type_is_layer_supported(oci_media_type_t mt) +{ + switch (mt) { + case OCI_MT_LAYER_OCI_TAR: + case OCI_MT_LAYER_OCI_TAR_GZIP: + case OCI_MT_LAYER_OCI_TAR_ZSTD: + case OCI_MT_LAYER_DOCKER_TAR_GZIP: + case OCI_MT_LAYER_DOCKER_TAR_ZSTD: + return true; + default: + return false; + } +} + +bool oci_media_type_is_foreign(oci_media_type_t mt) +{ + switch (mt) { + case OCI_MT_LAYER_FOREIGN_OCI: + case OCI_MT_LAYER_FOREIGN_OCI_GZIP: + case OCI_MT_LAYER_FOREIGN_DOCKER: + case OCI_MT_LAYER_FOREIGN_DOCKER_GZIP: + return true; + default: + return false; + } +} + +oci_compression_t oci_media_type_compression(oci_media_type_t mt) +{ + switch (mt) { + case OCI_MT_LAYER_OCI_TAR_GZIP: + case OCI_MT_LAYER_DOCKER_TAR_GZIP: + case OCI_MT_LAYER_FOREIGN_OCI_GZIP: + case OCI_MT_LAYER_FOREIGN_DOCKER_GZIP: + return OCI_COMPRESSION_GZIP; + case OCI_MT_LAYER_OCI_TAR_ZSTD: + case OCI_MT_LAYER_DOCKER_TAR_ZSTD: + return OCI_COMPRESSION_ZSTD; + default: + return OCI_COMPRESSION_NONE; + } +} diff --git a/src/oci/media-type.h b/src/oci/media-type.h new file mode 100644 index 0000000..66a2a1b --- /dev/null +++ b/src/oci/media-type.h @@ -0,0 +1,93 @@ +/* OCI / Docker media-type canonicalization + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * OCI image references carry media-type strings on every descriptor. The + * registry client, manifest parser, and unpack stage all branch on the media + * type, so a single canonical enum lookup keeps the comparisons one place + * away from string typos. Docker registries continue to serve the legacy + * docker-namespaced media types (vnd.docker.distribution.manifest.v2+json) + * even when the image-spec wire format is OCI v1; the table accepts both. + * + * Foreign (nondistributable) layers are recognized but classified as + * unsupported per oci-roadmap.md Q3: elfuse cannot fetch the out-of-band + * payload those layers reference, so rejecting them at parse time is the + * honest answer rather than carrying a half-supported code path. + */ + +#pragma once + +#include + +typedef enum { + OCI_MT_UNKNOWN = 0, + + /* Manifest documents (single platform). */ + OCI_MT_MANIFEST_OCI, + OCI_MT_MANIFEST_DOCKER, + + /* Image index / manifest list (multi-platform). */ + OCI_MT_INDEX_OCI, + OCI_MT_INDEX_DOCKER, + + /* Image config blob. */ + OCI_MT_CONFIG_OCI, + OCI_MT_CONFIG_DOCKER, + + /* Layer blobs that elfuse can actually consume. */ + OCI_MT_LAYER_OCI_TAR, + OCI_MT_LAYER_OCI_TAR_GZIP, + OCI_MT_LAYER_OCI_TAR_ZSTD, + OCI_MT_LAYER_DOCKER_TAR_GZIP, + OCI_MT_LAYER_DOCKER_TAR_ZSTD, + + /* Foreign layers: distinguishable but explicitly unsupported. */ + OCI_MT_LAYER_FOREIGN_OCI, + OCI_MT_LAYER_FOREIGN_OCI_GZIP, + OCI_MT_LAYER_FOREIGN_DOCKER, + OCI_MT_LAYER_FOREIGN_DOCKER_GZIP, +} oci_media_type_t; + +typedef enum { + OCI_COMPRESSION_NONE, + OCI_COMPRESSION_GZIP, + OCI_COMPRESSION_ZSTD, +} oci_compression_t; + +/* Classify a media-type string. Trailing parameters after ';' (e.g. charset) + * are stripped before matching; surrounding whitespace is ignored. Returns + * OCI_MT_UNKNOWN for any string not in the recognized table. NULL is treated + * as OCI_MT_UNKNOWN. + */ +oci_media_type_t oci_media_type_parse(const char *s); + +/* Lookup the canonical name string for a media-type enum. Returns NULL for + * OCI_MT_UNKNOWN or an out-of-range enum value. The returned pointer is to + * static storage. + */ +const char *oci_media_type_name(oci_media_type_t mt); + +/* Predicates by document category. Each returns false for OCI_MT_UNKNOWN. */ +bool oci_media_type_is_manifest(oci_media_type_t mt); +bool oci_media_type_is_index(oci_media_type_t mt); +bool oci_media_type_is_config(oci_media_type_t mt); +bool oci_media_type_is_layer(oci_media_type_t mt); + +/* True when the layer media type is one elfuse can actually decode. Foreign + * layers and OCI_MT_UNKNOWN return false; the manifest parser rejects layer + * descriptors that fail this check. + */ +bool oci_media_type_is_layer_supported(oci_media_type_t mt); + +/* True for the four foreign-layer media types. The manifest parser keeps + * these distinguishable so the error message can name the actual layer type + * instead of a generic 'unsupported'. + */ +bool oci_media_type_is_foreign(oci_media_type_t mt); + +/* Compression algorithm carried by a layer media type. Non-layer or unknown + * inputs return OCI_COMPRESSION_NONE; callers should gate on + * oci_media_type_is_layer first. + */ +oci_compression_t oci_media_type_compression(oci_media_type_t mt); diff --git a/src/oci/origin-meta.c b/src/oci/origin-meta.c new file mode 100644 index 0000000..8095c99 --- /dev/null +++ b/src/oci/origin-meta.c @@ -0,0 +1,330 @@ +/* OCI unpacked-tree provenance sidecar implementation + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "../../externals/cjson/cJSON.h" +#include "oci/origin-meta.h" + +#define OCI_ORIGIN_FILE ".elfuse-origin.json" + +/* Conservative ceiling. A realistic sidecar is a few hundred bytes + * (three short JSON strings plus an array of ~64 byte diff_id entries). + * Anything past 1 MiB indicates a corrupt or hostile file and is + * rejected so the reader does not try to malloc gigabytes from a + * stat-spoofed input. + */ +#define OCI_ORIGIN_MAX_BYTES (1u << 20) + +static int set_err(const char **err, const char *msg, int err_no) +{ + if (err) + *err = msg; + errno = err_no; + return -1; +} + +static char *build_path(const char *root_dir, const char *name, int tmp) +{ + size_t want = strlen(root_dir) + 1 + strlen(name) + (tmp ? 4 : 0) + 1; + char *p = malloc(want); + if (!p) + return NULL; + snprintf(p, want, "%s/%s%s", root_dir, name, tmp ? ".tmp" : ""); + return p; +} + +int oci_origin_write(const char *root_dir, + const char *manifest_digest, + const char *config_digest, + char *const *diff_ids, + const char **err) +{ + static const char *dummy_err; + if (!err) + err = &dummy_err; + *err = NULL; + if (!root_dir || !manifest_digest || !config_digest) + return set_err(err, "origin write: NULL argument", EINVAL); + if (!root_dir[0] || !manifest_digest[0] || !config_digest[0]) + return set_err(err, "origin write: empty argument", EINVAL); + + cJSON *root = cJSON_CreateObject(); + if (!root) + return set_err(err, "origin write: cJSON_CreateObject failed", ENOMEM); + if (!cJSON_AddStringToObject(root, "manifest_digest", manifest_digest)) { + cJSON_Delete(root); + return set_err(err, "origin write: manifest_digest add failed", ENOMEM); + } + if (!cJSON_AddStringToObject(root, "config_digest", config_digest)) { + cJSON_Delete(root); + return set_err(err, "origin write: config_digest add failed", ENOMEM); + } + cJSON *arr = cJSON_AddArrayToObject(root, "layer_diffids"); + if (!arr) { + cJSON_Delete(root); + return set_err(err, "origin write: layer_diffids add failed", ENOMEM); + } + if (diff_ids) { + for (char *const *p = diff_ids; *p; p++) { + cJSON *s = cJSON_CreateString(*p); + if (!s) { + cJSON_Delete(root); + return set_err(err, "origin write: diff_id string failed", + ENOMEM); + } + cJSON_AddItemToArray(arr, s); + } + } + + char *json = cJSON_PrintUnformatted(root); + cJSON_Delete(root); + if (!json) + return set_err(err, "origin write: cJSON_Print failed", ENOMEM); + size_t jlen = strlen(json); + + char *tmp_path = build_path(root_dir, OCI_ORIGIN_FILE, 1); + char *final_path = build_path(root_dir, OCI_ORIGIN_FILE, 0); + if (!tmp_path || !final_path) { + free(tmp_path); + free(final_path); + free(json); + return set_err(err, "origin write: path allocation failed", ENOMEM); + } + + int fd = open(tmp_path, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 0644); + if (fd < 0) { + int saved = errno; + free(tmp_path); + free(final_path); + free(json); + return set_err(err, "origin write: open tmp failed", saved); + } + if (write(fd, json, jlen) != (ssize_t) jlen) { + int saved = errno ? errno : EIO; + close(fd); + unlink(tmp_path); + free(tmp_path); + free(final_path); + free(json); + return set_err(err, "origin write: write failed", saved); + } + if (fsync(fd) < 0) { + int saved = errno; + close(fd); + unlink(tmp_path); + free(tmp_path); + free(final_path); + free(json); + return set_err(err, "origin write: fsync failed", saved); + } + close(fd); + if (rename(tmp_path, final_path) < 0) { + int saved = errno; + unlink(tmp_path); + free(tmp_path); + free(final_path); + free(json); + return set_err(err, "origin write: rename failed", saved); + } + + free(tmp_path); + free(final_path); + free(json); + return 0; +} + +void oci_origin_free(oci_origin_t *o) +{ + if (!o) + return; + free(o->manifest_digest); + free(o->config_digest); + if (o->layer_diffids) { + for (char **p = o->layer_diffids; *p; p++) + free(*p); + free((void *) o->layer_diffids); + } + o->manifest_digest = NULL; + o->config_digest = NULL; + o->layer_diffids = NULL; +} + +/* Slurp / into a heap buffer. Returns the + * buffer (NUL-terminated for safety against cJSON) on success and + * writes the byte length into *out_len; returns NULL on failure with + * errno and *err populated. + */ +static char *slurp_origin(const char *root_dir, + size_t *out_len, + const char **err) +{ + char *path = build_path(root_dir, OCI_ORIGIN_FILE, 0); + if (!path) { + set_err(err, "origin read: path alloc failed", ENOMEM); + return NULL; + } + int fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) { + int saved = errno; + set_err(err, "origin read: open failed", saved); + free(path); + return NULL; + } + free(path); + struct stat st; + if (fstat(fd, &st) < 0) { + int saved = errno; + close(fd); + set_err(err, "origin read: fstat failed", saved); + return NULL; + } + if (st.st_size <= 0 || (unsigned long) st.st_size > OCI_ORIGIN_MAX_BYTES) { + close(fd); + set_err(err, "origin read: file size out of bounds", EINVAL); + return NULL; + } + size_t len = (size_t) st.st_size; + char *buf = malloc(len + 1); + if (!buf) { + close(fd); + set_err(err, "origin read: alloc failed", ENOMEM); + return NULL; + } + size_t off = 0; + while (off < len) { + ssize_t got = read(fd, buf + off, len - off); + if (got < 0) { + if (errno == EINTR) + continue; + int saved = errno; + free(buf); + close(fd); + set_err(err, "origin read: read failed", saved); + return NULL; + } + if (got == 0) + break; + off += (size_t) got; + } + close(fd); + if (off != len) { + free(buf); + set_err(err, "origin read: short read", EIO); + return NULL; + } + buf[len] = '\0'; + *out_len = len; + return buf; +} + +int oci_origin_read(const char *root_dir, oci_origin_t *out, const char **err) +{ + static const char *dummy_err; + if (!err) + err = &dummy_err; + *err = NULL; + if (!root_dir || !out) + return set_err(err, "origin read: NULL argument", EINVAL); + if (!root_dir[0]) + return set_err(err, "origin read: empty root_dir", EINVAL); + out->manifest_digest = NULL; + out->config_digest = NULL; + out->layer_diffids = NULL; + + size_t json_len = 0; + char *json = slurp_origin(root_dir, &json_len, err); + if (!json) + return -1; + + cJSON *root = cJSON_ParseWithLength(json, json_len); + free(json); + if (!root) + return set_err(err, "origin read: JSON parse failed", EINVAL); + + const cJSON *m_field = + cJSON_GetObjectItemCaseSensitive(root, "manifest_digest"); + const cJSON *c_field = + cJSON_GetObjectItemCaseSensitive(root, "config_digest"); + const cJSON *l_field = + cJSON_GetObjectItemCaseSensitive(root, "layer_diffids"); + if (!cJSON_IsString(m_field) || !m_field->valuestring || + !m_field->valuestring[0]) { + cJSON_Delete(root); + return set_err(err, "origin read: missing manifest_digest field", + EINVAL); + } + if (!cJSON_IsString(c_field) || !c_field->valuestring || + !c_field->valuestring[0]) { + cJSON_Delete(root); + return set_err(err, "origin read: missing config_digest field", EINVAL); + } + if (!cJSON_IsArray(l_field)) { + cJSON_Delete(root); + return set_err(err, "origin read: missing layer_diffids array", EINVAL); + } + + int n_diffs = cJSON_GetArraySize(l_field); + if (n_diffs < 0) { + cJSON_Delete(root); + return set_err(err, "origin read: layer_diffids size invalid", EINVAL); + } + + /* +1 slot for the NULL terminator so callers can iterate with the + * idiomatic `for (p = arr; *p; p++)` pattern. + */ + void *diffs_raw = calloc((size_t) n_diffs + 1, sizeof(char *)); + if (!diffs_raw) { + cJSON_Delete(root); + return set_err(err, "origin read: diff array alloc failed", ENOMEM); + } + char **diffs = (char **) diffs_raw; + for (int i = 0; i < n_diffs; i++) { + const cJSON *entry = cJSON_GetArrayItem(l_field, i); + if (!cJSON_IsString(entry) || !entry->valuestring || + !entry->valuestring[0]) { + for (int k = 0; k < i; k++) + free(diffs[k]); + free((void *) diffs); + cJSON_Delete(root); + return set_err(err, + "origin read: layer_diffids entry is not a " + "non-empty string", + EINVAL); + } + diffs[i] = strdup(entry->valuestring); + if (!diffs[i]) { + for (int k = 0; k < i; k++) + free(diffs[k]); + free((void *) diffs); + cJSON_Delete(root); + return set_err(err, "origin read: diff strdup failed", ENOMEM); + } + } + + char *m_copy = strdup(m_field->valuestring); + char *c_copy = strdup(c_field->valuestring); + cJSON_Delete(root); + if (!m_copy || !c_copy) { + free(m_copy); + free(c_copy); + for (int k = 0; k < n_diffs; k++) + free(diffs[k]); + free((void *) diffs); + return set_err(err, "origin read: digest strdup failed", ENOMEM); + } + + out->manifest_digest = m_copy; + out->config_digest = c_copy; + out->layer_diffids = diffs; + return 0; +} diff --git a/src/oci/origin-meta.h b/src/oci/origin-meta.h new file mode 100644 index 0000000..90aeea4 --- /dev/null +++ b/src/oci/origin-meta.h @@ -0,0 +1,71 @@ +/* OCI unpacked-tree provenance sidecar + * + * Records which manifest produced an unpacked image directory so the + * Plan 1 garbage collector can walk unpacked sysroots and recover the + * full set of blobs (manifest, image-config, layer tars + diff-id + * pre-images) still referenced by on-disk state. Without this file the + * mark phase has no way to attribute an unpacked tree back to a stored + * manifest, and a prune sweep would happily delete layer blobs that are + * still backing a live sysroot. + * + * Serialization format: /.elfuse-origin.json with the shape + * { "manifest_digest": "sha256:...", + * "config_digest": "sha256:...", + * "layer_diffids": ["sha256:...", "sha256:..."] } + * + * The diff_ids come from the image-config blob's rootfs.diff_ids field, + * not from the manifest's layer descriptors: per the OCI image spec a + * diff_id is the digest of the uncompressed layer tar, while the + * manifest's layer.digest references the (possibly compressed) blob on + * disk. Recording both lets C1.2's root-set walker map each unpacked + * tree back to every blob it depends on regardless of layer media + * type. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +/* Parsed contents of an unpacked image tree's .elfuse-origin.json. All + * three fields are heap-allocated and owned by the struct: free via + * oci_origin_free. layer_diffids is a NULL-terminated array of + * ":" strings; the array itself is malloc'd and so is every + * entry. An image with zero layers parses to a one-element array + * containing only the NULL terminator. + */ +typedef struct { + char *manifest_digest; + char *config_digest; + char **layer_diffids; +} oci_origin_t; + +/* Write /.elfuse-origin.json via atomic rename. manifest_digest + * and config_digest are NUL-terminated ":" strings; diff_ids + * is a NULL-terminated array of the same form (an empty array is + * permitted and serializes to []). Returns 0 on success, -1 on failure + * with errno set and *err pointing to a static diagnostic. err may be + * NULL. + */ +int oci_origin_write(const char *root_dir, + const char *manifest_digest, + const char *config_digest, + char *const *diff_ids, + const char **err); + +/* Read /.elfuse-origin.json into *out. The struct is + * populated only on success; on failure out is left zeroed. Validates + * that manifest_digest and config_digest are present and string-typed + * and that layer_diffids is an array of strings; missing or + * mistyped fields surface as -1 with errno=EINVAL so the C1.3 garbage + * collector treats a malformed sidecar as a fatal root-set hole rather + * than silently dropping the tree's blobs from the keep set. Returns 0 + * on success, -1 on failure with errno set and *err pointing to a + * static diagnostic. err may be NULL. + */ +int oci_origin_read(const char *root_dir, oci_origin_t *out, const char **err); + +/* Release every heap field in o and zero the struct. Safe on a + * zero-initialised struct and on NULL. + */ +void oci_origin_free(oci_origin_t *o); diff --git a/src/oci/path-resolve.c b/src/oci/path-resolve.c new file mode 100644 index 0000000..9304872 --- /dev/null +++ b/src/oci/path-resolve.c @@ -0,0 +1,405 @@ +/* OCI guest PATH resolver: argv0 + PATH + cwd_guest -> host_path/guest_path + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Implementation notes that did not fit in the header: + * + * The candidate probe order matters. Each PATH entry is probed in turn. + * The first entry whose candidate exists, contains, and is executable + * wins (returns success immediately, like execvp). If no entry succeeds + * but at least one entry was found-and-contained-but-not-executable, the + * call surfaces EACCES with the rejected argv0 quoted. Otherwise the + * call surfaces ENOENT with the directories actually probed listed in + * the diagnostic so the operator can tell whether PATH itself was wrong + * or whether the binary just is not in the image. + * + * "Probed" specifically means "the candidate had a realpath that landed + * inside the sysroot". Escape symlinks and broken chains do NOT show up + * in the searched-dirs annotation: they were directories that exist on + * the guest side but contributed no host candidate, which is closer to + * what an operator who reads the error wants to see. + * + * realpath(3) is used once for sysroot_dir (so the prefix is canonical + * regardless of whether the caller passed it with or without trailing + * slashes or with intermediate symlinks) and once per probed candidate + * (so the containment check sees the post-symlink path). Both calls + * pass NULL for the resolved-path argument and let libc allocate. + */ + +#include "path-resolve.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Diagnostic scratch shared across the module. Thread-local so a future + * multiplexed oci run that probes several launches in parallel does not + * trample the err pointer between calls. + */ +static _Thread_local char path_err_buf[1024]; + +static void set_err_static(const char **err, const char *msg) +{ + if (err) + *err = msg; +} + +static void set_err_fmt(const char **err, const char *fmt, ...) + __attribute__((format(printf, 2, 3))); + +static void set_err_fmt(const char **err, const char *fmt, ...) +{ + va_list ap; + va_start(ap, fmt); + vsnprintf(path_err_buf, sizeof(path_err_buf), fmt, ap); + va_end(ap); + if (err) + *err = path_err_buf; +} + +/* Geometric-grow string buffer used to assemble the searched-dirs list + * for the ENOENT diagnostic. Keeping it separate from runspec's strvec + * because the surface area here is small enough that introducing a + * cross-module utility header would be over-engineering for a two-call + * use case. + */ +typedef struct { + char *buf; + size_t len; + size_t cap; +} sbuf_t; + +static int sbuf_append(sbuf_t *s, const char *str) +{ + size_t add = strlen(str); + if (s->len + add + 1 > s->cap) { + size_t newcap = s->cap ? s->cap : 64; + while (newcap < s->len + add + 1) + newcap *= 2; + char *np = realloc(s->buf, newcap); + if (!np) + return -1; + s->buf = np; + s->cap = newcap; + } + memcpy(s->buf + s->len, str, add); + s->len += add; + s->buf[s->len] = '\0'; + return 0; +} + +/* Concatenate two path components, normalising the slash boundary. NULL + * inputs propagate as NULL so call sites can chain without an extra + * branch per step. + */ +static char *path_join(const char *a, const char *b) +{ + if (!a || !b) + return NULL; + size_t alen = strlen(a); + size_t blen = strlen(b); + char *r = malloc(alen + blen + 2); + if (!r) + return NULL; + bool a_has_trailing = alen > 0 && a[alen - 1] == '/'; + bool b_has_leading = blen > 0 && b[0] == '/'; + if (a_has_trailing && b_has_leading) { + memcpy(r, a, alen); + memcpy(r + alen, b + 1, blen - 1); + r[alen + blen - 1] = '\0'; + } else if (!a_has_trailing && !b_has_leading && alen > 0 && blen > 0) { + memcpy(r, a, alen); + r[alen] = '/'; + memcpy(r + alen + 1, b, blen); + r[alen + 1 + blen] = '\0'; + } else { + memcpy(r, a, alen); + memcpy(r + alen, b, blen); + r[alen + blen] = '\0'; + } + return r; +} + +/* True when real_candidate equals real_sysroot or sits below it. Both + * arguments are absolute paths produced by realpath(3) so no trailing + * slash normalisation is needed on the candidate side; real_sysroot + * keeps any trailing slash stripped (realpath never adds one, but the + * defensive check below handles a caller that mutated it). + */ +static bool path_within_sysroot(const char *real_candidate, + const char *real_sysroot) +{ + size_t slen = strlen(real_sysroot); + while (slen > 1 && real_sysroot[slen - 1] == '/') + slen--; + if (strncmp(real_candidate, real_sysroot, slen) != 0) + return false; + char trailing = real_candidate[slen]; + return trailing == '\0' || trailing == '/'; +} + +enum probe_result { + PROBE_OK = 0, + PROBE_NOEXEC = 1, + PROBE_ESCAPE = 2, + PROBE_MISS = 3, +}; + +/* Probe one host candidate. The realpath()-based containment check runs + * first so escape symlinks never reach the stat() call. The stat() + * itself follows symlinks (POSIX stat semantics), matching how execvp + * would observe the candidate. + */ +static enum probe_result probe_candidate(const char *host_path, + const char *real_sysroot) +{ + char *real = realpath(host_path, NULL); + if (!real) + return PROBE_MISS; + bool contained = path_within_sysroot(real, real_sysroot); + free(real); + if (!contained) + return PROBE_ESCAPE; + struct stat st; + if (stat(host_path, &st) < 0) + return PROBE_MISS; + if (!S_ISREG(st.st_mode)) + return PROBE_MISS; + if ((st.st_mode & 0111) == 0) + return PROBE_NOEXEC; + return PROBE_OK; +} + +/* Direct-mode resolve: argv0 contains '/' so the PATH is bypassed. + * Absolute argv0 maps to ; relative argv0 anchors to + * cwd_guest. + */ +static int resolve_direct(const char *real_sysroot, + const char *argv0, + const char *cwd_guest, + char **out_host_path, + char **out_guest_path, + const char **err) +{ + char *guest_path = NULL; + if (argv0[0] == '/') + guest_path = strdup(argv0); + else + guest_path = path_join(cwd_guest, argv0); + if (!guest_path) { + set_err_static(err, "out of memory building guest path"); + errno = ENOMEM; + return -1; + } + char *host_path = path_join(real_sysroot, guest_path); + if (!host_path) { + free(guest_path); + set_err_static(err, "out of memory building host path"); + errno = ENOMEM; + return -1; + } + enum probe_result probe = probe_candidate(host_path, real_sysroot); + if (probe == PROBE_OK) { + *out_host_path = host_path; + *out_guest_path = guest_path; + return 0; + } + if (probe == PROBE_NOEXEC) { + set_err_fmt(err, "'%s' is not executable inside sysroot", argv0); + errno = EACCES; + } else if (probe == PROBE_ESCAPE) { + set_err_fmt(err, "'%s' resolves outside sysroot (refusing escape)", + argv0); + errno = ENOENT; + } else { + set_err_fmt(err, "cannot find '%s' inside sysroot", argv0); + errno = ENOENT; + } + free(host_path); + free(guest_path); + return -1; +} + +/* PATH-search resolve: walk path_env in order, returning the first + * executable candidate. Tracks the first NOEXEC for the late EACCES + * diagnostic and a sbuf_t of probed directories for the late ENOENT + * diagnostic. + */ +static int resolve_path_search(const char *real_sysroot, + const char *argv0, + const char *path_env, + const char *cwd_guest, + char **out_host_path, + char **out_guest_path, + const char **err) +{ + if (!path_env || !*path_env) { + set_err_fmt(err, + "cannot find '%s' on PATH inside sysroot (PATH is" + " empty)", + argv0); + errno = ENOENT; + return -1; + } + + sbuf_t searched = {0}; + char *first_noexec_argv0 = NULL; + int rc = -1; + + const char *p = path_env; + while (*p) { + const char *colon = strchr(p, ':'); + size_t elen = colon ? (size_t) (colon - p) : strlen(p); + char entry[PATH_MAX]; + if (elen >= sizeof(entry)) { + p = colon ? colon + 1 : p + elen; + continue; + } + memcpy(entry, p, elen); + entry[elen] = '\0'; + p = colon ? colon + 1 : p + elen; + + char *guest_dir; + if (elen == 0) + guest_dir = strdup(cwd_guest); + else if (entry[0] == '/') + guest_dir = strdup(entry); + else + guest_dir = path_join(cwd_guest, entry); + if (!guest_dir) { + set_err_static(err, "out of memory building PATH entry"); + errno = ENOMEM; + goto cleanup; + } + + char *guest_path = path_join(guest_dir, argv0); + char *host_path = + guest_path ? path_join(real_sysroot, guest_path) : NULL; + if (!guest_path || !host_path) { + free(guest_path); + free(host_path); + free(guest_dir); + set_err_static(err, "out of memory building host candidate"); + errno = ENOMEM; + goto cleanup; + } + + enum probe_result probe = probe_candidate(host_path, real_sysroot); + if (probe == PROBE_OK) { + *out_host_path = host_path; + *out_guest_path = guest_path; + free(guest_dir); + rc = 0; + goto cleanup; + } + if (probe == PROBE_NOEXEC) { + if (searched.len) + if (sbuf_append(&searched, ":") < 0) + goto oom_in_loop; + if (sbuf_append(&searched, guest_dir) < 0) + goto oom_in_loop; + if (!first_noexec_argv0) + first_noexec_argv0 = guest_path; + else + free(guest_path); + free(host_path); + free(guest_dir); + continue; + } + if (probe == PROBE_MISS) { + if (searched.len) + if (sbuf_append(&searched, ":") < 0) + goto oom_in_loop_full; + if (sbuf_append(&searched, guest_dir) < 0) + goto oom_in_loop_full; + } + free(guest_path); + free(host_path); + free(guest_dir); + continue; + + oom_in_loop_full: + free(guest_path); + oom_in_loop: + free(host_path); + free(guest_dir); + set_err_static(err, "out of memory recording PATH entry"); + errno = ENOMEM; + goto cleanup; + } + + if (first_noexec_argv0) { + set_err_fmt(err, "'%s' is not executable inside sysroot", argv0); + errno = EACCES; + free(first_noexec_argv0); + } else { + set_err_fmt(err, + "cannot find '%s' on PATH inside sysroot (searched:" + " %s)", + argv0, searched.buf ? searched.buf : ""); + errno = ENOENT; + } + +cleanup: + /* On success the saved first_noexec_argv0 has already been freed (it + * stays NULL because the OK branch hit first). On failure with no + * NOEXEC seen the same logic applies. The dual-free pattern here + * avoids leaking when a PATH miss happened to also have a noexec + * earlier in the walk. + */ + if (rc == 0 && first_noexec_argv0) + free(first_noexec_argv0); + free(searched.buf); + return rc; +} + +int oci_path_resolve(const char *sysroot_dir, + const char *argv0, + const char *path_env, + const char *cwd_guest, + char **out_host_path, + char **out_guest_path, + const char **err) +{ + if (out_host_path) + *out_host_path = NULL; + if (out_guest_path) + *out_guest_path = NULL; + if (err) + *err = NULL; + + if (!sysroot_dir || !argv0 || !*argv0 || !out_host_path || + !out_guest_path) { + set_err_static(err, "oci_path_resolve: NULL argument or empty argv0"); + errno = EINVAL; + return -1; + } + if (!cwd_guest || !*cwd_guest) + cwd_guest = "/"; + + char *real_sysroot = realpath(sysroot_dir, NULL); + if (!real_sysroot) { + set_err_fmt(err, "sysroot not accessible: %s", sysroot_dir); + /* errno preserved by realpath */ + return -1; + } + + int rc; + if (strchr(argv0, '/')) { + rc = resolve_direct(real_sysroot, argv0, cwd_guest, out_host_path, + out_guest_path, err); + } else { + rc = resolve_path_search(real_sysroot, argv0, path_env, cwd_guest, + out_host_path, out_guest_path, err); + } + + free(real_sysroot); + return rc; +} diff --git a/src/oci/path-resolve.h b/src/oci/path-resolve.h new file mode 100644 index 0000000..d779f8b --- /dev/null +++ b/src/oci/path-resolve.h @@ -0,0 +1,78 @@ +/* OCI guest PATH resolver inside a cloned rootfs + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Resolves a guest argv[0] against the merged guest PATH while keeping the + * lookup contained inside the per-run cloned rootfs. The function is the + * pre-launch bridge between oci_runspec_build (Phase 3 commit 2, which + * decides what argv and what PATH the guest should see) and elfuse_launch + * (Phase 3 commit 4, which expects a host filesystem path to open): + * + * - host_path is the absolute host path elfuse should open() to load + * the guest binary. It is the candidate exactly as found via PATH or + * directly via argv0; symlinks are NOT collapsed because the guest + * loader (and any execve() the guest runs internally) want to see the + * name they were invoked under, not the real path. Containment checks + * do use realpath internally; only the result handed back to the caller + * stays in symlink form. + * + * - guest_path is the guest-absolute path the guest itself thinks it is + * running (for argv0[0], for /proc/self/exe, for whatever a tool wants + * to learn about its own name). For PATH-search results it is + * /; for direct-mode results it is argv0 itself + * (absolute) or cwd_guest/argv0 (relative with '/'). + * + * Containment policy: every candidate path is fed to realpath(3) and the + * resolved absolute path must equal sysroot_dir or start with + * sysroot_dir + '/'. Symlink chains that resolve outside the sysroot are + * silently skipped (the PATH search continues) so a malicious or sloppy + * image layer cannot trick elfuse into loading a host-side binary. This + * matches how Docker's runc treats escape symlinks: drop them from the + * search instead of failing the entire launch. + * + * Executability is decided by host stat(2) (follows symlinks) against + * st_mode & 0111. PATH search records the first found-but-not-executable + * candidate and surfaces EACCES if no later entry succeeds, mirroring + * execvp's "first noexec wins" behaviour. + * + * The module deliberately does NOT reuse src/syscall/path.c's + * path_translate_at: that resolver is tied to the running guest's live + * sysroot/cwd plumbing, while this resolver runs before the vCPU starts + * and therefore needs a self-contained containment check. + */ + +#pragma once + +/* Resolve argv0 against PATH inside sysroot_dir. + * + * sysroot_dir must exist and be a directory; it is realpath'd once at + * entry so the containment check is stable across symlinks in the + * sysroot prefix itself. argv0 follows POSIX execvp semantics: when it + * contains '/' the PATH is bypassed and argv0 is resolved directly + * (absolute argv0 as a guest-absolute path, relative argv0 anchored to + * cwd_guest). When argv0 has no '/', path_env is split on ':' and each + * entry is treated as a guest-absolute directory (empty entries fall + * back to cwd_guest, matching POSIX). + * + * cwd_guest may be NULL; "/" is used in that case. path_env may be NULL + * or empty for the no-slash argv0 path -- the result is then a clean + * ENOENT with an empty searched-dirs annotation. + * + * On success returns 0 and writes heap-allocated *out_host_path and + * *out_guest_path. The caller frees both. On failure returns -1 with + * errno set (ENOENT for "not found", EACCES for "found but not + * executable", EINVAL for argument errors, ENOMEM for allocation) and + * leaves *out_host_path / *out_guest_path NULL. *err points at a + * diagnostic string; the pointer is valid until the next call from this + * thread. The diagnostic carries argv0 verbatim (quoted) and, for PATH + * search misses, a colon-separated list of the directories that were + * actually probed. + */ +int oci_path_resolve(const char *sysroot_dir, + const char *argv0, + const char *path_env, + const char *cwd_guest, + char **out_host_path, + char **out_guest_path, + const char **err); diff --git a/src/oci/policy.c b/src/oci/policy.c new file mode 100644 index 0000000..ba452ba --- /dev/null +++ b/src/oci/policy.c @@ -0,0 +1,1240 @@ +/* OCI policy.json schema and loader (C6.1) + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Walks the documented config-path chain, parses the JSON body via the + * vendored cJSON, and produces an oci_policy_t the fetcher consults via + * oci_policy_lookup. Implementation notes: + * + * - The loader is lenient on unknown keys (top-level and per-host). Each + * unknown key is recorded so a future audit / debug surface can + * surface it; the load itself never fails because of a key the + * reader has not learned about yet. This is what makes the C6.3 + * sigstore.publicKey reservation work without a coordinated rollout. + * - Known fields with wrong types are a hard error. The diagnostic + * spells out the JSON pointer-like path so an operator can fix the + * offending node directly. + * - String fields starting with "~/" or equal to "~" expand against + * $HOME at load time. "~user/" forms pass through verbatim (no + * getpwnam dependency, no surprise expansions). Other paths stay as + * authored. + * - ca_bundle existence is checked at load time so a fetcher that + * consults the policy never races a deleted file mid-pull. auth_file + * existence and 0600 mode are deferred to C6.2 because they share + * failure-mode ergonomics with the fetcher's credential loader. + */ + +#include "policy.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../externals/cjson/cJSON.h" + +#define POLICY_ERR_CAP 512 + +/* has_* flags drive the C6.3 overlay field-level merge: an overlay file only + * carries the fields the operator chose to override, so the merge step must + * distinguish "field was declared (possibly as null)" from "field omitted". + * For base entries the flags are set as side-effects of parsing and the + * lookup path keeps using the NULL-pointer check it already had; the flags + * are only consulted during overlay merge. + */ +typedef struct { + char *host; + bool has_insecure; + bool insecure; + bool has_ca_bundle; + char *ca_bundle; + bool has_auth_file; + char *auth_file; + bool has_sigstore_public_key; + char *sigstore_public_key; + char **unknown_keys; + size_t n_unknown_keys; +} policy_entry_t; + +struct oci_policy { + bool default_insecure; + char *default_ca_bundle; + policy_entry_t *entries; + size_t n_entries; + char *source_path; + char **unknown_top_keys; + size_t n_unknown_top_keys; + char *err_buf; +}; + +static char *xstrdup(const char *s) +{ + if (!s) + return NULL; + char *r = strdup(s); + if (!r) + errno = ENOMEM; + return r; +} + +/* Set p->err_buf to a printf-formatted message and return -1. Allocates the + * scratch buffer lazily so a load that never errors out does not pay for + * one. The caller propagates the pointer through *err_msg. + */ +static int set_err(oci_policy_t *p, const char **err_msg, const char *fmt, ...) + __attribute__((format(printf, 3, 4))); + +static int set_err(oci_policy_t *p, const char **err_msg, const char *fmt, ...) +{ + if (p) { + if (!p->err_buf) { + p->err_buf = malloc(POLICY_ERR_CAP); + if (!p->err_buf) { + if (err_msg) + *err_msg = "out of memory formatting policy error"; + errno = ENOMEM; + return -1; + } + } + va_list ap; + va_start(ap, fmt); + vsnprintf(p->err_buf, POLICY_ERR_CAP, fmt, ap); + va_end(ap); + if (err_msg) + *err_msg = p->err_buf; + } else if (err_msg) { + *err_msg = "policy load failed"; + } + return -1; +} + +/* Compose a per-field diagnostic that adapts to whether the field came from + * the base policy (host-scoped) or a registries.d overlay (file-scoped). The + * "what" suffix is whatever follows the field name in the message; callers + * pre-format the strerror tail when they need one so this stays plain + * non-variadic. + */ +static int field_err(oci_policy_t *p, + const char *src_path, + const char *host, + const char *field, + const char *what, + const char **err_msg) +{ + if (src_path) + return set_err(p, err_msg, "policy overlay '%s': '%s' %s", src_path, + field, what); + return set_err(p, err_msg, "policy 'registries[\"%s\"].%s' %s", host, field, + what); +} + +/* Append s (copied) to *arr / *n. Returns 0 on success, -1 on ENOMEM. */ +static int strarr_push(char ***arr, size_t *n, const char *s) +{ + char **next = (char **) realloc((void *) *arr, (*n + 1) * sizeof(char *)); + if (!next) { + errno = ENOMEM; + return -1; + } + *arr = next; + next[*n] = xstrdup(s); + if (!next[*n]) + return -1; + (*n)++; + return 0; +} + +static void strarr_free(char **arr, size_t n) +{ + if (!arr) + return; + for (size_t i = 0; i < n; i++) + free(arr[i]); + free((void *) arr); +} + +static void entry_free(policy_entry_t *e) +{ + if (!e) + return; + free(e->host); + free(e->ca_bundle); + free(e->auth_file); + free(e->sigstore_public_key); + strarr_free(e->unknown_keys, e->n_unknown_keys); +} + +void oci_policy_free(oci_policy_t *p) +{ + if (!p) + return; + free(p->default_ca_bundle); + for (size_t i = 0; i < p->n_entries; i++) + entry_free(&p->entries[i]); + free(p->entries); + free(p->source_path); + strarr_free(p->unknown_top_keys, p->n_unknown_top_keys); + free(p->err_buf); + free(p); +} + +const char *oci_policy_source(const oci_policy_t *p) +{ + return p && p->source_path ? p->source_path : ""; +} + +/* Expand a "~/..." prefix against $HOME. Pure "~" maps to $HOME. Other + * inputs (including "~user/...") pass through verbatim. Returns a heap- + * owned string the caller frees, or NULL on ENOMEM / missing $HOME. + */ +static char *expand_home(const char *in) +{ + if (!in) { + errno = EINVAL; + return NULL; + } + if (in[0] != '~') + return xstrdup(in); + if (in[1] != '/' && in[1] != '\0') + return xstrdup(in); + const char *home = getenv("HOME"); + if (!home || !*home) { + errno = ENOENT; + return NULL; + } + if (in[1] == '\0') + return xstrdup(home); + /* in[1] == '/': join home + (in + 1). The "~" replacement removes one + * byte; the joined string occupies strlen(home) + strlen(in + 1) + 1. + */ + size_t hl = strlen(home); + size_t tl = strlen(in + 1); + char *r = malloc(hl + tl + 1); + if (!r) { + errno = ENOMEM; + return NULL; + } + memcpy(r, home, hl); + memcpy(r + hl, in + 1, tl); + r[hl + tl] = '\0'; + return r; +} + +/* Join two path components with a single '/'. Returns a heap string or + * NULL on ENOMEM. Either arg may be empty; an empty base just yields + * "/" + tail which is sufficient for the fallback chain (HOME/XDG are + * never empty when present). + */ +static char *path_join(const char *a, const char *b) +{ + size_t al = a ? strlen(a) : 0; + size_t bl = b ? strlen(b) : 0; + char *r = malloc(al + 1 + bl + 1); + if (!r) { + errno = ENOMEM; + return NULL; + } + if (al) + memcpy(r, a, al); + r[al] = '/'; + if (bl) + memcpy(r + al + 1, b, bl); + r[al + 1 + bl] = '\0'; + return r; +} + +/* Slurp a file into a heap buffer. Returns a NUL-terminated string and + * writes the byte count to *out_len. Caller frees. On failure returns + * NULL with errno preserved. + */ +static char *slurp_file(const char *path, size_t *out_len) +{ + int fd = open(path, O_RDONLY); + if (fd < 0) + return NULL; + struct stat st; + if (fstat(fd, &st) < 0) { + int e = errno; + close(fd); + errno = e; + return NULL; + } + if (!S_ISREG(st.st_mode)) { + close(fd); + errno = EINVAL; + return NULL; + } + if (st.st_size < 0 || (uint64_t) st.st_size >= (uint64_t) SIZE_MAX) { + close(fd); + errno = EFBIG; + return NULL; + } + size_t len = (size_t) st.st_size; + char *buf = malloc(len + 1); + if (!buf) { + close(fd); + errno = ENOMEM; + return NULL; + } + size_t off = 0; + while (off < len) { + ssize_t n = read(fd, buf + off, len - off); + if (n < 0) { + if (errno == EINTR) + continue; + int e = errno; + free(buf); + close(fd); + errno = e; + return NULL; + } + if (n == 0) + break; + off += (size_t) n; + } + close(fd); + buf[off] = '\0'; + if (out_len) + *out_len = off; + return buf; +} + +/* Resolve the candidate path chain. On success writes a heap-owned + * absolute path into *out and returns 0; *out is NULL when no candidate + * exists (caller uses built-in default). Returns -1 on hard errors: + * $ELFUSE_POLICY_FILE points at a missing file, a fallback path fails to + * open with errno != ENOENT, or allocation fails. Diagnostic messages + * go through set_err on the caller-supplied policy. + */ +static int resolve_path(oci_policy_t *p, char **out, const char **err_msg) +{ + *out = NULL; + const char *env_override = getenv("ELFUSE_POLICY_FILE"); + if (env_override && *env_override) { + struct stat st; + if (stat(env_override, &st) < 0) { + int e = errno; + int rc = set_err(p, err_msg, + "ELFUSE_POLICY_FILE='%s' does not exist: %s", + env_override, strerror(e)); + errno = e; + return rc; + } + if (!S_ISREG(st.st_mode)) { + errno = EINVAL; + return set_err(p, err_msg, + "ELFUSE_POLICY_FILE='%s' is not a regular file", + env_override); + } + *out = xstrdup(env_override); + if (!*out) + return set_err(p, err_msg, + "out of memory recording ELFUSE_POLICY_FILE path"); + return 0; + } + + const char *home = getenv("HOME"); + + /* Fallback 1: XDG_CONFIG_HOME or $HOME/.config */ + const char *xdg = getenv("XDG_CONFIG_HOME"); + char *xdg_root = NULL; + if (xdg && *xdg) { + xdg_root = xstrdup(xdg); + } else if (home && *home) { + xdg_root = path_join(home, ".config"); + } + if (xdg_root) { + char *elf_dir = path_join(xdg_root, "elfuse"); + free(xdg_root); + char *candidate = NULL; + if (elf_dir) { + candidate = path_join(elf_dir, "policy.json"); + free(elf_dir); + } + if (!candidate) + return set_err(p, err_msg, + "out of memory composing XDG policy path"); + struct stat st; + if (stat(candidate, &st) == 0 && S_ISREG(st.st_mode)) { + *out = candidate; + return 0; + } + if (errno != ENOENT) { + int e = errno; + int rc = set_err(p, err_msg, + "policy candidate '%s' could not be stat'd: %s", + candidate, strerror(e)); + free(candidate); + errno = e; + return rc; + } + free(candidate); + } + + /* Fallback 2: $HOME/Library/Application Support/elfuse/policy.json */ + if (home && *home) { + const char *suffix = "/Library/Application Support/elfuse/policy.json"; + size_t n = strlen(home) + strlen(suffix) + 1; + char *candidate = malloc(n); + if (!candidate) { + errno = ENOMEM; + return set_err(p, err_msg, + "out of memory composing Library policy path"); + } + snprintf(candidate, n, "%s%s", home, suffix); + struct stat st; + if (stat(candidate, &st) == 0 && S_ISREG(st.st_mode)) { + *out = candidate; + return 0; + } + if (errno != ENOENT) { + int e = errno; + int rc = set_err(p, err_msg, + "policy candidate '%s' could not be stat'd: %s", + candidate, strerror(e)); + free(candidate); + errno = e; + return rc; + } + free(candidate); + } + + /* Empty chain: caller falls back to built-in default. */ + return 0; +} + +/* Type checks used while walking the schema. cJSON treats true/false as + * separate node types so the unified bool predicate calls both probes. + */ +static bool json_is_bool(const cJSON *n) +{ + return n && (cJSON_IsBool(n) || cJSON_IsTrue(n) || cJSON_IsFalse(n)); +} + +static bool known_default_key(const char *k) +{ + return !strcmp(k, "insecure") || !strcmp(k, "ca_bundle"); +} + +static bool known_entry_key(const char *k) +{ + return !strcmp(k, "insecure") || !strcmp(k, "ca_bundle") || + !strcmp(k, "auth_file") || !strcmp(k, "sigstore"); +} + +static bool known_top_key(const char *k) +{ + return !strcmp(k, "default") || !strcmp(k, "registries"); +} + +/* Parse a "default" block. Missing block leaves the policy on its + * zero-value defaults. Bad shapes raise hard errors via set_err. + */ +static int parse_default_block(oci_policy_t *p, + cJSON *node, + const char **err_msg) +{ + if (!cJSON_IsObject(node)) + return set_err(p, err_msg, "policy 'default' must be a JSON object"); + cJSON *child; + cJSON_ArrayForEach(child, node) + { + const char *k = child->string; + if (!k) + continue; + if (!strcmp(k, "insecure")) { + if (!json_is_bool(child)) + return set_err(p, err_msg, + "policy 'default.insecure' must be boolean"); + p->default_insecure = cJSON_IsTrue(child); + } else if (!strcmp(k, "ca_bundle")) { + if (cJSON_IsNull(child)) { + free(p->default_ca_bundle); + p->default_ca_bundle = NULL; + } else if (cJSON_IsString(child) && child->valuestring) { + char *expanded = expand_home(child->valuestring); + if (!expanded) + return set_err( + p, err_msg, + "policy 'default.ca_bundle' expansion failed: %s", + strerror(errno)); + free(p->default_ca_bundle); + p->default_ca_bundle = expanded; + } else { + return set_err( + p, err_msg, + "policy 'default.ca_bundle' must be a string or null"); + } + } else if (!known_default_key(k)) { + /* Forward-compat: future shaped defaults silently accepted. + * No record on the default block; the per-host slot does. + */ + } + } + if (p->default_ca_bundle) { + struct stat st; + if (stat(p->default_ca_bundle, &st) < 0 || !S_ISREG(st.st_mode)) + return set_err( + p, err_msg, + "policy 'default.ca_bundle' file '%s' is not accessible", + p->default_ca_bundle); + } + return 0; +} + +/* Parse a "sigstore" sub-object. Only publicKey is read; other keys go onto + * the parent entry's unknown_keys list with a "sigstore." prefix so the + * diagnostic stays unambiguous. src_path is NULL for base policy parsing and + * the overlay file path for the registries.d path; field_err picks the right + * shape. + */ +static int parse_sigstore_fields(oci_policy_t *p, + policy_entry_t *e, + cJSON *node, + const char *src_path, + const char **err_msg) +{ + if (!cJSON_IsObject(node)) + return field_err(p, src_path, e->host, "sigstore", + "must be a JSON object", err_msg); + cJSON *child; + cJSON_ArrayForEach(child, node) + { + const char *k = child->string; + if (!k) + continue; + if (!strcmp(k, "publicKey")) { + if (!cJSON_IsString(child) || !child->valuestring) + return field_err(p, src_path, e->host, "sigstore.publicKey", + "must be a string", err_msg); + char *expanded = expand_home(child->valuestring); + if (!expanded) { + char what[256]; + snprintf(what, sizeof(what), "expansion failed: %s", + strerror(errno)); + return field_err(p, src_path, e->host, "sigstore.publicKey", + what, err_msg); + } + free(e->sigstore_public_key); + e->sigstore_public_key = expanded; + e->has_sigstore_public_key = true; + } else { + char composed[256]; + snprintf(composed, sizeof(composed), "sigstore.%s", k); + if (strarr_push(&e->unknown_keys, &e->n_unknown_keys, composed) < + 0) { + if (src_path) + return set_err(p, err_msg, + "policy overlay '%s' sigstore unknown-key " + "recording failed", + src_path); + return set_err(p, err_msg, + "policy 'registries[\"%s\"].sigstore' " + "unknown-key recording failed", + e->host); + } + } + } + return 0; +} + +/* Shared field parser for per-host entries. Used by the base policy parser + * (src_path == NULL, e is the live array slot) and by the C6.3 overlay parser + * (src_path is the overlay file path, e is a scratch policy_entry_t the + * caller merges into the target). Only sets fields; the ca_bundle stat check + * is left to the caller so each context can tailor the diagnostic. + */ +static int parse_entry_fields(oci_policy_t *p, + policy_entry_t *e, + cJSON *node, + const char *src_path, + const char **err_msg) +{ + cJSON *child; + cJSON_ArrayForEach(child, node) + { + const char *k = child->string; + if (!k) + continue; + if (!strcmp(k, "insecure")) { + if (!json_is_bool(child)) + return field_err(p, src_path, e->host, "insecure", + "must be boolean", err_msg); + e->has_insecure = true; + e->insecure = cJSON_IsTrue(child); + } else if (!strcmp(k, "ca_bundle")) { + if (cJSON_IsNull(child)) { + free(e->ca_bundle); + e->ca_bundle = NULL; + e->has_ca_bundle = true; + } else if (cJSON_IsString(child) && child->valuestring) { + char *expanded = expand_home(child->valuestring); + if (!expanded) { + char what[256]; + snprintf(what, sizeof(what), "expansion failed: %s", + strerror(errno)); + return field_err(p, src_path, e->host, "ca_bundle", what, + err_msg); + } + free(e->ca_bundle); + e->ca_bundle = expanded; + e->has_ca_bundle = true; + } else { + return field_err(p, src_path, e->host, "ca_bundle", + "must be a string or null", err_msg); + } + } else if (!strcmp(k, "auth_file")) { + if (!cJSON_IsString(child) || !child->valuestring) + return field_err(p, src_path, e->host, "auth_file", + "must be a string", err_msg); + char *expanded = expand_home(child->valuestring); + if (!expanded) { + char what[256]; + snprintf(what, sizeof(what), "expansion failed: %s", + strerror(errno)); + return field_err(p, src_path, e->host, "auth_file", what, + err_msg); + } + free(e->auth_file); + e->auth_file = expanded; + e->has_auth_file = true; + } else if (!strcmp(k, "sigstore")) { + if (parse_sigstore_fields(p, e, child, src_path, err_msg) < 0) + return -1; + } else if (!known_entry_key(k)) { + if (strarr_push(&e->unknown_keys, &e->n_unknown_keys, k) < 0) { + if (src_path) + return set_err(p, err_msg, + "policy overlay '%s' unknown-key " + "recording failed", + src_path); + return set_err(p, err_msg, + "policy 'registries[\"%s\"]' " + "unknown-key recording failed", + e->host); + } + } + } + return 0; +} + +static int parse_entry_block(oci_policy_t *p, + policy_entry_t *e, + cJSON *node, + const char **err_msg) +{ + if (!cJSON_IsObject(node)) + return set_err(p, err_msg, + "policy 'registries[\"%s\"]' must be a JSON object", + e->host); + if (parse_entry_fields(p, e, node, NULL, err_msg) < 0) + return -1; + if (e->ca_bundle) { + struct stat st; + if (stat(e->ca_bundle, &st) < 0 || !S_ISREG(st.st_mode)) + return set_err(p, err_msg, + "policy 'registries[\"%s\"].ca_bundle' file '%s' " + "is not accessible", + e->host, e->ca_bundle); + } + return 0; +} + +static int parse_registries_block(oci_policy_t *p, + cJSON *node, + const char **err_msg) +{ + if (!cJSON_IsObject(node)) + return set_err(p, err_msg, "policy 'registries' must be a JSON object"); + size_t n = (size_t) cJSON_GetArraySize(node); + if (n == 0) + return 0; + p->entries = calloc(n, sizeof(policy_entry_t)); + if (!p->entries) { + errno = ENOMEM; + return set_err(p, err_msg, + "policy 'registries' entry allocation failed"); + } + cJSON *child; + cJSON_ArrayForEach(child, node) + { + const char *host = child->string; + if (!host) + continue; + policy_entry_t *e = &p->entries[p->n_entries]; + e->host = xstrdup(host); + if (!e->host) + return set_err(p, err_msg, + "policy 'registries[\"%s\"]' host copy failed", + host); + p->n_entries++; + if (parse_entry_block(p, e, child, err_msg) < 0) + return -1; + } + return 0; +} + +/* Find an existing entry by host, or grow the entries array by one and + * initialise a new slot. Returns the slot or NULL on ENOMEM. New slots have + * host set and all fields zeroed; the caller (overlay merge) fills them via + * merge_overlay_into_entry. On xstrdup failure for the new slot's host, the + * grown array stays allocated but n_entries is not bumped, so oci_policy_free + * walks the same n_entries it already had. + */ +static policy_entry_t *entry_grow_and_get(oci_policy_t *p, const char *host) +{ + for (size_t i = 0; i < p->n_entries; i++) { + if (p->entries[i].host && !strcmp(p->entries[i].host, host)) + return &p->entries[i]; + } + policy_entry_t *next = + realloc(p->entries, (p->n_entries + 1) * sizeof(*next)); + if (!next) { + errno = ENOMEM; + return NULL; + } + p->entries = next; + policy_entry_t *e = &p->entries[p->n_entries]; + memset(e, 0, sizeof(*e)); + e->host = xstrdup(host); + if (!e->host) + return NULL; + p->n_entries++; + return e; +} + +/* Move declared fields from an overlay-parsed scratch entry into the target + * entry, freeing whatever the target previously held. Pointer ownership + * transfers to the target; the overlay's pointers are nulled so the caller's + * entry_free does not double-free. unknown_keys are appended by copy via + * strarr_push -- the originals stay in the overlay for entry_free to release. + */ +static int merge_overlay_into_entry(policy_entry_t *tgt, + policy_entry_t *ov, + const char **err_msg) +{ + if (ov->has_insecure) { + tgt->has_insecure = true; + tgt->insecure = ov->insecure; + } + if (ov->has_ca_bundle) { + free(tgt->ca_bundle); + tgt->ca_bundle = ov->ca_bundle; + ov->ca_bundle = NULL; + tgt->has_ca_bundle = true; + } + if (ov->has_auth_file) { + free(tgt->auth_file); + tgt->auth_file = ov->auth_file; + ov->auth_file = NULL; + tgt->has_auth_file = true; + } + if (ov->has_sigstore_public_key) { + free(tgt->sigstore_public_key); + tgt->sigstore_public_key = ov->sigstore_public_key; + ov->sigstore_public_key = NULL; + tgt->has_sigstore_public_key = true; + } + for (size_t i = 0; i < ov->n_unknown_keys; i++) { + if (strarr_push(&tgt->unknown_keys, &tgt->n_unknown_keys, + ov->unknown_keys[i]) < 0) { + (void) err_msg; + errno = ENOMEM; + return -1; + } + } + return 0; +} + +/* Parse a single registries.d/.json overlay file and field-merge it + * into the target entry (created if absent). Failure leaves the target + * unchanged on any path past the merge call; failures before merge never + * touched the target. The overlay scratch entry is always released here. + */ +static int parse_overlay_file(oci_policy_t *p, + const char *host, + const char *file_path, + const char **err_msg) +{ + size_t body_len = 0; + char *body = slurp_file(file_path, &body_len); + if (!body) + return set_err(p, err_msg, "policy overlay '%s' could not be read: %s", + file_path, strerror(errno)); + cJSON *root = cJSON_ParseWithLength(body, body_len); + free(body); + if (!root) + return set_err(p, err_msg, "policy overlay '%s' is not valid JSON", + file_path); + + int rc = -1; + policy_entry_t overlay; + memset(&overlay, 0, sizeof(overlay)); + + if (!cJSON_IsObject(root)) { + (void) set_err(p, err_msg, "policy overlay '%s' must be a JSON object", + file_path); + goto out; + } + overlay.host = xstrdup(host); + if (!overlay.host) { + (void) set_err(p, err_msg, "out of memory parsing policy overlay '%s'", + file_path); + goto out; + } + if (parse_entry_fields(p, &overlay, root, file_path, err_msg) < 0) + goto out; + if (overlay.has_ca_bundle && overlay.ca_bundle) { + struct stat st; + if (stat(overlay.ca_bundle, &st) < 0 || !S_ISREG(st.st_mode)) { + (void) set_err(p, err_msg, + "policy overlay '%s': ca_bundle file '%s' " + "is not accessible", + file_path, overlay.ca_bundle); + goto out; + } + } + policy_entry_t *target = entry_grow_and_get(p, host); + if (!target) { + (void) set_err(p, err_msg, + "policy overlay '%s' target entry allocation failed", + file_path); + goto out; + } + if (merge_overlay_into_entry(target, &overlay, err_msg) < 0) { + (void) set_err(p, err_msg, + "policy overlay '%s' merge failed: out of memory", + file_path); + goto out; + } + rc = 0; +out: + entry_free(&overlay); + cJSON_Delete(root); + return rc; +} + +static int overlay_name_cmp(const void *a, const void *b) +{ + return strcmp(*(const char *const *) a, *(const char *const *) b); +} + +/* Scan /registries.d/ for *.json overlay files and merge each into + * the policy. The directory itself is optional: opendir returning ENOENT is + * silent; any other errno (ENOTDIR, EACCES, ...) is a hard error so an + * operator pointing at an unreadable overlay tree learns about it. Each + * filename minus the .json suffix is the target host. Files are processed + * in lexicographic order for determinism; same-host duplicates cannot exist + * on POSIX filesystems so the order is mostly observable in diagnostics. + */ +static int load_overlay_dir(oci_policy_t *p, + const char *base_path, + const char **err_msg) +{ + if (!base_path || !*base_path) + return 0; + const char *last_slash = strrchr(base_path, '/'); + if (!last_slash) + return 0; + size_t parent_len = (size_t) (last_slash - base_path); + static const char overlay_suffix[] = "/registries.d"; + size_t suffix_len = sizeof(overlay_suffix) - 1; + char *dir_path = malloc(parent_len + suffix_len + 1); + if (!dir_path) + return set_err(p, err_msg, + "out of memory composing policy overlay path"); + memcpy(dir_path, base_path, parent_len); + memcpy(dir_path + parent_len, overlay_suffix, suffix_len); + dir_path[parent_len + suffix_len] = '\0'; + + DIR *d = opendir(dir_path); + if (!d) { + int e = errno; + if (e == ENOENT) { + free(dir_path); + return 0; + } + int rc = set_err(p, err_msg, + "policy overlay directory '%s' cannot be opened: %s", + dir_path, strerror(e)); + free(dir_path); + errno = e; + return rc; + } + + char **names = NULL; + size_t n_names = 0; + size_t cap_names = 0; + struct dirent *de; + while ((de = readdir(d)) != NULL) { + const char *n = de->d_name; + if (n[0] == '.') + continue; + size_t nl = strlen(n); + if (nl <= 5) + continue; + if (strcmp(n + nl - 5, ".json") != 0) + continue; + if (n_names == cap_names) { + size_t new_cap = cap_names ? cap_names * 2 : 8; + char **next = + (char **) realloc((void *) names, new_cap * sizeof(char *)); + if (!next) { + closedir(d); + strarr_free(names, n_names); + int rc = set_err(p, err_msg, + "policy overlay '%s' name-list allocation " + "failed", + dir_path); + free(dir_path); + return rc; + } + names = next; + cap_names = new_cap; + } + names[n_names] = strdup(n); + if (!names[n_names]) { + closedir(d); + strarr_free(names, n_names); + int rc = set_err(p, err_msg, "policy overlay '%s' name copy failed", + dir_path); + free(dir_path); + return rc; + } + n_names++; + } + closedir(d); + + qsort((void *) names, n_names, sizeof(char *), overlay_name_cmp); + + int rc = 0; + for (size_t i = 0; i < n_names; i++) { + const char *fname = names[i]; + size_t nl = strlen(fname); + size_t host_len = nl - 5; /* trim ".json" */ + if (host_len == 0) + continue; /* literal ".json" filename: not a host */ + char *host = malloc(host_len + 1); + if (!host) { + rc = set_err(p, err_msg, + "out of memory composing overlay host name"); + goto cleanup; + } + memcpy(host, fname, host_len); + host[host_len] = '\0'; + size_t file_path_size = strlen(dir_path) + 1 + nl + 1; + char *file_path = malloc(file_path_size); + if (!file_path) { + free(host); + rc = set_err(p, err_msg, + "out of memory composing overlay file path"); + goto cleanup; + } + snprintf(file_path, file_path_size, "%s/%s", dir_path, fname); + struct stat st; + if (stat(file_path, &st) < 0 || !S_ISREG(st.st_mode)) { + /* Filename ending in .json that turned out not to be a regular + * file (e.g. a directory named "foo.json"). Silently skip: + * defensive, leaves base policy load undisturbed. + */ + free(file_path); + free(host); + continue; + } + rc = parse_overlay_file(p, host, file_path, err_msg); + free(file_path); + free(host); + if (rc < 0) + goto cleanup; + } + +cleanup: + strarr_free(names, n_names); + free(dir_path); + return rc; +} + +static int parse_body(oci_policy_t *p, + const char *body, + size_t body_len, + const char **err_msg) +{ + cJSON *root = cJSON_ParseWithLength(body, body_len); + if (!root) + return set_err(p, err_msg, "policy file '%s' is not valid JSON", + p->source_path ? p->source_path : "(stdin)"); + int rc = -1; + if (!cJSON_IsObject(root)) { + (void) set_err(p, err_msg, "policy root must be a JSON object"); + goto out; + } + cJSON *child; + cJSON_ArrayForEach(child, root) + { + const char *k = child->string; + if (!k) + continue; + if (!strcmp(k, "default")) { + if (parse_default_block(p, child, err_msg) < 0) + goto out; + } else if (!strcmp(k, "registries")) { + if (parse_registries_block(p, child, err_msg) < 0) + goto out; + } else if (!known_top_key(k)) { + if (strarr_push(&p->unknown_top_keys, &p->n_unknown_top_keys, k) < + 0) { + (void) set_err(p, err_msg, + "policy unknown-key recording failed"); + goto out; + } + } + } + rc = 0; +out: + cJSON_Delete(root); + return rc; +} + +int oci_policy_load(oci_policy_t **out, const char **err_msg) +{ + if (!out) { + if (err_msg) + *err_msg = "invalid arguments"; + errno = EINVAL; + return -1; + } + *out = NULL; + + oci_policy_t *p = calloc(1, sizeof(*p)); + if (!p) { + if (err_msg) + *err_msg = "out of memory allocating policy"; + errno = ENOMEM; + return -1; + } + p->err_buf = malloc(POLICY_ERR_CAP); + if (!p->err_buf) { + free(p); + if (err_msg) + *err_msg = "out of memory allocating policy diagnostic"; + errno = ENOMEM; + return -1; + } + p->err_buf[0] = '\0'; + + /* On every failure past this point, *out keeps the partially built + * policy so the caller can free it (and the err_buf the diagnostic + * lives in) via oci_policy_free. The policy.h contract requires + * exactly this shape for diagnostic lifetime. + */ + *out = p; + + char *path = NULL; + if (resolve_path(p, &path, err_msg) < 0) + return -1; + + if (!path) { + /* Empty chain: built-in default. source_path stays empty. */ + p->source_path = xstrdup(""); + if (!p->source_path) + return set_err(p, err_msg, "out of memory recording source path"); + return 0; + } + + p->source_path = path; /* takes ownership */ + + size_t body_len = 0; + char *body = slurp_file(p->source_path, &body_len); + if (!body) + return set_err(p, err_msg, "policy file '%s' could not be read: %s", + p->source_path, strerror(errno)); + int rc = parse_body(p, body, body_len, err_msg); + free(body); + if (rc < 0) + return rc; + return load_overlay_dir(p, p->source_path, err_msg); +} + +void oci_policy_lookup(const oci_policy_t *p, + const char *host, + oci_policy_effective_t *eff) +{ + if (!eff) + return; + eff->insecure = p ? p->default_insecure : false; + eff->ca_bundle = p ? p->default_ca_bundle : NULL; + eff->auth_file = NULL; + eff->sigstore_public_key = NULL; + if (!p || !host) + return; + for (size_t i = 0; i < p->n_entries; i++) { + const policy_entry_t *e = &p->entries[i]; + if (strcmp(e->host, host) != 0) + continue; + if (e->has_insecure) + eff->insecure = e->insecure; + if (e->ca_bundle) + eff->ca_bundle = e->ca_bundle; + if (e->auth_file) + eff->auth_file = e->auth_file; + if (e->sigstore_public_key) + eff->sigstore_public_key = e->sigstore_public_key; + return; + } +} + +/* Static error literals. The auth-file load path has no policy_t err_buf to + * share, and the diagnostic is short enough that a fixed table beats a + * dynamic format. The caller composes the final user-facing message with + * the path it already knows. + */ +static const char AUTH_ERR_OPEN[] = "auth file could not be opened"; +static const char AUTH_ERR_FSTAT[] = "auth file could not be stat'd"; +static const char AUTH_ERR_NOT_REG[] = "auth file is not a regular file"; +static const char AUTH_ERR_MODE[] = + "auth file has insecure mode (must be 0600)"; +static const char AUTH_ERR_TOO_BIG[] = "auth file is too large"; +static const char AUTH_ERR_READ[] = "auth file could not be read"; +static const char AUTH_ERR_NOMEM[] = "out of memory parsing auth file"; +static const char AUTH_ERR_BAD_JSON[] = "auth file is not valid JSON"; +static const char AUTH_ERR_NOT_OBJ[] = "auth file body must be a JSON object"; +static const char AUTH_ERR_NO_USER[] = + "auth file missing required string field 'username'"; +static const char AUTH_ERR_NO_PASS[] = + "auth file missing required string field 'password'"; + +int oci_policy_load_auth(const char *path, + char **out_user, + char **out_pass, + const char **err_msg) +{ + if (!path || !out_user || !out_pass) { + if (err_msg) + *err_msg = "invalid arguments"; + errno = EINVAL; + return -1; + } + *out_user = NULL; + *out_pass = NULL; + + int fd = open(path, O_RDONLY); + if (fd < 0) { + int e = errno; + if (err_msg) + *err_msg = AUTH_ERR_OPEN; + errno = e; + return -1; + } + struct stat st; + if (fstat(fd, &st) < 0) { + int e = errno; + close(fd); + if (err_msg) + *err_msg = AUTH_ERR_FSTAT; + errno = e; + return -1; + } + if (!S_ISREG(st.st_mode)) { + close(fd); + if (err_msg) + *err_msg = AUTH_ERR_NOT_REG; + errno = EINVAL; + return -1; + } + /* Mode must grant access to the owner only. 0600 is the canonical + * shape; 0400 (read-only) is also accepted. Any bit in the group or + * other triad fails the check. + */ + if ((st.st_mode & 077) != 0) { + close(fd); + if (err_msg) + *err_msg = AUTH_ERR_MODE; + errno = EPERM; + return -1; + } + if (st.st_size < 0 || (uint64_t) st.st_size >= (uint64_t) SIZE_MAX) { + close(fd); + if (err_msg) + *err_msg = AUTH_ERR_TOO_BIG; + errno = EFBIG; + return -1; + } + size_t len = (size_t) st.st_size; + char *buf = malloc(len + 1); + if (!buf) { + close(fd); + if (err_msg) + *err_msg = AUTH_ERR_NOMEM; + errno = ENOMEM; + return -1; + } + size_t off = 0; + while (off < len) { + ssize_t n = read(fd, buf + off, len - off); + if (n < 0) { + if (errno == EINTR) + continue; + int e = errno; + free(buf); + close(fd); + if (err_msg) + *err_msg = AUTH_ERR_READ; + errno = e; + return -1; + } + if (n == 0) + break; + off += (size_t) n; + } + close(fd); + buf[off] = '\0'; + + cJSON *json = cJSON_ParseWithLength(buf, off); + free(buf); + if (!json) { + if (err_msg) + *err_msg = AUTH_ERR_BAD_JSON; + errno = EINVAL; + return -1; + } + if (!cJSON_IsObject(json)) { + cJSON_Delete(json); + if (err_msg) + *err_msg = AUTH_ERR_NOT_OBJ; + errno = EINVAL; + return -1; + } + cJSON *ju = cJSON_GetObjectItemCaseSensitive(json, "username"); + if (!cJSON_IsString(ju) || !ju->valuestring) { + cJSON_Delete(json); + if (err_msg) + *err_msg = AUTH_ERR_NO_USER; + errno = EINVAL; + return -1; + } + cJSON *jp = cJSON_GetObjectItemCaseSensitive(json, "password"); + if (!cJSON_IsString(jp) || !jp->valuestring) { + cJSON_Delete(json); + if (err_msg) + *err_msg = AUTH_ERR_NO_PASS; + errno = EINVAL; + return -1; + } + *out_user = xstrdup(ju->valuestring); + *out_pass = xstrdup(jp->valuestring); + cJSON_Delete(json); + if (!*out_user || !*out_pass) { + if (err_msg) + *err_msg = AUTH_ERR_NOMEM; + errno = ENOMEM; + return -1; + } + return 0; +} diff --git a/src/oci/policy.h b/src/oci/policy.h new file mode 100644 index 0000000..bbb45d3 --- /dev/null +++ b/src/oci/policy.h @@ -0,0 +1,140 @@ +/* OCI policy.json schema and loader + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Plan 6 C6.1: parse a podman/skopeo-style policy.json out of one of the + * standard config locations and expose a per-host effective view that + * fetch.c (in C6.2) consults before applying CLI overrides. + * + * Load order: + * 1. $ELFUSE_POLICY_FILE (when set and non-empty) + * 2. $XDG_CONFIG_HOME/elfuse/policy.json (fallback: + * $HOME/.config/elfuse/policy.json) + * 3. $HOME/Library/Application Support/elfuse/policy.json + * 4. Built-in default (insecure=false, ca_bundle=NULL, no per-host entries) + * + * An $ELFUSE_POLICY_FILE that points at a missing file is a hard error so + * an operator that explicitly named a path always learns about typos. + * Missing fallback files silently fall through to the next candidate; + * a fully empty chain yields the built-in default with source_path == "". + * + * Supported schema subset (additional keys at any level are recorded for + * forward-compat diagnostics but never reject the load): + * + * { + * "default": { "insecure": bool, "ca_bundle": string|null }, + * "registries": { + * "": { + * "insecure": bool, + * "ca_bundle": string|null, + * "auth_file": string, + * "sigstore": { "publicKey": string } // C6.3 reservation; ignored + * }, + * ... + * } + * } + * + * String fields starting with "~/" or equal to "~" expand against $HOME at + * load time. Any other path passes through verbatim. ca_bundle is stat'd + * during load and a missing target is a hard error; auth_file is not + * accessed by the loader (the fetcher reads and mode-checks it in C6.2). + * + * Thread safety: oci_policy_t is read-only after load. Multiple threads may + * call oci_policy_lookup concurrently. The loader itself is not reentrant; + * one fetcher loads its own copy and frees on destruction. + */ + +#pragma once + +#include +#include + +typedef struct oci_policy oci_policy_t; + +/* Effective per-host view. Strings are owned by the parent oci_policy_t; + * callers must not free them or use them past oci_policy_free. A NULL + * string field means the policy did not declare a value at either the + * per-host entry or the default block, so the caller should fall back to + * whatever default it would otherwise use. + */ +typedef struct { + bool insecure; + const char *ca_bundle; + const char *auth_file; + /* C6.3 reservation: a registries[""].sigstore.publicKey field + * parses into this slot for forward-compat introspection. fetch.c in + * Plan 6 never reads it; a future sigstore-verify hook lights up + * after Phase 4+. + */ + const char *sigstore_public_key; +} oci_policy_effective_t; + +/* Load the policy, walking the candidate path chain documented above. + * On success returns 0 and stores a heap-allocated oci_policy_t in *out + * which the caller frees via oci_policy_free. On failure returns -1 with + * errno set; *err_msg (when non-NULL) points at a description owned by + * the partially constructed policy (released by oci_policy_free even on + * failure when *out is non-NULL) or a static literal when allocation + * failed before the struct existed. Pass NULL for err_msg to skip the + * diagnostic. + * + * The loader is tolerant of unknown JSON keys at every level: they are + * accepted and recorded so future schema extensions (sigstore beyond the + * minimal subset, registries.d overlays in C6.3, mirror chains, ...) do + * not require a coordinated reader rollout. + */ +int oci_policy_load(oci_policy_t **out, const char **err_msg); + +/* Release a policy. Safe on NULL and on the partially constructed object + * a failed oci_policy_load may have produced. + */ +void oci_policy_free(oci_policy_t *p); + +/* Fill *eff with the merged view for host. Unknown host falls back to the + * default block. host is matched exactly (case-sensitive); the OCI ref + * parser already lowercases registry hostnames, so this is the same key + * shape policy.json uses. + * + * eff is always fully populated: fields the policy did not declare come + * out as NULL strings or as the default-block values (for the insecure + * flag). A NULL p or NULL host treats the call as a request for the + * zero-value default. + */ +void oci_policy_lookup(const oci_policy_t *p, + const char *host, + oci_policy_effective_t *eff); + +/* Return the absolute filesystem path of the policy file that produced + * this object, or "" when the built-in default fired. The pointer is + * owned by the policy and stays valid until oci_policy_free. NULL p + * returns "". + */ +const char *oci_policy_source(const oci_policy_t *p); + +/* Read a podman/skopeo-style auth file from path. The body must be JSON of + * the shape: + * + * { "username": "", "password": "" } + * + * Both fields are required; either may not be NULL. A missing field, a + * malformed JSON body, a non-regular file, or a mode that grants group or + * other access is a hard error (the file must satisfy (st_mode & 077) == 0, + * matching the credential-handling discipline ssh and curl both use). + * + * On success returns 0 and writes heap-owned strings into *out_user and + * *out_pass which the caller frees. On failure returns -1 with errno set + * (ENOENT, EACCES, EPERM for mode, EINVAL for missing fields / malformed + * JSON) and *err_msg (when non-NULL) pointing at a static description + * suitable for direct caller-side use. *out_user and *out_pass may be + * partially populated on failure (one strdup succeeded, another failed); the + * caller must free both unconditionally, including on rc != 0. NULL path, + * NULL out_user, or NULL out_pass is EINVAL. + * + * The diagnostic does not include the path; the caller already knows it and + * is free to compose its own message ("auth file %s: %s", path, err). + */ +int oci_policy_load_auth(const char *path, + char **out_user, + char **out_pass, + const char **err_msg); diff --git a/src/oci/pull.c b/src/oci/pull.c new file mode 100644 index 0000000..50d2be0 --- /dev/null +++ b/src/oci/pull.c @@ -0,0 +1,785 @@ +/* elfuse oci pull pipeline + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The pull function is intentionally linear: every state transition (top-level + * fetch, index recurse, config fetch, layer fetch, pin write) flows top-to- + * bottom in oci_pull below. Helpers exist only to remove pure boilerplate + * (response cleanup, hex equality, progress prints), so that a reader of + * oci_pull can follow the registry round trips without chasing through + * indirection. + */ + +#include "pull.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "blob-store.h" +#include "digest.h" +#include "manifest.h" +#include "media-type.h" + +static const char *const PULL_ACCEPT[] = { + "application/vnd.oci.image.index.v1+json", + "application/vnd.docker.distribution.manifest.list.v2+json", + "application/vnd.oci.image.manifest.v1+json", + "application/vnd.docker.distribution.manifest.v2+json", + NULL, +}; + +static FILE *pick_progress(const oci_pull_options_t *opts) +{ + if (!opts) + return stderr; + if (opts->quiet) + return NULL; + return opts->progress ? opts->progress : stderr; +} + +static void progress_line(FILE *fp, + const char *kind, + const char *digest_str, + int64_t size, + const char *state, + const char *media_type) +{ + if (!fp) + return; + /* Truncated digest keeps the line readable; full hex still goes into the + * pin file and the blob store for verification. + */ + char short_digest[24]; + snprintf(short_digest, sizeof(short_digest), "%.19s...", digest_str); + fprintf(fp, " %-9s %-22s %12lldB %-11s %s\n", kind, short_digest, + (long long) size, state ? state : "", media_type ? media_type : ""); + fflush(fp); +} + +/* In-progress per-blob slot for the batch fetcher's xferinfo callback to + * update. Kept file-local because pull.c is the only caller. A second + * subcommand wiring the same progress callback would be the trigger to + * lift this and pull_progress_t into src/oci/progress.{c,h}. + */ +typedef struct { + const oci_descriptor_t *desc; + const char *kind; /* "config" or "layer" */ + const char *media_type; + int64_t bytes_dl; + int64_t bytes_total; + bool done_emitted; /* non-TTY mode: per-blob one-shot guard */ +} pull_progress_slot_t; + +typedef struct { + FILE *fp; + bool is_tty; + bool started; /* TTY: placeholder lines already printed */ + pull_progress_slot_t *slots; + size_t n_slots; +} pull_progress_t; + +static void pull_progress_print_inplace_line(FILE *fp, + const pull_progress_slot_t *slot) +{ + char short_digest[24]; + snprintf(short_digest, sizeof(short_digest), "%.19s...", + slot->desc->digest_str); + int percent = 0; + if (slot->bytes_total > 0) { + int64_t p = (slot->bytes_dl * 100) / slot->bytes_total; + if (p < 0) + p = 0; + if (p > 100) + p = 100; + percent = (int) p; + } + const char *state = + slot->bytes_total > 0 && slot->bytes_dl >= slot->bytes_total + ? "downloaded" + : "pulling"; + /* CSI 2K clears the entire line under the cursor; the trailing newline + * advances to the next slot row. + */ + fprintf(fp, "\033[2K %-9s %-22s %8lld/%lldB %3d%% %-11s %s\n", slot->kind, + short_digest, (long long) slot->bytes_dl, + (long long) slot->bytes_total, percent, state, + slot->media_type ? slot->media_type : ""); +} + +/* TTY render path: cursor-up to the top of the redraw zone, reprint every + * slot in place. The redraw zone is exactly n_slots rows tall and the + * cursor ends up one row below the last slot, matching the post-init + * position. + */ +static void pull_progress_tty_redraw(pull_progress_t *pp) +{ + if (!pp->fp || pp->n_slots == 0) + return; + /* CSI nF moves the cursor up n lines to column 0; "1F" is a single row. + * n_slots is at most a few dozen so the integer width fits trivially. + */ + fprintf(pp->fp, "\033[%zuF", pp->n_slots); + for (size_t i = 0; i < pp->n_slots; i++) + pull_progress_print_inplace_line(pp->fp, &pp->slots[i]); + fflush(pp->fp); +} + +/* Walk descs[] and split it into already-cached (printed immediately, no + * slot) and to-be-downloaded (one slot each). Cached lines preserve the + * pre-C5.3 byte-identical wording so existing log-parsing pipelines do + * not regress. In TTY mode, after the cached lines, n_slots placeholder + * lines are printed and the cursor lands one row below the zone so the + * xferinfo redraw loop can repeatedly hop back to the zone top. + */ +static int pull_progress_init(pull_progress_t *pp, + FILE *fp, + const oci_descriptor_t *config, + bool config_cached, + const oci_descriptor_t *layers, + size_t n_layers, + const bool *layer_cached) +{ + memset(pp, 0, sizeof(*pp)); + pp->fp = fp; + pp->is_tty = fp != NULL && isatty(fileno(fp)); + /* ELFUSE_OCI_PROGRESS=plain (or =lines, =off) forces the + * line-per-completion path even on a real TTY. Some terminal panes + * (notably embedded ones that emulate a pty without honoring CSI + * cursor-up) leave the in-place redraw stacking copies down the + * screen instead of rewriting the active rows; the env override + * gives the operator a stable opt-out without touching code. + */ + if (pp->is_tty) { + const char *override = getenv("ELFUSE_OCI_PROGRESS"); + if (override && + (!strcmp(override, "plain") || !strcmp(override, "lines") || + !strcmp(override, "off"))) + pp->is_tty = false; + } + + size_t cap = 1 + n_layers; + pp->slots = calloc(cap, sizeof(*pp->slots)); + if (!pp->slots) + return -1; + + /* Emit cached lines immediately and reserve a slot for everything else. + * The slot's bytes_total is desc->size; bytes_dl starts at zero so the + * TTY placeholder shows 0/B 0%. + */ + if (config_cached) { + progress_line(fp, "config", config->digest_str, config->size, "cached", + oci_media_type_name(config->media_type)); + } else { + pp->slots[pp->n_slots++] = (pull_progress_slot_t) { + .desc = config, + .kind = "config", + .media_type = oci_media_type_name(config->media_type), + .bytes_dl = 0, + .bytes_total = config->size, + }; + } + for (size_t i = 0; i < n_layers; i++) { + const oci_descriptor_t *L = &layers[i]; + if (layer_cached[i]) { + progress_line(fp, "layer", L->digest_str, L->size, "cached", + oci_media_type_name(L->media_type)); + } else { + pp->slots[pp->n_slots++] = (pull_progress_slot_t) { + .desc = L, + .kind = "layer", + .media_type = oci_media_type_name(L->media_type), + .bytes_dl = 0, + .bytes_total = L->size, + }; + } + } + + /* TTY: print n_slots placeholder lines; the cursor lands on the row + * immediately below the zone. Non-TTY: defer per-blob output until + * the bytes_dl == bytes_total event in the callback. + */ + if (pp->is_tty && pp->n_slots > 0) { + for (size_t i = 0; i < pp->n_slots; i++) + pull_progress_print_inplace_line(fp, &pp->slots[i]); + fflush(fp); + pp->started = true; + } + return 0; +} + +static pull_progress_slot_t *pull_progress_find(pull_progress_t *pp, + const oci_descriptor_t *desc) +{ + for (size_t i = 0; i < pp->n_slots; i++) { + if (pp->slots[i].desc == desc) + return &pp->slots[i]; + } + return NULL; +} + +/* Callback handed to oci_fetch_blob_batch. Runs on the fetcher's thread + * (single-threaded curl_multi event loop) so the renderer's pp state + * needs no locking. Returning 0 lets the transfer continue; the C5.3 + * renderer never aborts (a future cancellable pull would return non-zero + * here). + */ +static int pull_progress_cb(const oci_descriptor_t *desc, + int64_t bytes_dl, + int64_t bytes_total, + void *user) +{ + pull_progress_t *pp = user; + if (!pp) + return 0; + pull_progress_slot_t *slot = pull_progress_find(pp, desc); + if (!slot) + return 0; + slot->bytes_dl = bytes_dl; + slot->bytes_total = bytes_total; + if (pp->is_tty) { + pull_progress_tty_redraw(pp); + } else if (!slot->done_emitted && bytes_total > 0 && + bytes_dl >= bytes_total) { + /* Single line per blob on completion. Matches the line-per-event + * log shape that scripts grep against (digest, size, state). + */ + progress_line(pp->fp, slot->kind, slot->desc->digest_str, + slot->bytes_total, "downloaded", slot->media_type); + slot->done_emitted = true; + } + return 0; +} + +static void pull_progress_dispose(pull_progress_t *pp) +{ + free(pp->slots); + pp->slots = NULL; + pp->n_slots = 0; +} + +/* Case-insensitive prefix check for "sha256:" / "sha512:". */ +static bool digest_str_matches(const char *want, const char *got) +{ + if (!want || !got) + return false; + return strcasecmp(want, got) == 0; +} + +/* Cross-check the manifest body against the registry-supplied + * Docker-Content-Digest header. Servers usually emit one; when they do not, + * trust the body's local SHA-256. The local hex is what we use to address the + * blob in the store regardless, so a missing header degrades to local-only + * verification but not to silent corruption. + */ +static int verify_manifest_digest(const oci_fetch_response_t *resp, + const char *expected_digest_str, + char *out_digest_str, + size_t out_cap, + const char **err_msg) +{ + char hex[OCI_DIGEST_HEX_MAX + 1]; + if (oci_digest_bytes(OCI_DIGEST_SHA256, resp->body, resp->body_len, hex) == + 0) { + if (err_msg) + *err_msg = "failed to hash manifest body"; + errno = EIO; + return -1; + } + int n = snprintf(out_digest_str, out_cap, "sha256:%s", hex); + if (n < 0 || (size_t) n >= out_cap) { + if (err_msg) + *err_msg = "manifest digest buffer too small"; + errno = ENAMETOOLONG; + return -1; + } + if (resp->docker_content_digest && + !digest_str_matches(resp->docker_content_digest, out_digest_str)) { + if (err_msg) + *err_msg = + "manifest body digest does not match " + "Docker-Content-Digest header"; + errno = EPROTO; + return -1; + } + if (expected_digest_str && + !digest_str_matches(expected_digest_str, out_digest_str)) { + if (err_msg) + *err_msg = "manifest body digest does not match expected digest"; + errno = EPROTO; + return -1; + } + return 0; +} + +/* Fetch a manifest document (image index, image manifest, or sub-manifest) by + * selector, hash its body, cross-check against expected_digest_str (when + * non-NULL), and write it into the local blob store. Returns 0 on success and + * fills *out_digest_str with the canonical "sha256:" representation. The + * caller frees *out_response via oci_fetch_response_free. + * + * if_none_match is forwarded as the conditional GET header; when set and the + * registry responds 304 Not Modified the helper returns 0, writes nothing to + * the store, leaves *out_digest_str empty, and sets *out_unchanged (when + * non-NULL). The caller seeds the digest string from the pin before calling. + */ +static int fetch_and_persist_manifest(oci_fetcher_t *f, + oci_store_t *store, + const oci_ref_t *ref, + const char *selector, + const char *expected_digest_str, + const char *if_none_match, + oci_fetch_response_t *out_resp, + char *out_digest_str, + size_t out_cap, + bool *out_unchanged, + const char **err_msg) +{ + if (out_unchanged) + *out_unchanged = false; + memset(out_resp, 0, sizeof(*out_resp)); + if (oci_fetch_manifest(f, ref, selector, PULL_ACCEPT, if_none_match, + out_resp, err_msg) < 0) { + return -1; + } + if (out_resp->http_status == 304) { + if (out_unchanged) + *out_unchanged = true; + return 0; + } + if (out_resp->body_len == 0 || !out_resp->body) { + if (err_msg) + *err_msg = "manifest response had an empty body"; + errno = EPROTO; + return -1; + } + if (verify_manifest_digest(out_resp, expected_digest_str, out_digest_str, + out_cap, err_msg) < 0) { + return -1; + } + char hex[OCI_DIGEST_HEX_MAX + 1]; + oci_digest_algo_t algo; + if (!oci_digest_parse(out_digest_str, &algo, hex)) { + if (err_msg) + *err_msg = "computed manifest digest is malformed"; + errno = EINVAL; + return -1; + } + if (oci_blob_store_put_bytes(oci_store_blobs(store), OCI_DIGEST_SHA256, hex, + out_resp->body, out_resp->body_len) < 0) { + if (err_msg) + *err_msg = "failed to persist manifest body to local store"; + return -1; + } + return 0; +} + +/* Load a manifest blob already present in the local store into a heap buffer. + * Used by the refresh path after the registry confirms an unchanged digest: + * the manifest body must still be parsed (to drive the layer-cache sweep) + * but no network round trip is needed because the bytes are already on disk. + * Returns 0 on success with *out_buf newly malloc'd (caller frees) and + * *out_len set; -1 on IO failure with errno preserved. + */ +static int load_manifest_blob(oci_blob_store_t *blobs, + oci_digest_algo_t algo, + const char *hex, + char **out_buf, + size_t *out_len, + const char **err_msg) +{ + char path[1024]; + int n = oci_blob_store_path(blobs, algo, hex, path, sizeof(path)); + if (n < 0 || (size_t) n >= sizeof(path)) { + if (err_msg) + *err_msg = "manifest blob path overflow"; + errno = ENAMETOOLONG; + return -1; + } + int fd = open(path, O_RDONLY); + if (fd < 0) { + if (err_msg) + *err_msg = "failed to open cached manifest blob"; + return -1; + } + struct stat st; + if (fstat(fd, &st) < 0) { + int e = errno; + close(fd); + if (err_msg) + *err_msg = "failed to stat cached manifest blob"; + errno = e; + return -1; + } + if (st.st_size <= 0 || (uintmax_t) st.st_size > (uintmax_t) SIZE_MAX - 1) { + close(fd); + if (err_msg) + *err_msg = "cached manifest blob has an unreasonable size"; + errno = EFBIG; + return -1; + } + size_t want = (size_t) st.st_size; + char *buf = malloc(want + 1); + if (!buf) { + close(fd); + if (err_msg) + *err_msg = "out of memory loading cached manifest"; + errno = ENOMEM; + return -1; + } + size_t got = 0; + while (got < want) { + ssize_t r = read(fd, buf + got, want - got); + if (r < 0) { + int e = errno; + free(buf); + close(fd); + if (err_msg) + *err_msg = "read failed on cached manifest blob"; + errno = e; + return -1; + } + if (r == 0) + break; + got += (size_t) r; + } + close(fd); + if (got != want) { + free(buf); + if (err_msg) + *err_msg = "cached manifest blob truncated mid-read"; + errno = EIO; + return -1; + } + buf[want] = '\0'; + *out_buf = buf; + *out_len = want; + return 0; +} + +static int parse_top_level(const oci_fetch_response_t *resp, + oci_media_type_t *out_mt, + const char **err_msg) +{ + oci_media_type_t mt = oci_media_type_parse(resp->content_type); + if (mt == OCI_MT_UNKNOWN) { + if (err_msg) + *err_msg = "registry returned an unrecognized Content-Type"; + errno = EPROTO; + return -1; + } + if (!oci_media_type_is_index(mt) && !oci_media_type_is_manifest(mt)) { + if (err_msg) + *err_msg = "registry returned a non-manifest Content-Type"; + errno = EPROTO; + return -1; + } + *out_mt = mt; + return 0; +} + +int oci_pull(oci_fetcher_t *fetcher, + oci_store_t *store, + const oci_ref_t *ref, + const oci_pull_options_t *opts, + const char **err_msg) +{ + if (!fetcher || !store || !ref) { + if (err_msg) + *err_msg = "invalid arguments"; + errno = EINVAL; + return -1; + } + + FILE *progress = pick_progress(opts); + int rc = -1; + oci_fetch_response_t top_resp = {0}; + oci_fetch_response_t sub_resp = {0}; + oci_index_t idx_doc = {0}; + oci_manifest_t manifest = {0}; + bool top_unchanged = false; + char top_digest_str[OCI_DIGEST_HEX_MAX + 16]; + char sub_digest_str[OCI_DIGEST_HEX_MAX + 16]; + top_digest_str[0] = '\0'; + sub_digest_str[0] = '\0'; + char *cached_top_body = NULL; + size_t cached_top_body_len = 0; + char *pin_digest_for_refresh = NULL; + char if_none_match_buf[OCI_DIGEST_HEX_MAX + 32]; + const char *if_none_match = NULL; + + /* 0. Refresh prologue. Only fires when --refresh is set, the ref carries + * a tag (digest-only refs are content-addressed and cannot drift), the + * pin exists, and the pinned manifest blob is still on disk. Otherwise + * the call falls through to the normal pull path. + */ + if (opts && opts->refresh && ref->tag) { + const char *pin_err = NULL; + if (oci_store_get_ref(store, ref, &pin_digest_for_refresh, &pin_err) == + 0 && + pin_digest_for_refresh) { + oci_digest_algo_t pin_algo; + char pin_hex[OCI_DIGEST_HEX_MAX + 1]; + if (oci_digest_parse(pin_digest_for_refresh, &pin_algo, pin_hex) && + oci_blob_store_has(oci_store_blobs(store), pin_algo, pin_hex)) { + snprintf(if_none_match_buf, sizeof(if_none_match_buf), "\"%s\"", + pin_digest_for_refresh); + if_none_match = if_none_match_buf; + /* Seed top_digest_str so the 304 path can echo the pin into + * progress and the layer-cache sweep without re-deriving it + * from a body that the registry just omitted. + */ + snprintf(top_digest_str, sizeof(top_digest_str), "%s", + pin_digest_for_refresh); + } + } + } + + /* 1. Top-level fetch. Selector defaults to ref->digest, falling through + * to ref->tag, inside oci_fetch_manifest. When the user pulled by digest, + * expected_digest_str is the locked target; pulls by tag accept whatever + * the server resolves the tag to. if_none_match is set only by the + * refresh prologue above; a 304 response keeps the pin and re-uses the + * cached manifest body from the local store. + */ + if (fetch_and_persist_manifest(fetcher, store, ref, NULL, ref->digest, + if_none_match, &top_resp, top_digest_str, + sizeof(top_digest_str), &top_unchanged, + err_msg) < 0) { + goto out; + } + + const char *manifest_body = NULL; + size_t manifest_body_len = 0; + oci_media_type_t top_mt = OCI_MT_UNKNOWN; + const char *pin_digest_str = top_digest_str; + + if (top_unchanged) { + /* Registry confirmed the pinned digest still matches. Load the + * persisted manifest blob and run the rest of the pipeline against + * it so the layer-cache sweep can re-fetch any blob the user has + * pruned since the last pull. + */ + oci_digest_algo_t cached_algo; + char cached_hex[OCI_DIGEST_HEX_MAX + 1]; + if (!oci_digest_parse(top_digest_str, &cached_algo, cached_hex)) { + if (err_msg) + *err_msg = "pinned manifest digest is malformed"; + errno = EINVAL; + goto out; + } + if (load_manifest_blob(oci_store_blobs(store), cached_algo, cached_hex, + &cached_top_body, &cached_top_body_len, + err_msg) < 0) { + goto out; + } + /* The persisted manifest blob has no Content-Type header. Try the + * image-index media type first, fall back to image-manifest. The + * parse step below is the actual gate; this only steers the index + * drill decision. + */ + oci_index_t probe = {0}; + if (oci_index_parse(cached_top_body, cached_top_body_len, &probe, + NULL) == 0) { + top_mt = OCI_MT_INDEX_OCI; + oci_index_free(&probe); + } else { + top_mt = OCI_MT_MANIFEST_OCI; + } + manifest_body = cached_top_body; + manifest_body_len = cached_top_body_len; + progress_line(progress, "manifest", top_digest_str, + (int64_t) cached_top_body_len, "unchanged", + oci_media_type_name(top_mt)); + } else { + if (parse_top_level(&top_resp, &top_mt, err_msg) < 0) + goto out; + progress_line(progress, "manifest", top_digest_str, + (int64_t) top_resp.body_len, "downloaded", + oci_media_type_name(top_mt)); + manifest_body = top_resp.body; + manifest_body_len = top_resp.body_len; + } + + /* 2. If top-level was an image index, pick linux/arm64 and refetch. */ + if (oci_media_type_is_index(top_mt)) { + if (oci_index_parse(manifest_body, manifest_body_len, &idx_doc, + err_msg) < 0) { + goto out; + } + const oci_index_entry_t *entry = oci_index_pick_linux_arm64(&idx_doc); + if (!entry) { + if (err_msg) + *err_msg = "image index has no linux/arm64 entry"; + errno = ENOENT; + goto out; + } + if (progress) { + fprintf( + progress, " picked %-22s %12lldB linux/arm64%s%s\n", + entry->desc.digest_str, (long long) entry->desc.size, + entry->platform.variant && *entry->platform.variant ? " " : "", + entry->platform.variant ? entry->platform.variant : ""); + fflush(progress); + } + + /* Sub-manifest fetch never carries If-None-Match: the index drill + * targets a specific digest, so a conditional GET there has no + * semantic anchor (the local blob, if cached, is already the answer + * by content-address). When the sub-manifest blob is already in the + * store oci_fetch_manifest would still re-GET; the linear shape + * leaves that as future work because manifests are small. + */ + if (fetch_and_persist_manifest( + fetcher, store, ref, entry->desc.digest_str, + entry->desc.digest_str, NULL, &sub_resp, sub_digest_str, + sizeof(sub_digest_str), NULL, err_msg) < 0) { + goto out; + } + oci_media_type_t sub_mt = OCI_MT_UNKNOWN; + if (parse_top_level(&sub_resp, &sub_mt, err_msg) < 0) + goto out; + if (!oci_media_type_is_manifest(sub_mt)) { + if (err_msg) + *err_msg = "index entry resolved to a non-manifest document"; + errno = EPROTO; + goto out; + } + progress_line(progress, "manifest", sub_digest_str, + (int64_t) sub_resp.body_len, "downloaded", + oci_media_type_name(sub_mt)); + + manifest_body = sub_resp.body; + manifest_body_len = sub_resp.body_len; + /* pin_digest_str stays as top_digest_str: the user pulled the tag, + * the registry resolved that tag to the index, so the pin records the + * index digest. Future inspect re-walks index -> manifest. + */ + } + + /* 3. Parse the manifest body. */ + if (oci_manifest_parse(manifest_body, manifest_body_len, &manifest, + err_msg) < 0) { + goto out; + } + + /* 4+5. Fetch config + every layer blob in parallel via the batch fetcher + * (oci-improvements-plan Plan 5 C5.1). The progress lines below are + * still per-blob and still preserve the cached/downloaded annotation, + * so the store-has lookup is captured before the batch call hides the + * transfer / cache decision behind the multi event loop. The batch is + * atomic: any blob fail aborts every writer and the function bails. + */ + { + bool batch_ok = false; + size_t batch_n = 1 + manifest.nlayers; + const oci_descriptor_t **batch_descs = + calloc(batch_n, sizeof(*batch_descs)); + bool config_cached = false; + bool *layer_cached = + manifest.nlayers > 0 + ? calloc(manifest.nlayers, sizeof(*layer_cached)) + : NULL; + if (!batch_descs || (manifest.nlayers > 0 && !layer_cached)) { + free(batch_descs); + free(layer_cached); + if (err_msg) + *err_msg = "out of memory composing blob batch"; + errno = ENOMEM; + goto out; + } + batch_descs[0] = &manifest.config; + config_cached = oci_blob_store_has( + oci_store_blobs(store), manifest.config.algo, manifest.config.hex); + for (size_t i = 0; i < manifest.nlayers; i++) { + batch_descs[1 + i] = &manifest.layers[i]; + layer_cached[i] = oci_blob_store_has(oci_store_blobs(store), + manifest.layers[i].algo, + manifest.layers[i].hex); + } + + /* Set up the per-blob progress renderer before the batch call so + * the cached lines land in the same paragraph the C5.1 code path + * used to produce post-hoc. The downloaded blobs get rendered in + * place during the transfer (TTY) or one line each on completion + * (non-TTY) via pull_progress_cb. A NULL progress fp (--quiet) + * still produces zero output because pp.fp is NULL and the + * formatter functions short-circuit on that. + */ + pull_progress_t pp; + if (pull_progress_init(&pp, progress, &manifest.config, config_cached, + manifest.layers, manifest.nlayers, + layer_cached) < 0) { + free(batch_descs); + free(layer_cached); + if (err_msg) + *err_msg = "out of memory initialising progress"; + errno = ENOMEM; + goto out; + } + + if (oci_fetch_blob_batch(fetcher, ref, batch_descs, batch_n, + oci_store_blobs(store), pull_progress_cb, &pp, + err_msg) == 0) { + batch_ok = true; + } + + pull_progress_dispose(&pp); + free(batch_descs); + free(layer_cached); + if (!batch_ok) + goto out; + } + + /* 6. Pin tag -> top-level digest. Digest-only refs are self-pinning and + * skip this step (oci_store_put_ref refuses them). On 304 the pin is + * already at the right digest, so the put_ref call is skipped to avoid + * an unnecessary tmp + rename round trip. + */ + if (ref->tag) { + if (!top_unchanged) { + if (oci_store_put_ref(store, ref, pin_digest_str, err_msg) < 0) + goto out; + } + if (progress) { + fprintf(progress, " pin %s:%s -> %s%s\n", ref->repository, + ref->tag, pin_digest_str, + top_unchanged ? " (unchanged)" : ""); + fflush(progress); + } + } + + rc = 0; + +out: + /* Preserve the caller-visible errno across cleanup. free / fclose can + * stomp on errno even when they succeed, which would defeat callers that + * key tests off specific values (EPROTO / ENOENT / EINVAL). + */ + { + int saved_errno = errno; + oci_manifest_free(&manifest); + oci_index_free(&idx_doc); + /* sub_resp is zero-initialised and oci_fetch_response_free is + * null-safe, so free it unconditionally. fetch_and_persist_manifest + * can populate the response body and then fail a later validation + * step (digest mismatch, persist error) without freeing it, so a + * guarded free would leak on those error paths. + */ + oci_fetch_response_free(&sub_resp); + oci_fetch_response_free(&top_resp); + free(cached_top_body); + free(pin_digest_for_refresh); + if (rc != 0) + errno = saved_errno; + } + return rc; +} diff --git a/src/oci/pull.h b/src/oci/pull.h new file mode 100644 index 0000000..ef00c55 --- /dev/null +++ b/src/oci/pull.h @@ -0,0 +1,75 @@ +/* elfuse oci pull pipeline + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Glues the slice 4a/4b fetcher and the slice 3 manifest parser to the + * slice 5a local store. One call to oci_pull resolves an image reference into + * a fully populated blob graph on disk: + * + * 1. Fetch the top-level descriptor by ref->digest or ref->tag. + * 2. Cross-check the response Docker-Content-Digest against a local SHA-256 + * of the body; a mismatch is a hostile-registry signal and aborts. + * 3. If the response is an image index, parse it, pick the linux/arm64 + * sub-manifest (oci-roadmap Q3), and re-fetch by that digest. + * 4. Parse the manifest, fetch the config blob, fetch each layer blob. + * 5. Write the tag-to-manifest-digest pin so the next pull or inspect for + * the same tag is reproducible. + * + * The function is best-effort idempotent: a re-pull of the same reference + * short-circuits all already-present blobs through the slice 4a oci_fetch_blob + * cache check, only the top-level manifest is re-fetched (small bytes; future + * slice can add a manifest cache). + * + * Foreign / nondistributable layers and schema v1 manifests are rejected by + * the parsers in slice 3; oci_pull surfaces the diagnostic and aborts before + * any partial layer hits the store. + */ + +#pragma once + +#include + +#include "fetch.h" +#include "ref.h" +#include "store.h" + +typedef struct { + /* Per-blob progress is written here as one line per descriptor. Set to + * NULL to suppress all output. Defaults to stderr when opts is NULL or + * progress is NULL but suppress_progress is not requested explicitly. + */ + FILE *progress; + /* When true, suppress progress output even if progress is NULL (the + * NULL/default interpretation lands on stderr). Used by elfuse oci + * pull -q. + */ + bool quiet; + /* Opt-in tag revalidation. When the pinned manifest digest and its blob + * are both already in the store, the top-level manifest GET carries + * If-None-Match: ""; on 304 Not Modified the pull + * short-circuits without re-fetching layer blobs and leaves the pin in + * place. Without this flag the default pull re-runs every step (never + * trusts the pin), which keeps stale-tag detection responsive but pays + * the network cost. + * + * The flag is a no-op for digest-only refs (no tag to revalidate + * against), and silently falls through to a normal pull when no pin + * exists yet or the pinned manifest blob has been pruned from the + * store. Servers may ignore If-None-Match and respond 200 with a new + * digest; the pull then runs the full pipeline against the new + * manifest. The previous manifest blob stays in the store until prune + * collects it. + */ + bool refresh; +} oci_pull_options_t; + +/* Run the pull pipeline. Returns 0 on success, -1 on failure with errno + * preserved and *err_msg (when non-NULL) pointing at a static description. + * The store and fetcher must outlive the call; both are reused across phases. + */ +int oci_pull(oci_fetcher_t *fetcher, + oci_store_t *store, + const oci_ref_t *ref, + const oci_pull_options_t *opts, + const char **err_msg); diff --git a/src/oci/rebuild-cache.c b/src/oci/rebuild-cache.c new file mode 100644 index 0000000..573f0b2 --- /dev/null +++ b/src/oci/rebuild-cache.c @@ -0,0 +1,281 @@ +/* OCI stack cache back-fill driver. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * See rebuild-cache.h for the externally visible contract. Implementation + * sketch: oci_volume_list_unpacked yields every /images/ + * sha256-/ directory; for each one this module reads the origin + * sidecar, recomputes the terminating ChainID from the recorded diff_id + * list, probes the stack cache for an existing entry, and (when + * opts->commit is set) clonefiles the tree into a staged stack snapshot, + * strips the .elfuse-origin.json that the unpacker wrote AFTER the fresh- + * unpack snapshot was taken, and atomically promotes the staged tree via + * oci_store_stack_commit. Failures inside the per-tree loop are aggregated + * into stats counters and reported on stderr; the loop never aborts so a + * single corrupt tree cannot block the rest of the back-fill. + */ + +#include "rebuild-cache.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "digest.h" +#include "origin-meta.h" +#include "volume.h" + +#define RC_PATH_MAX 4096 + +static int set_err(const char **err, const char *msg, int err_no) +{ + if (err) + *err = msg; + errno = err_no; + return -1; +} + +/* Recursive rmdir / unlink for clean-up after a failed snapshot stage. + * Duplicates the rm_recursive in src/oci/unpack.c because pulling unpack.c + * into a small back-fill module would drag the full layer-apply / decompress + * graph along for one helper. Lift to a shared util if a third copy + * appears. + */ +static int rm_recursive(const char *path) +{ + struct stat st; + if (lstat(path, &st) < 0) + return errno == ENOENT ? 0 : -1; + if (!S_ISDIR(st.st_mode)) + return unlink(path); + DIR *d = opendir(path); + if (!d) + return -1; + struct dirent *de; + int rc = 0; + while ((de = readdir(d))) { + if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) + continue; + char child[RC_PATH_MAX]; + int n = snprintf(child, sizeof(child), "%s/%s", path, de->d_name); + if (n < 0 || (size_t) n >= sizeof(child)) { + errno = ENAMETOOLONG; + rc = -1; + break; + } + if (rm_recursive(child) < 0) { + rc = -1; + break; + } + } + closedir(d); + if (rc == 0 && rmdir(path) < 0) + rc = -1; + return rc; +} + +static size_t count_diff_ids(char *const *diff_ids) +{ + size_t n = 0; + if (!diff_ids) + return 0; + while (diff_ids[n]) + n++; + return n; +} + +/* Compute the terminating ChainID over a non-empty diff_id list. Iterates + * oci_chainid_compute with a two-buffer ping-pong so the running chain + * value can be threaded through each step without per-iteration alloc. + * Writes the result into out (must hold at least OCI_DIGEST_HEX_MAX + 16 + * bytes). Returns 0 on success, -1 with errno set on failure. + */ +static int compute_terminal_chain(char *const *diff_ids, + size_t n, + char *out, + size_t cap) +{ + char buf_a[OCI_DIGEST_HEX_MAX + 16]; + char buf_b[OCI_DIGEST_HEX_MAX + 16]; + char *cur = buf_a; + char *nxt = buf_b; + if (oci_chainid_compute(NULL, diff_ids[0], cur, sizeof(buf_a)) < 0) + return -1; + for (size_t i = 1; i < n; i++) { + if (oci_chainid_compute(cur, diff_ids[i], nxt, sizeof(buf_b)) < 0) + return -1; + char *tmp = cur; + cur = nxt; + nxt = tmp; + } + size_t need = strlen(cur) + 1; + if (need > cap) { + errno = ENAMETOOLONG; + return -1; + } + memcpy(out, cur, need); + return 0; +} + +/* Stage + clonefile + sanitize + stack_commit one tree. The caller verified + * that oci_store_stack_has returned 0 for chain_id; this function does not + * re-probe. Returns 0 on success, -1 with errno preserved on failure. On + * failure the staging path (if any) is removed. + */ +static int snapshot_tree_to_stack(oci_store_t *store, + const char *tree_path, + const char *chain_id, + const char **err) +{ + char stage_path[RC_PATH_MAX]; + if (oci_store_stack_stage_path(store, chain_id, stage_path, + sizeof(stage_path)) < 0) { + if (err) + *err = "rebuild-cache: stack stage_path resolve failed"; + return -1; + } + if (clonefile(tree_path, stage_path, CLONE_NOFOLLOW) < 0) { + int saved = errno; + if (err) + *err = saved == EXDEV + ? "rebuild-cache: stack snapshot EXDEV (store and " + "volume must share an APFS volume)" + : "rebuild-cache: stack snapshot clonefile failed"; + errno = saved; + return -1; + } + /* Strip the origin sidecar so the rebuilt snapshot matches the shape + * a fresh-unpack snapshot would have produced (oci_unpack writes the + * origin sidecar AFTER it takes the stack snapshot). + */ + char origin_path[RC_PATH_MAX]; + int n = snprintf(origin_path, sizeof(origin_path), "%s/.elfuse-origin.json", + stage_path); + if (n < 0 || (size_t) n >= sizeof(origin_path)) { + (void) rm_recursive(stage_path); + if (err) + *err = "rebuild-cache: origin sidecar path overflow"; + errno = ENAMETOOLONG; + return -1; + } + if (unlink(origin_path) < 0 && errno != ENOENT) { + int saved = errno; + (void) rm_recursive(stage_path); + if (err) + *err = "rebuild-cache: origin sidecar unlink failed"; + errno = saved; + return -1; + } + const char *scerr = NULL; + if (oci_store_stack_commit(store, stage_path, chain_id, &scerr) < 0) { + int saved = errno; + (void) rm_recursive(stage_path); + if (err) + *err = scerr ? scerr : "rebuild-cache: stack_commit failed"; + errno = saved; + return -1; + } + return 0; +} + +int oci_rebuild_cache(oci_store_t *store, + const char *volume_root, + oci_rebuild_cache_options_t *opts, + const char **err) +{ + static const char *dummy_err; + if (!err) + err = &dummy_err; + *err = NULL; + if (!store || !opts) + return set_err(err, "rebuild-cache: NULL argument", EINVAL); + + /* Zero only output fields; the caller's commit flag must survive. */ + opts->trees_scanned = 0; + opts->trees_rebuilt = 0; + opts->trees_skipped_cached = 0; + opts->trees_skipped_no_origin = 0; + opts->trees_skipped_bad_origin = 0; + opts->trees_skipped_empty_diffids = 0; + opts->trees_failed = 0; + opts->stack_entries_added = 0; + + oci_volume_list_t trees = {0}; + const char *list_err = NULL; + if (oci_volume_list_unpacked(volume_root, &trees, &list_err) < 0) { + int saved = errno; + set_err(err, + list_err ? list_err : "rebuild-cache: list unpacked failed", + saved); + return -1; + } + + for (size_t i = 0; i < trees.count; i++) { + const char *tree = trees.items[i]; + opts->trees_scanned++; + + oci_origin_t origin = {0}; + if (oci_origin_read(tree, &origin, NULL) < 0) { + if (errno == ENOENT) + opts->trees_skipped_no_origin++; + else + opts->trees_skipped_bad_origin++; + continue; + } + + size_t n_diffs = count_diff_ids(origin.layer_diffids); + if (n_diffs == 0) { + opts->trees_skipped_empty_diffids++; + oci_origin_free(&origin); + continue; + } + + char chain[OCI_DIGEST_HEX_MAX + 16]; + if (compute_terminal_chain(origin.layer_diffids, n_diffs, chain, + sizeof(chain)) < 0) { + fprintf(stderr, "rebuild-cache: %s: chainid compute failed: %s\n", + tree, strerror(errno)); + opts->trees_failed++; + oci_origin_free(&origin); + continue; + } + oci_origin_free(&origin); + + int has = oci_store_stack_has(store, chain); + if (has < 0) { + fprintf(stderr, "rebuild-cache: %s: stack_has probe failed: %s\n", + tree, strerror(errno)); + opts->trees_failed++; + continue; + } + if (has > 0) { + opts->trees_skipped_cached++; + continue; + } + + if (!opts->commit) { + opts->trees_rebuilt++; + opts->stack_entries_added++; + continue; + } + + const char *snap_err = NULL; + if (snapshot_tree_to_stack(store, tree, chain, &snap_err) < 0) { + fprintf(stderr, "rebuild-cache: %s: %s: %s\n", tree, + snap_err ? snap_err : "snapshot failed", strerror(errno)); + opts->trees_failed++; + continue; + } + opts->trees_rebuilt++; + opts->stack_entries_added++; + } + + oci_volume_list_free(&trees); + return 0; +} diff --git a/src/oci/rebuild-cache.h b/src/oci/rebuild-cache.h new file mode 100644 index 0000000..71ff807 --- /dev/null +++ b/src/oci/rebuild-cache.h @@ -0,0 +1,101 @@ +/* OCI stack cache back-fill for legacy unpacked sysroots + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Plan 3 C3.3c introduced the ChainID-keyed stack cache at + * /layers/stacks/sha256// so a re-unpack of any image whose + * layer prefix matches an already-extracted tree short-circuits the per- + * layer apply loop into a single APFS clonefile restore. The cache only + * grows as a side effect of oci_unpack, which means any image that was + * unpacked before C3.3c landed (or that was unpacked into a store whose + * C3.3b schema marker had just wiped v1 entries) leaves no stack snapshot + * on disk: the next unpack still pays the full extract cost even though a + * usable assembled stage_dir already sits under /images/sha256-/. + * + * oci_rebuild_cache walks every unpacked sysroot under /images/, + * reads its .elfuse-origin.json sidecar to recover the original layer + * diff_id ordering, recomputes ChainID for the terminating layer, and (when + * commit is true) clonefiles the tree into a fresh stack cache entry keyed + * by that ChainID. Subsequent unpacks of any image sharing the same ordered + * layer list short-circuit immediately. Intermediate-prefix entries are NOT + * back-filled because an unpacked tree only captures the final overlay + * state; the per-layer raw cache /layers/sha256// similarly + * remains empty until a re-pull + re-unpack of the source image repopulates + * it. + * + * The walk is purely additive: existing stack cache entries are never + * modified or deleted, and the rename(2) used by oci_store_stack_commit + * treats EEXIST as a benign loss to a racing writer so repeated invocations + * are idempotent. No interaction with raw cache entries, blob storage, or + * pin metadata; rebuild-cache only manipulates layers/stacks/. + */ + +#pragma once + +#include +#include + +#include "store.h" + +/* Inputs and outputs for oci_rebuild_cache. Output counters are zeroed on + * entry so a caller can render a uniform report regardless of dry-run vs + * commit, and they reflect the same "what would happen" view: a dry-run + * reports trees_rebuilt as the number of trees that WOULD have landed a + * stack snapshot, while a commit run reports the number that actually did. + * trees_failed counts trees whose snapshot pipeline (chainid compute, + * clonefile, stack_commit) raised an error and which were therefore left + * out of trees_rebuilt; the offending tree is reported on stderr and the + * walk continues so a single bad tree does not abort the whole back-fill. + */ +typedef struct { + /* Inputs */ + bool commit; /* false = dry-run, true = back-fill */ + + /* Outputs */ + size_t trees_scanned; /* candidates encountered under images/ */ + size_t trees_rebuilt; /* would-rebuild (dry-run) or rebuilt (commit) */ + size_t trees_skipped_cached; /* terminating ChainID already on disk */ + size_t trees_skipped_no_origin; /* .elfuse-origin.json missing (ENOENT) */ + size_t + trees_skipped_bad_origin; /* origin sidecar parse or schema failure */ + size_t trees_skipped_empty_diffids; /* origin diff_ids array is empty */ + size_t trees_failed; /* chainid_compute / clonefile / commit IO */ + size_t stack_entries_added; /* sum across rebuilt trees */ +} oci_rebuild_cache_options_t; + +/* Walk /images/sha256-/ and back-fill the stack cache + * entry at /layers/stacks/sha256// for every + * tree whose .elfuse-origin.json carries a non-empty layer_diffids array + * and whose terminating ChainID is not already on disk. + * + * volume_root may be NULL; in that case oci_volume_list_unpacked treats the + * request as the empty case and trees_scanned stays 0. A missing + * /images/ directory is also treated as empty. + * + * The walk uses oci_volume_list_unpacked to enumerate candidates; only + * directories shaped sha256- are returned, so dotfiles and + * the .staging/ subtree are skipped automatically. + * + * The unpacked tree contains .elfuse-origin.json (written by oci_unpack + * AFTER the stack snapshot is taken on the fresh-unpack path). The + * back-fill strips that file from the staged snapshot before commit so a + * rebuilt stack cache entry is byte-identical to a fresh-unpack one. + * + * Failure policy: + * - per-tree origin read failure (ENOENT vs other) is counted in + * trees_skipped_no_origin / trees_skipped_bad_origin and the walk + * continues. + * - per-tree chainid_compute, clonefile, or stack_commit failure is + * counted in trees_failed and the walk continues. A diagnostic line is + * written to stderr identifying the tree path. + * - listing failure (failure to traverse images/) returns -1 with errno + * preserved and *err populated; opts->trees_scanned reflects the trees + * processed before the failure surfaced. + * + * Returns 0 on success and -1 on listing-level failure. err may be NULL. + */ +int oci_rebuild_cache(oci_store_t *store, + const char *volume_root, + oci_rebuild_cache_options_t *opts, + const char **err); diff --git a/src/oci/ref.c b/src/oci/ref.c new file mode 100644 index 0000000..ec4b409 --- /dev/null +++ b/src/oci/ref.c @@ -0,0 +1,467 @@ +/* OCI image reference parser + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * See ref.h for the grammar and design notes. The parser is split into: + * 1. find the optional @digest suffix and validate it + * 2. find the optional :tag suffix on the remainder + * 3. split the rest into registry vs path using the containerd domain rule + * 4. apply Docker defaults (docker.io, library/, latest) + * 5. validate every component against the OCI character class rules + */ + +#include "ref.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#define DEFAULT_REGISTRY "docker.io" +#define DEFAULT_LIBRARY_NAMESPACE "library" +#define DEFAULT_TAG "latest" + +#define MAX_REFERENCE_LEN 4096 +#define MAX_TAG_LEN 128 + +static char *strndup_local(const char *src, size_t n) +{ + char *dst = (char *) malloc(n + 1); + if (!dst) + return NULL; + memcpy(dst, src, n); + dst[n] = '\0'; + return dst; +} + +static void set_err(const char **slot, const char *msg) +{ + if (slot) + *slot = msg; +} + +static bool is_lower_alnum(char c) +{ + return (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9'); +} + +static bool is_path_separator(char c) +{ + return c == '.' || c == '_' || c == '-'; +} + +/* Validate one path component against [a-z0-9]+ (([._-]|__) [a-z0-9]+)*. + * Empty components and uppercase letters are rejected. + */ +static bool valid_path_component(const char *s, size_t len) +{ + if (len == 0) + return false; + if (!is_lower_alnum(s[0]) || !is_lower_alnum(s[len - 1])) + return false; + + size_t i = 0; + while (i < len) { + if (is_lower_alnum(s[i])) { + i++; + continue; + } + /* Separator run per the distribution-spec grammar + * [a-z0-9]+((\.|_|__|-+)[a-z0-9]+)*: a run of one-or-more '-', a + * single '.' or '_', or exactly "__". Anything else (e.g. "a..b", + * "a___b") is rejected. Note dashes may repeat ("my--repo") but + * dots and underscores may not. + */ + if (s[i] == '-') { + while (i < len && s[i] == '-') + i++; + } else if (s[i] == '_' && i + 1 < len && s[i + 1] == '_') { + i += 2; + } else if (is_path_separator(s[i])) { + i++; + } else { + return false; + } + if (i >= len || !is_lower_alnum(s[i])) + return false; + } + return true; +} + +/* Validate a multi-component path (components separated by '/'). */ +static bool valid_repository_path(const char *s, size_t len) +{ + if (len == 0) + return false; + size_t start = 0; + for (size_t i = 0; i < len; i++) { + if (s[i] == '/') { + if (!valid_path_component(s + start, i - start)) + return false; + start = i + 1; + } + } + return valid_path_component(s + start, len - start); +} + +/* Domain detection per containerd: a leading slash component is a registry + * only when it contains '.' or ':', or when it is exactly "localhost". + */ +static bool looks_like_domain(const char *s, size_t len) +{ + if (len == 9 && memcmp(s, "localhost", 9) == 0) + return true; + for (size_t i = 0; i < len; i++) { + if (s[i] == '.' || s[i] == ':') + return true; + } + return false; +} + +/* Portable rightmost-match: Darwin libc does not ship memrchr. */ +static const char *memrchr_local(const char *s, int c, size_t n) +{ + while (n > 0) { + n--; + if ((unsigned char) s[n] == (unsigned char) c) + return s + n; + } + return NULL; +} + +/* Validate a registry host[:port]. The host portion is permissive (DNS + * label rules plus IPv6 brackets are not enforced) but uppercase letters + * are accepted because hostnames are case-insensitive. The optional port + * suffix must be a 1..5 digit decimal number. + */ +static bool valid_registry(const char *s, size_t len) +{ + if (len == 0) + return false; + /* Reject embedded whitespace or path separators outright. */ + for (size_t i = 0; i < len; i++) { + unsigned char c = (unsigned char) s[i]; + if (c <= ' ' || c == '/' || c == '@') + return false; + } + /* If there is a ':' it must be followed by 1..5 decimal digits and must + * be the last colon (IPv6 in brackets is not yet supported). + */ + const char *colon = memchr(s, ':', len); + if (colon) { + size_t host_len = (size_t) (colon - s); + size_t port_len = len - host_len - 1; + if (host_len == 0 || port_len == 0 || port_len > 5) + return false; + for (size_t i = 0; i < port_len; i++) { + if (colon[1 + i] < '0' || colon[1 + i] > '9') + return false; + } + } + return true; +} + +static bool valid_tag(const char *s, size_t len) +{ + if (len == 0 || len > MAX_TAG_LEN) + return false; + /* First char: word character (letter, digit, underscore). */ + unsigned char c0 = (unsigned char) s[0]; + if (!isalnum(c0) && c0 != '_') + return false; + for (size_t i = 1; i < len; i++) { + unsigned char c = (unsigned char) s[i]; + if (!isalnum(c) && c != '_' && c != '.' && c != '-') + return false; + } + return true; +} + +static bool is_lower_hex(char c) +{ + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f'); +} + +/* Validate ":" with algo in {sha256, sha512}. The hex digits are + * required to be lowercase per the OCI image-spec descriptor canonicalisation + * rules; uppercase encodings would otherwise cause silent dedup misses in + * the local store. + */ +static bool valid_digest(const char *s, size_t len, const char **err_msg) +{ + const char *colon = memchr(s, ':', len); + if (!colon) { + set_err(err_msg, "digest missing ':' separator"); + return false; + } + size_t algo_len = (size_t) (colon - s); + size_t hex_len = len - algo_len - 1; + + size_t expected_hex; + if (algo_len == 6 && memcmp(s, "sha256", 6) == 0) { + expected_hex = 64; + } else if (algo_len == 6 && memcmp(s, "sha512", 6) == 0) { + expected_hex = 128; + } else { + set_err(err_msg, "digest algorithm must be sha256 or sha512"); + return false; + } + if (hex_len != expected_hex) { + set_err(err_msg, "digest hex length does not match algorithm"); + return false; + } + for (size_t i = 0; i < hex_len; i++) { + if (!is_lower_hex(colon[1 + i])) { + set_err(err_msg, "digest hex must be lowercase 0-9 a-f"); + return false; + } + } + return true; +} + +void oci_ref_free(oci_ref_t *ref) +{ + if (!ref) + return; + free(ref->registry); + free(ref->repository); + free(ref->tag); + free(ref->digest); + ref->registry = NULL; + ref->repository = NULL; + ref->tag = NULL; + ref->digest = NULL; +} + +int oci_ref_parse(const char *input, oci_ref_t *out, const char **err_msg) +{ + set_err(err_msg, NULL); + if (!out) + return -1; + memset(out, 0, sizeof(*out)); + + if (!input) { + set_err(err_msg, "reference is NULL"); + return -1; + } + size_t total = strlen(input); + if (total == 0) { + set_err(err_msg, "reference is empty"); + return -1; + } + if (total > MAX_REFERENCE_LEN) { + set_err(err_msg, "reference exceeds 4096 characters"); + return -1; + } + + /* Step 1: split off "@digest" (rightmost '@' wins because '@' cannot + * legally appear elsewhere in a well-formed reference). + */ + const char *digest_start = NULL; + size_t digest_len = 0; + const char *at = memchr(input, '@', total); + if (at) { + /* Reject multiple '@' separators outright. */ + const char *second = + memchr(at + 1, '@', total - (size_t) (at + 1 - input)); + if (second) { + set_err(err_msg, "reference contains multiple '@' separators"); + return -1; + } + digest_start = at + 1; + digest_len = total - (size_t) (digest_start - input); + if (digest_len == 0) { + set_err(err_msg, "digest is empty after '@'"); + return -1; + } + if (!valid_digest(digest_start, digest_len, err_msg)) + return -1; + total = (size_t) (at - input); + if (total == 0) { + set_err(err_msg, "reference has no name before '@'"); + return -1; + } + } + + /* Step 2: peel off ":tag" if present. The tag separator is the rightmost + * ':' that follows the last '/' (a colon before any '/' belongs to the + * registry's port). + */ + const char *tag_start = NULL; + size_t tag_len = 0; + size_t name_len = total; + const char *last_slash = memrchr_local(input, '/', total); + const char *scan_from = last_slash ? last_slash + 1 : input; + const char *scan_end = input + total; + const char *tag_colon = + memchr(scan_from, ':', (size_t) (scan_end - scan_from)); + if (tag_colon) { + tag_start = tag_colon + 1; + tag_len = total - (size_t) (tag_start - input); + if (tag_len == 0) { + set_err(err_msg, "tag is empty after ':'"); + return -1; + } + if (!valid_tag(tag_start, tag_len)) { + set_err(err_msg, "tag has invalid characters or length"); + return -1; + } + name_len = (size_t) (tag_colon - input); + if (name_len == 0) { + set_err(err_msg, "reference has no name before ':'"); + return -1; + } + } + + /* Step 3: split name into [registry "/"] path. */ + const char *registry_start = NULL; + size_t registry_len = 0; + const char *path_start = input; + size_t path_len = name_len; + + const char *first_slash = memchr(input, '/', name_len); + if (first_slash) { + size_t head_len = (size_t) (first_slash - input); + if (looks_like_domain(input, head_len)) { + registry_start = input; + registry_len = head_len; + path_start = first_slash + 1; + path_len = name_len - head_len - 1; + if (path_len == 0) { + set_err(err_msg, "reference has no repository after registry"); + return -1; + } + } + } + + /* Step 4: validate path components and detect single-segment defaults. */ + if (!valid_repository_path(path_start, path_len)) { + set_err(err_msg, + "repository path has invalid component (lowercase letters," + " digits, '.', '_', '-' only)"); + return -1; + } + + if (registry_len > 0 && !valid_registry(registry_start, registry_len)) { + set_err(err_msg, "registry host has invalid characters"); + return -1; + } + + /* Step 5: materialise the canonical fields. */ + out->registry = registry_len > 0 + ? strndup_local(registry_start, registry_len) + : strdup(DEFAULT_REGISTRY); + if (!out->registry) + goto oom; + + /* Registry hostnames are case-insensitive (DNS), so "Docker.io" must + * resolve to the docker.io default namespace just like "docker.io". + */ + bool needs_library_prefix = + strcasecmp(out->registry, DEFAULT_REGISTRY) == 0 && + memchr(path_start, '/', path_len) == NULL; + if (needs_library_prefix) { + size_t prefix_len = strlen(DEFAULT_LIBRARY_NAMESPACE); + size_t total_len = prefix_len + 1 + path_len; + out->repository = (char *) malloc(total_len + 1); + if (!out->repository) + goto oom; + memcpy(out->repository, DEFAULT_LIBRARY_NAMESPACE, prefix_len); + out->repository[prefix_len] = '/'; + memcpy(out->repository + prefix_len + 1, path_start, path_len); + out->repository[total_len] = '\0'; + } else { + out->repository = strndup_local(path_start, path_len); + if (!out->repository) + goto oom; + } + + if (tag_len > 0) { + out->tag = strndup_local(tag_start, tag_len); + if (!out->tag) + goto oom; + } else if (digest_len == 0) { + out->tag = strdup(DEFAULT_TAG); + if (!out->tag) + goto oom; + } + + if (digest_len > 0) { + out->digest = strndup_local(digest_start, digest_len); + if (!out->digest) + goto oom; + } + + return 0; + +oom: + set_err(err_msg, "out of memory"); + oci_ref_free(out); + return -1; +} + +char *oci_ref_canonical(const oci_ref_t *ref) +{ + if (!ref || !ref->registry || !ref->repository) + return NULL; + size_t reg_len = strlen(ref->registry); + size_t repo_len = strlen(ref->repository); + size_t tag_len = ref->tag ? strlen(ref->tag) : 0; + size_t dig_len = ref->digest ? strlen(ref->digest) : 0; + size_t total = reg_len + 1 + repo_len + (tag_len ? tag_len + 1 : 0) + + (dig_len ? dig_len + 1 : 0) + 1; + char *buf = (char *) malloc(total); + if (!buf) + return NULL; + char *p = buf; + memcpy(p, ref->registry, reg_len); + p += reg_len; + *p++ = '/'; + memcpy(p, ref->repository, repo_len); + p += repo_len; + if (tag_len) { + *p++ = ':'; + memcpy(p, ref->tag, tag_len); + p += tag_len; + } + if (dig_len) { + *p++ = '@'; + memcpy(p, ref->digest, dig_len); + p += dig_len; + } + *p = '\0'; + return buf; +} + +char *oci_ref_canonical_name(const oci_ref_t *ref) +{ + if (!ref || !ref->registry || !ref->repository || !ref->tag) { + errno = EINVAL; + return NULL; + } + size_t reg_len = strlen(ref->registry); + size_t repo_len = strlen(ref->repository); + size_t tag_len = strlen(ref->tag); + size_t total = reg_len + 1 + repo_len + 1 + tag_len + 1; + char *buf = (char *) malloc(total); + if (!buf) { + errno = ENOMEM; + return NULL; + } + char *p = buf; + memcpy(p, ref->registry, reg_len); + p += reg_len; + *p++ = '/'; + memcpy(p, ref->repository, repo_len); + p += repo_len; + *p++ = ':'; + memcpy(p, ref->tag, tag_len); + p += tag_len; + *p = '\0'; + return buf; +} diff --git a/src/oci/ref.h b/src/oci/ref.h new file mode 100644 index 0000000..3129c28 --- /dev/null +++ b/src/oci/ref.h @@ -0,0 +1,70 @@ +/* Parse OCI image references (REGISTRY/REPO[:TAG][@DIGEST]) + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Implements the de-facto containerd/docker reference grammar so that user + * input like alpine, alpine:3.20, myuser/myrepo:tag, ghcr.io/owner/img:tag, + * or repo@sha256: resolves to a canonical (registry, repository, tag, + * digest) tuple. Defaults match Docker conventions: bare names land under + * docker.io/library/ with tag latest. + * + * Grammar (informal): + * + * reference := name [":" tag] ["@" digest] + * name := [domain "/"] path + * domain := first slash component containing "." or ":" or == "localhost" + * path := component ("/" component)* + * component := [a-z0-9]+ ((["._-"] | "__") [a-z0-9]+)* + * tag := [A-Za-z0-9_] [A-Za-z0-9_.-]{0,127} + * digest := ("sha256" | "sha512") ":" hex (lowercase hex) + * + * Domain detection follows containerd: the first slash-separated component + * is treated as a registry only when it carries a domain marker. Bare + * single-segment names (alpine) and two-segment names (user/repo) default + * to docker.io. Single-segment defaults additionally pick up the library/ + * prefix. + */ + +#pragma once + +typedef struct { + /* Registry hostname (and optional :port). Always non-NULL after parse. */ + char *registry; + /* Repository path with namespace, e.g. "library/alpine". Always non-NULL. + */ + char *repository; + /* Tag name. NULL when the reference is pinned by digest only. Defaults + * to "latest" when neither tag nor digest is present. + */ + char *tag; + /* Digest ":", or NULL. */ + char *digest; +} oci_ref_t; + +/* Parse input into out. Returns 0 on success or -1 on malformed input. On + * error, *err_msg (when err_msg != NULL) is set to a static description; the + * string must not be freed. On success the caller owns out and must call + * oci_ref_free. + */ +int oci_ref_parse(const char *input, oci_ref_t *out, const char **err_msg); + +/* Render a canonical "registry/repository[:tag][@digest]" string. Always + * heap-allocated; the caller frees. Returns NULL on allocation failure. + */ +char *oci_ref_canonical(const oci_ref_t *ref); + +/* Render the canonical pin-name form "registry/repository:tag" used as the + * value of the org.opencontainers.image.ref.name annotation in the store's + * index.json. The digest segment is intentionally dropped: pin entries are + * keyed by tag-name and the digest is stored separately in the descriptor. + * Returns NULL with errno=EINVAL when ref->tag is unset (digest-only refs + * are self-pinning and cannot be inserted into the pin table) or with + * errno=ENOMEM on allocation failure. The caller frees the result. + */ +char *oci_ref_canonical_name(const oci_ref_t *ref); + +/* Release any heap fields. Safe on a zero-initialised or partially populated + * struct; resets all fields to NULL. + */ +void oci_ref_free(oci_ref_t *ref); diff --git a/src/oci/run.c b/src/oci/run.c new file mode 100644 index 0000000..0c0e689 --- /dev/null +++ b/src/oci/run.c @@ -0,0 +1,790 @@ +/* elfuse oci run -- unpack + clone + runspec + path resolve + launch + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Implementation walks the orchestration steps in the order specified + * by the Phase 3 plan: + * + * 1. resolve ref against the local store (must already be pulled) + * 2. oci_unpack into the APFS sysroot volume (idempotent; no-op if + * layers already extracted) + * 3. oci_clone_rootfs into /runs// + * 4. read + parse the manifest, then the image config, off the blob + * store via oci_blob_store_path + a small read-into-heap helper + * (parallel to src/oci/inspect.c's read_blob_file; intentionally + * duplicated rather than re-exported because the inspect copy + * lives behind a static and a cross-module hoist would expand the + * public surface for a 50-line helper) + * 5. oci_runspec_build folds the image runtime + CLI flags into + * argv/envp/cwd/uid + * 6. materialize spec.cwd under run_dir (mkdir -p, mode 0755) + * 7. oci_path_resolve resolves spec.argv[0] against the merged PATH + * inside run_dir (sysroot containment included) + * 8. replace spec.argv[0] with the guest-absolute path the resolver + * handed back, so the guest sees its own canonical name + * 9. save host cwd, chdir to so the guest's + * inherited cwd matches the OCI WorkingDir + * 10. assemble launch_args_t and dispatch to elfuse_launch (or to the + * test override when oci_run_set_launch_for_testing is in effect) + * 11. restore host cwd, free intermediate state, remove the clone dir + * unless keep_rootfs is set + * + * Errors at any stage funnel through the same cleanup epilogue. The + * clone directory is removed on launch failure too, matching the + * "ephemeral by default" decision the Phase 3 roadmap commits to; + * --keep is the only way to opt out. + */ + +#include "run.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "blob-store.h" +#include "clone-rootfs.h" +#include "digest.h" +#include "manifest.h" +#include "path-resolve.h" +#include "runtime-files.h" +#include "unpack.h" +#include "volume.h" + +#include "core/launch.h" + +#include "debug/log.h" + +/* Process-global launch backend pointer. NULL means "use the default + * (elfuse_launch)". Toggled by oci_run_set_launch_for_testing; nobody + * else writes it. The indirection lets tests assert on the + * launch_args_t shape without spinning up an HVF VM. + */ +static oci_run_launch_fn_t g_launch_override = NULL; + +void oci_run_set_launch_for_testing(oci_run_launch_fn_t fn) +{ + g_launch_override = fn; +} + +/* Diagnostic scratch shared with the rest of oci_run. Thread-local so + * future parallel runs do not stomp each other. The buffer is sized + * for the longest dynamic message (a quoted argv[0] plus the searched + * PATH list propagated up from path-resolve). + */ +static _Thread_local char run_err_buf[2048]; + +static void set_err_static(const char **err, const char *msg) +{ + if (err) + *err = msg; +} + +static void set_err_fmt(const char **err, const char *fmt, ...) + __attribute__((format(printf, 2, 3))); + +#include + +static void set_err_fmt(const char **err, const char *fmt, ...) +{ + va_list ap; + va_start(ap, fmt); + vsnprintf(run_err_buf, sizeof(run_err_buf), fmt, ap); + va_end(ap); + if (err) + *err = run_err_buf; +} + +/* Read a blob from the store into a heap buffer. Parallel to the + * read_blob_file helper that lives behind a static in src/oci/inspect.c; + * the 50 lines here intentionally duplicate that helper rather than + * hoisting it into a public utility, because the inspect path needs + * subtly different error reporting (a "blob missing" warning that goes + * to stderr) and a hoist would have to keep both shapes alive. + */ +static int load_blob(oci_blob_store_t *blobs, + oci_digest_algo_t algo, + const char *hex, + char **out_body, + size_t *out_len, + const char **err) +{ + char path[4096]; + int n = oci_blob_store_path(blobs, algo, hex, path, sizeof(path)); + if (n < 0 || (size_t) n >= sizeof(path)) { + set_err_static(err, "blob path too long"); + errno = ENAMETOOLONG; + return -1; + } + int fd = open(path, O_RDONLY); + if (fd < 0) { + set_err_fmt(err, "cannot open blob %s: %s", path, strerror(errno)); + return -1; + } + struct stat st; + if (fstat(fd, &st) < 0) { + int saved = errno; + close(fd); + set_err_static(err, "fstat on blob failed"); + errno = saved; + return -1; + } + if (st.st_size < 0 || st.st_size > 64 * 1024 * 1024) { + close(fd); + set_err_static(err, "blob too large"); + errno = EFBIG; + return -1; + } + size_t want = (size_t) st.st_size; + char *buf = malloc(want + 1); + if (!buf) { + close(fd); + set_err_static(err, "out of memory loading blob"); + errno = ENOMEM; + return -1; + } + size_t off = 0; + while (off < want) { + ssize_t r = read(fd, buf + off, want - off); + if (r < 0) { + int saved = errno; + free(buf); + close(fd); + set_err_static(err, "read on blob failed"); + errno = saved; + return -1; + } + if (r == 0) + break; + off += (size_t) r; + } + close(fd); + if (off != want) { + free(buf); + set_err_static(err, "short read on blob"); + errno = EIO; + return -1; + } + buf[want] = '\0'; + *out_body = buf; + *out_len = want; + return 0; +} + +/* Resolve the manifest blob pinned at digest_str. If the blob is an + * image index (the shape docker.io multi-arch tags such as alpine:3 + * pin to by default), drill into the linux/arm64 leaf descriptor and + * re-load that blob; the leaf body is what oci_manifest_parse expects. + * The mirror of this classify-then-walk path lives in src/oci/inspect.c; + * keep the two in sync. On success returns 0 with *out_body holding the + * leaf-manifest bytes, *out_len its length, and *out_mf the parsed + * shape. The caller frees *out_body via free() and *out_mf via + * oci_manifest_free(); on failure both stay untouched and the helper + * cleans up its own intermediate state. + */ +static int resolve_image_manifest(oci_store_t *store, + const char *digest_str, + char **out_body, + size_t *out_len, + oci_manifest_t *out_mf, + const char **err) +{ + oci_digest_algo_t algo; + char hex[OCI_DIGEST_HEX_MAX + 1]; + if (!oci_digest_parse(digest_str, &algo, hex)) { + set_err_static(err, "pinned manifest digest is malformed"); + errno = EINVAL; + return -1; + } + + char *body = NULL; + size_t len = 0; + if (load_blob(oci_store_blobs(store), algo, hex, &body, &len, err) < 0) + return -1; + + /* Classify. Index and manifest are disjoint JSON shapes (one + * requires "manifests", the other requires "config" + "layers"), + * so a successful parse is unambiguous. Drilling happens only when + * the index parse wins; the leaf-pinned shape (the fixture-builder + * and tests/test-oci-compat.sh path) falls through directly to the + * manifest parser below. + */ + oci_index_t idx = {0}; + if (oci_index_parse(body, len, &idx, NULL) == 0) { + const oci_index_entry_t *picked = oci_index_pick_linux_arm64(&idx); + if (!picked) { + oci_index_free(&idx); + free(body); + set_err_static(err, "image index has no linux/arm64 entry"); + errno = ENOENT; + return -1; + } + char *sub_body = NULL; + size_t sub_len = 0; + if (load_blob(oci_store_blobs(store), picked->desc.algo, + picked->desc.hex, &sub_body, &sub_len, err) < 0) { + oci_index_free(&idx); + free(body); + return -1; + } + oci_index_free(&idx); + free(body); + body = sub_body; + len = sub_len; + } + + oci_manifest_t mf = {0}; + const char *mparse_err = NULL; + if (oci_manifest_parse(body, len, &mf, &mparse_err) < 0) { + set_err_fmt(err, "manifest parse failed: %s", + mparse_err ? mparse_err : "(no message)"); + free(body); + errno = EPROTO; + return -1; + } + + *out_body = body; + *out_len = len; + *out_mf = mf; + return 0; +} + +int oci_run_resolve_image_manifest_for_testing(oci_store_t *store, + const char *digest_str, + char **out_body, + size_t *out_len, + oci_manifest_t *out_mf, + const char **err) +{ + if (err) + *err = NULL; + if (!store || !digest_str || !out_body || !out_len || !out_mf) { + set_err_static(err, "resolve_image_manifest: NULL argument"); + errno = EINVAL; + return -1; + } + return resolve_image_manifest(store, digest_str, out_body, out_len, out_mf, + err); +} + +/* Concatenate two path components with one slash boundary. Caller frees + * the result. + */ +static char *path_join_heap(const char *a, const char *b) +{ + if (!a) + return b ? strdup(b) : NULL; + if (!b) + return strdup(a); + size_t alen = strlen(a); + size_t blen = strlen(b); + bool a_trail = alen > 0 && a[alen - 1] == '/'; + bool b_lead = blen > 0 && b[0] == '/'; + char *r = malloc(alen + blen + 2); + if (!r) + return NULL; + if (a_trail && b_lead) { + memcpy(r, a, alen); + memcpy(r + alen, b + 1, blen - 1); + r[alen + blen - 1] = '\0'; + } else if (!a_trail && !b_lead && alen > 0 && blen > 0) { + memcpy(r, a, alen); + r[alen] = '/'; + memcpy(r + alen + 1, b, blen); + r[alen + 1 + blen] = '\0'; + } else { + memcpy(r, a, alen); + memcpy(r + alen, b, blen); + r[alen + blen] = '\0'; + } + return r; +} + +/* Recursive mkdir; tolerates existing intermediate directories. The + * Phase 3 plan calls for chowning each newly created segment to + * (spec.uid, spec.gid) when has_creds is set; macOS rejects fchownat + * for non-root callers spoofing arbitrary uids, so the chown is best + * effort and silently ignored when it fails. The intended ownership + * is also recorded by the sidecar metadata that the syscall layer + * (Phase 4) will read; for Phase 3 the host inode owner stays the + * invoking user. + */ +static int mkdir_p_owned(const char *path, + mode_t mode, + uint32_t uid, + uint32_t gid, + bool has_creds, + const char **err) +{ + if (!path || !*path) { + set_err_static(err, "empty path in mkdir_p"); + errno = EINVAL; + return -1; + } + /* Walk segments; create each prefix. */ + char *dup = strdup(path); + if (!dup) { + set_err_static(err, "out of memory in mkdir_p"); + errno = ENOMEM; + return -1; + } + for (char *p = dup + 1; *p; p++) { + if (*p != '/') + continue; + *p = '\0'; + if (mkdir(dup, mode) < 0 && errno != EEXIST) { + int saved = errno; + set_err_fmt(err, "mkdir %s failed: %s", dup, strerror(saved)); + free(dup); + errno = saved; + return -1; + } + if (has_creds) + (void) chown(dup, uid, gid); + *p = '/'; + } + if (mkdir(dup, mode) < 0 && errno != EEXIST) { + int saved = errno; + set_err_fmt(err, "mkdir %s failed: %s", dup, strerror(saved)); + free(dup); + errno = saved; + return -1; + } + if (has_creds) + (void) chown(dup, uid, gid); + free(dup); + return 0; +} + +/* Pull PATH=... out of a NULL-terminated envp. Returns the value + * pointer (not strdup'd) or NULL if no PATH key is present. + */ +static const char *envp_get_path(const char *const *envp) +{ + if (!envp) + return NULL; + for (size_t i = 0; envp[i]; i++) { + if (strncmp(envp[i], "PATH=", 5) == 0) + return envp[i] + 5; + } + return NULL; +} + +/* Count entries in a NULL-terminated string vector. */ +static int strvec_count(char *const *v) +{ + int n = 0; + if (!v) + return 0; + while (v[n]) + n++; + return n; +} + +int oci_run(oci_store_t *store, + const oci_ref_t *ref, + const oci_run_options_t *opts, + const char *const *host_environ, + const char **err) +{ + if (err) + *err = NULL; + if (!store || !ref || !opts) { + set_err_static(err, "oci_run: NULL store/ref/opts"); + errno = EINVAL; + return -1; + } + /* Suppress unused warnings for fields kept on the options struct + * for forward compatibility with Phase 3 plan items not yet wired + * (clone_name -> deterministic run-dir, store_dir -> consumed by + * the caller before this function sees the store). + */ + (void) opts->store_dir; + (void) opts->clone_name; + + char *image_dir = NULL; + char *volume_root = NULL; + char *run_dir = NULL; + char *manifest_body = NULL; + char *config_body = NULL; + char *host_argv0 = NULL; + char *guest_argv0 = NULL; + char *cwd_host = NULL; + oci_manifest_t mf = {0}; + oci_image_config_t cfg = {0}; + oci_runspec_t spec = {0}; + int rc = -1; + char host_cwd[PATH_MAX]; + bool have_host_cwd = (getcwd(host_cwd, sizeof(host_cwd)) != NULL); + + /* 1. unpack (idempotent). */ + oci_unpack_options_t uopts = { + .volume_root = opts->volume_dir, .quiet = true, .force_relayer = false}; + const char *unpack_err = NULL; + if (oci_unpack(store, ref, &uopts, &image_dir, &unpack_err) < 0) { + set_err_fmt(err, "unpack failed: %s", + unpack_err ? unpack_err : strerror(errno)); + goto out; + } + + /* 2. resolve volume root (clone-rootfs lands under /runs/). */ + const char *vol_err = NULL; + if (oci_volume_ensure(opts->volume_dir, &volume_root, &vol_err) < 0) { + set_err_fmt(err, "volume_ensure failed: %s", + vol_err ? vol_err : strerror(errno)); + goto out; + } + + /* 3. clone-rootfs. image_dir has a trailing slash; strip for the + * source argument so the inner code joins correctly. + */ + size_t il = strlen(image_dir); + if (il > 1 && image_dir[il - 1] == '/') + image_dir[il - 1] = '\0'; + const char *clone_err = NULL; + if (oci_clone_rootfs(image_dir, volume_root, &run_dir, &clone_err) < 0) { + set_err_fmt(err, "clone-rootfs failed: %s", + clone_err ? clone_err : strerror(errno)); + goto out; + } + + /* 3.5. inject host-truth /etc/{resolv.conf,hosts,hostname} so + * guest libc lookups (getaddrinfo, gethostname, /etc/hosts walks) + * match the macOS host instead of the image's containerd + * defaults. Failure tears the clone-rootfs back down through the + * existing cleanup epilogue. + */ + const char *rfi_err = NULL; + if (oci_runtime_files_inject(run_dir, &rfi_err) < 0) { + set_err_fmt(err, "runtime-files inject failed: %s", + rfi_err ? rfi_err : strerror(errno)); + goto out; + } + + /* 4. read manifest blob, then config blob, then parse both. The + * manifest read goes through resolve_image_manifest, which + * transparently walks one image-index indirection when the pin + * lands on a multi-arch index (the docker.io default for tags like + * alpine:3). + */ + char *manifest_digest_str = NULL; + const char *getref_err = NULL; + if (oci_store_get_ref(store, ref, &manifest_digest_str, &getref_err) < 0) { + set_err_fmt(err, "no pin for ref: %s", + getref_err ? getref_err : strerror(errno)); + goto out; + } + size_t manifest_len = 0; + int rcm = resolve_image_manifest(store, manifest_digest_str, &manifest_body, + &manifest_len, &mf, err); + free(manifest_digest_str); + if (rcm < 0) + goto out; + size_t config_len = 0; + if (load_blob(oci_store_blobs(store), mf.config.algo, mf.config.hex, + &config_body, &config_len, err) < 0) + goto out; + const char *cparse_err = NULL; + if (oci_image_config_parse(config_body, config_len, &cfg, &cparse_err) < + 0) { + set_err_fmt(err, "image config parse failed: %s", + cparse_err ? cparse_err : "(no message)"); + goto out; + } + + /* 5. fold runtime + CLI overrides into argv/envp/cwd/uid. Layer + * the unpacked clone-rootfs over the caller's flags so the runspec + * resolver can read /etc/passwd and /etc/group for symbolic User + * (Phase 4 F4.7). The caller-side flags stay const; only the local + * copy points the resolver at the rootfs. + */ + oci_runspec_flags_t spec_flags = opts->spec; + spec_flags.rootfs_for_nss = run_dir; + const char *rs_err = NULL; + if (oci_runspec_build(&cfg.config, &spec_flags, host_environ, &spec, + &rs_err) < 0) { + set_err_fmt(err, "runspec build failed: %s", + rs_err ? rs_err : strerror(errno)); + goto out; + } + + /* 6. materialize spec.cwd under run_dir. */ + cwd_host = path_join_heap(run_dir, spec.cwd); + if (!cwd_host) { + set_err_static(err, "out of memory building cwd host path"); + errno = ENOMEM; + goto out; + } + if (mkdir_p_owned(cwd_host, 0755, spec.uid, spec.gid, spec.has_creds, err) < + 0) + goto out; + + /* 7. PATH-resolve argv[0]. */ + if (!spec.argv || !spec.argv[0]) { + set_err_static(err, "runspec produced empty argv"); + errno = EINVAL; + goto out; + } + const char *path_env = envp_get_path((const char *const *) spec.envp); + const char *pr_err = NULL; + if (oci_path_resolve(run_dir, spec.argv[0], path_env, spec.cwd, &host_argv0, + &guest_argv0, &pr_err) < 0) { + set_err_fmt(err, "%s", pr_err ? pr_err : "argv[0] resolution failed"); + goto out; + } + + /* 8. swap spec.argv[0] for the guest-absolute path the resolver + * handed back. The guest reads /proc/self/exe + argv[0] and expects + * the canonical name it would have seen via execvp. + */ + free(spec.argv[0]); + spec.argv[0] = guest_argv0; + guest_argv0 = NULL; /* ownership transferred to spec */ + + /* 9. chdir into the materialized WorkingDir so the guest inherits + * its OCI cwd. + */ + if (chdir(cwd_host) < 0) { + set_err_fmt(err, "chdir into '%s' failed: %s", cwd_host, + strerror(errno)); + goto out; + } + + /* 10. assemble launch_args and dispatch. */ + launch_args_t la = { + .elf_path = host_argv0, + .sysroot = run_dir, + .guest_argc = strvec_count(spec.argv), + .guest_argv = (const char **) spec.argv, + .envp = (const char **) spec.envp, + .has_creds = spec.has_creds, + .uid = spec.uid, + .gid = spec.gid, + .cwd_guest = spec.cwd, + .gdb_port = 0, + .gdb_stop_on_entry = false, + .timeout_sec = 10, + .fork_child_fd = -1, + .vfork_notify_fd = -1, + .verbose = false, + }; + oci_run_launch_fn_t launch = + g_launch_override ? g_launch_override : elfuse_launch; + rc = launch(&la); + + /* 11. restore host cwd before cleanup so subsequent paths + * (clone-rootfs-remove, sysroot detach) operate from a sane place. + */ + if (have_host_cwd && chdir(host_cwd) < 0) { + log_warn("could not restore host cwd to %s: %s", host_cwd, + strerror(errno)); + (void) chdir("/"); + } + +out: + if (run_dir && !opts->keep_rootfs) { + const char *rm_err = NULL; + if (oci_clone_rootfs_remove(run_dir, &rm_err) < 0) + log_warn("clone-rootfs cleanup partial: %s", + rm_err ? rm_err : strerror(errno)); + } + free(host_argv0); + free(guest_argv0); + free(cwd_host); + oci_runspec_free(&spec); + oci_image_config_free(&cfg); + oci_manifest_free(&mf); + free(config_body); + free(manifest_body); + free(run_dir); + free(volume_root); + free(image_dir); + return rc; +} + +/* ── CLI entry ─────────────────────────────────────────────────── */ + +static int print_run_usage(FILE *out) +{ + fputs( + "usage: elfuse oci run [OPTIONS] IMAGE [ARG...]\n" + "\n" + "Run a binary from an already-pulled OCI image. The image's\n" + "Entrypoint/Cmd/Env/WorkingDir/User are honored; CLI flags\n" + "override individual fields.\n" + "\n" + "Options:\n" + " --store DIR Override the local store root\n" + " --volume DIR Override the sysroot APFS volume\n" + " --entrypoint PROG Replace the image Entrypoint\n" + " -e, --env KEY=VAL Set or replace env var\n" + " -e, --env KEY Import KEY from host environ\n" + " -w, --workdir DIR Override image WorkingDir\n" + " -u, --user UID[:GID] Override image User (numeric or " + "name[:group])\n" + " --keep Keep the per-run rootfs after exit\n" + " --name NAME Reserved: deterministic clone dir\n" + " (currently ignored)\n" + "\n" + "IMAGE follows the docker/containerd grammar (alpine,\n" + "alpine:3.20, ghcr.io/owner/img:tag, etc.). The image must\n" + "already be pulled; this subcommand does not auto-pull.\n", + out); + return out == stderr ? 2 : 0; +} + +extern char **environ; + +int oci_cli_run(int argc, char **argv) +{ + oci_run_options_t opts = {0}; + const char **env_overrides = NULL; + size_t n_overrides = 0; + size_t cap_overrides = 0; + + int i = 1; + while (i < argc) { + const char *a = argv[i]; + if (a[0] != '-') + break; + if (!strcmp(a, "--")) { + i++; + break; + } + if (!strcmp(a, "-h") || !strcmp(a, "--help")) { + free(env_overrides); + return print_run_usage(stdout); + } + if (!strcmp(a, "--store")) { + if (++i >= argc) { + fputs("error: --store needs an argument\n", stderr); + free(env_overrides); + return 2; + } + opts.store_dir = argv[i]; + } else if (!strcmp(a, "--volume")) { + if (++i >= argc) { + fputs("error: --volume needs an argument\n", stderr); + free(env_overrides); + return 2; + } + opts.volume_dir = argv[i]; + } else if (!strcmp(a, "--entrypoint")) { + if (++i >= argc) { + fputs("error: --entrypoint needs an argument\n", stderr); + free(env_overrides); + return 2; + } + opts.spec.entrypoint_override = argv[i]; + } else if (!strcmp(a, "-e") || !strcmp(a, "--env")) { + if (++i >= argc) { + fputs("error: -e/--env needs an argument\n", stderr); + free(env_overrides); + return 2; + } + if (n_overrides + 1 > cap_overrides) { + size_t newcap = cap_overrides ? cap_overrides * 2 : 8; + const char **np = realloc(env_overrides, newcap * sizeof(*np)); + if (!np) { + fputs("error: out of memory\n", stderr); + free(env_overrides); + return 1; + } + env_overrides = np; + cap_overrides = newcap; + } + env_overrides[n_overrides++] = argv[i]; + } else if (!strcmp(a, "-w") || !strcmp(a, "--workdir")) { + if (++i >= argc) { + fputs("error: -w/--workdir needs an argument\n", stderr); + free(env_overrides); + return 2; + } + opts.spec.workdir_override = argv[i]; + } else if (!strcmp(a, "-u") || !strcmp(a, "--user")) { + if (++i >= argc) { + fputs("error: -u/--user needs an argument\n", stderr); + free(env_overrides); + return 2; + } + opts.spec.user_override = argv[i]; + } else if (!strcmp(a, "--keep")) { + opts.keep_rootfs = true; + } else if (!strcmp(a, "--name")) { + if (++i >= argc) { + fputs("error: --name needs an argument\n", stderr); + free(env_overrides); + return 2; + } + opts.clone_name = argv[i]; + } else { + fprintf(stderr, "error: unknown option: %s\n", a); + free(env_overrides); + return 2; + } + i++; + } + if (i >= argc) { + fputs("error: oci run needs IMAGE\n", stderr); + free(env_overrides); + return 2; + } + const char *ref_str = argv[i]; + i++; + opts.spec.env_overrides = env_overrides; + opts.spec.nenv_overrides = n_overrides; + opts.spec.positional_argc = argc - i; + opts.spec.positional_argv = (const char *const *) (argv + i); + + oci_ref_t ref = {0}; + const char *parse_err = NULL; + if (oci_ref_parse(ref_str, &ref, &parse_err) < 0) { + fprintf(stderr, "error: invalid reference: %s\n", + parse_err ? parse_err : "(unknown)"); + free(env_overrides); + return 1; + } + + char *default_root = NULL; + const char *store_root = opts.store_dir; + if (!store_root) { + default_root = oci_store_default_root(); + if (!default_root) { + fputs("error: cannot determine default store root (HOME?)\n", + stderr); + oci_ref_free(&ref); + free(env_overrides); + return 1; + } + store_root = default_root; + } + oci_store_t *store = oci_store_open(store_root); + if (!store) { + fprintf(stderr, "error: cannot open store at %s: %s\n", store_root, + strerror(errno)); + oci_ref_free(&ref); + free(default_root); + free(env_overrides); + return 1; + } + + const char *run_err = NULL; + int rc = + oci_run(store, &ref, &opts, (const char *const *) environ, &run_err); + if (rc < 0) { + fprintf(stderr, "error: oci run failed: %s\n", + run_err ? run_err : strerror(errno)); + rc = 1; + } + + oci_store_close(store); + oci_ref_free(&ref); + free(default_root); + free(env_overrides); + return rc; +} diff --git a/src/oci/run.h b/src/oci/run.h new file mode 100644 index 0000000..0e33770 --- /dev/null +++ b/src/oci/run.h @@ -0,0 +1,114 @@ +/* elfuse oci run -- launch a guest binary from a pulled OCI image + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Closes the Phase 3 loop: unpack + clone-rootfs + image-config parse + + * runspec build + PATH resolve + elfuse_launch under one subcommand. + * The user runs an OCI image directly, with the image's + * Entrypoint/Cmd/Env/WorkingDir/User honored as configured by the + * image producer and overridable by the elfuse CLI. + * + * Dependencies (all already landed in earlier slices): + * + * - oci_unpack (Phase 2) -- layers -> image sysroot under + * the APFS sysroot volume + * - oci_clone_rootfs (Phase 2) -- clonefile-based per-run rootfs + * - oci_image_config_parse (Phase 1) + * - oci_runspec_build (Phase 3 C2) -- folds image config + CLI flags + * into argv/envp/cwd/uid bundle + * - oci_path_resolve (Phase 3 C3) -- argv0 + PATH -> host_path/ + * guest_path with sysroot + * containment + * - elfuse_launch (Phase 3 C4) -- VM bring-up shared with main() + * + * Lifetime / cleanup contract: + * + * - oci_run owns its intermediate state (run_dir, parsed manifest / + * config blobs, oci_runspec_t, resolved host_argv0). It frees + * everything before returning. + * - On success it removes the clone dir unless opts->keep_rootfs is + * set. On launch failure (elfuse_launch returns non-zero) it still + * removes the clone dir by default so a failed run does not leave + * stale clones on the volume. + * - host cwd is saved and restored across the call. The launch + * itself chdir's into so the guest sees its + * WorkingDir as cwd. + */ + +#pragma once + +#include +#include + +#include "core/launch.h" + +#include "manifest.h" +#include "ref.h" +#include "runspec.h" +#include "store.h" + +/* Flags assembled by oci_cli_run from the elfuse oci run command line. + * store_dir / volume_dir override the default Library/Application + * Support paths; clone_name reserves a slot for a future deterministic + * run-dir naming option that Phase 2's oci_clone_rootfs does not yet + * support, so the field is currently ignored. spec carries every + * runspec-relevant override (Entrypoint, -e, -w, -u, IMAGE, ARGV tail); + * it is forwarded verbatim to oci_runspec_build. + */ +typedef struct { + const char *store_dir; + const char *volume_dir; + oci_runspec_flags_t spec; + bool keep_rootfs; + const char *clone_name; +} oci_run_options_t; + +/* `elfuse oci run` subcommand entry. Argument parsing, ref parse, store + * open, oci_run dispatch. Returns a process exit code (0 success, 1 on + * runtime failure, 2 on usage / argument error to match the rest of + * src/oci/cli.c). + */ +int oci_cli_run(int argc, char **argv); + +/* Programmatic entry: drive the full unpack -> clone -> runspec -> path + * resolve -> elfuse_launch pipeline against an already-opened store and + * parsed ref. host_environ is forwarded to oci_runspec_build for the + * Env merge policy; pass the process environ. *err is populated with a + * static diagnostic on failure; the pointer is valid until the next + * call (or until oci_runspec_build / oci_path_resolve overwrite their + * own thread-local buffer for a different diagnostic class). + * + * Returns: + * >= 0 exit code of the guest binary + * -1 pre-launch failure (unpack, clone, parse, runspec, path + * resolve, or directory materialization) + */ +int oci_run(oci_store_t *store, + const oci_ref_t *ref, + const oci_run_options_t *opts, + const char *const *host_environ, + const char **err); + +/* Test hook: swap the underlying launch backend. Pass NULL to restore + * the default (elfuse_launch). The override is process-global and + * exists only so unit tests can run the orchestrator without spinning + * up a real HVF VM. Production code must never call this. + */ +typedef int (*oci_run_launch_fn_t)(const launch_args_t *args); +void oci_run_set_launch_for_testing(oci_run_launch_fn_t fn); + +/* Test hook: drive the manifest-resolution step (load blob, classify + * index vs leaf, drill linux/arm64 on index, parse) in isolation. The + * production caller is oci_run; this hook exists so unit tests can + * verify the multi-arch index-walk without spinning up an APFS + * sysroot volume. Output ownership matches the production internal + * helper: caller frees *out_body and *out_mf via free() and + * oci_manifest_free() respectively. Production code must use oci_run. + */ +int oci_run_resolve_image_manifest_for_testing(oci_store_t *store, + const char *digest_str, + char **out_body, + size_t *out_len, + oci_manifest_t *out_mf, + const char **err); diff --git a/src/oci/runspec.c b/src/oci/runspec.c new file mode 100644 index 0000000..3e01cdd --- /dev/null +++ b/src/oci/runspec.c @@ -0,0 +1,520 @@ +/* OCI launch-spec resolver: image runtime + CLI overrides -> argv/envp/... + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Pure-data merge. No filesystem touches. The translation unit deliberately + * carries its own small string-vector helpers instead of pulling in + * src/utils.h: the rest of elfuse uses arena-style allocation patterns + * tied to the guest VM lifetime, but a runspec lives across an + * elfuse oci run invocation and ships back to the caller via + * oci_runspec_t. Owning every char* with plain malloc/free makes that + * lifetime contract auditable. + */ + +#include "runspec.h" + +#include +#include +#include +#include +#include +#include + +#include "user-lookup.h" + +/* Diagnostic scratch. Thread-local so concurrent oci_runspec_build calls + * (one per --keep run dir, in a future multiplexed oci run) do not clobber + * each other's err pointer. Buffer size is generous enough for the + * longest dynamic message: the rejected User value plus the fixed Phase 4 + * pointer text. + */ +static _Thread_local char runspec_err_buf[512]; + +static const char *set_err_static(const char **err, const char *msg) +{ + if (err) + *err = msg; + return msg; +} + +static const char *set_err_fmt(const char **err, const char *fmt, ...) + __attribute__((format(printf, 2, 3))); + +static const char *set_err_fmt(const char **err, const char *fmt, ...) +{ + va_list ap; + va_start(ap, fmt); + vsnprintf(runspec_err_buf, sizeof(runspec_err_buf), fmt, ap); + va_end(ap); + if (err) + *err = runspec_err_buf; + return runspec_err_buf; +} + +/* Min-grow string vector. Always NUL-terminates the backing array when + * finalized so the result is environ/argv-shaped. + */ +typedef struct { + char **items; + size_t n; + size_t cap; +} strvec_t; + +static int strvec_reserve(strvec_t *v, size_t need) +{ + if (need <= v->cap) + return 0; + size_t newcap = v->cap ? v->cap : 8; + while (newcap < need) + newcap *= 2; + char **np = realloc(v->items, newcap * sizeof(*np)); + if (!np) + return -1; + v->items = np; + v->cap = newcap; + return 0; +} + +/* Append a heap string to the vector, taking ownership. On allocation + * failure the input pointer stays unfreed (caller frees on cleanup). + */ +static int strvec_push_take(strvec_t *v, char *s_owned) +{ + if (strvec_reserve(v, v->n + 1) < 0) + return -1; + v->items[v->n++] = s_owned; + return 0; +} + +static int strvec_push_strdup(strvec_t *v, const char *s) +{ + char *d = strdup(s); + if (!d) + return -1; + if (strvec_push_take(v, d) < 0) { + free(d); + return -1; + } + return 0; +} + +/* Hand the items array to the caller as a NULL-terminated argv/envp. + * Resets the vector so subsequent free is a no-op. + */ +static int strvec_finalize(strvec_t *v, char ***out) +{ + char **arr = realloc(v->items, (v->n + 1) * sizeof(*arr)); + if (!arr) + return -1; + arr[v->n] = NULL; + *out = arr; + v->items = NULL; + v->n = 0; + v->cap = 0; + return 0; +} + +static void strvec_free(strvec_t *v) +{ + if (!v || !v->items) + return; + for (size_t i = 0; i < v->n; i++) + free(v->items[i]); + free(v->items); + v->items = NULL; + v->n = 0; + v->cap = 0; +} + +/* Return length of the KEY portion of a "KEY=VAL" or bare "KEY" string. */ +static size_t env_key_len(const char *kv) +{ + const char *eq = strchr(kv, '='); + return eq ? (size_t) (eq - kv) : strlen(kv); +} + +static ssize_t env_lookup_idx(const strvec_t *v, + const char *key, + size_t key_len) +{ + for (size_t i = 0; i < v->n; i++) { + const char *e = v->items[i]; + size_t elen = env_key_len(e); + if (elen == key_len && memcmp(e, key, key_len) == 0) + return (ssize_t) i; + } + return -1; +} + +/* Set-or-replace env entry by KEY (computed from kv_owned's pre-'=' prefix). + * Takes ownership of kv_owned. On replace, the old entry is freed. On + * allocation failure, kv_owned is freed and -1 is returned. + */ +static int env_set_take(strvec_t *v, char *kv_owned) +{ + size_t klen = env_key_len(kv_owned); + ssize_t idx = env_lookup_idx(v, kv_owned, klen); + if (idx >= 0) { + free(v->items[idx]); + v->items[idx] = kv_owned; + return 0; + } + if (strvec_push_take(v, kv_owned) < 0) { + free(kv_owned); + return -1; + } + return 0; +} + +static int env_set_strdup(strvec_t *v, const char *kv) +{ + char *d = strdup(kv); + if (!d) + return -1; + return env_set_take(v, d); +} + +/* Build "KEY=VAL" on the heap from the two halves and feed it through + * env_set_take. Used for host-import and TERM auto-import paths where + * the source is not already a single KEY=VAL string. + */ +static int env_set_pair(strvec_t *v, const char *key, const char *val) +{ + size_t klen = strlen(key); + size_t vlen = strlen(val); + char *kv = malloc(klen + 1 + vlen + 1); + if (!kv) + return -1; + memcpy(kv, key, klen); + kv[klen] = '='; + memcpy(kv + klen + 1, val, vlen); + kv[klen + 1 + vlen] = '\0'; + return env_set_take(v, kv); +} + +static const char *host_env_lookup(const char *const *host_environ, + const char *key, + size_t key_len) +{ + if (!host_environ) + return NULL; + for (size_t i = 0; host_environ[i]; i++) { + const char *e = host_environ[i]; + size_t elen = env_key_len(e); + if (elen == key_len && memcmp(e, key, key_len) == 0 && e[elen] == '=') { + return e + elen + 1; + } + } + return NULL; +} + +/* "set" in the override matrix means "non-NULL array containing at least + * one entry". Explicit empty arrays ([]) count as "unset" so an image + * Cmd=[] does not produce a zero-length argv when paired with Entrypoint. + */ +static bool runtime_arr_is_set(char *const *arr) +{ + return arr != NULL && arr[0] != NULL; +} + +/* WorkingDir must be absolute and free of ".." path components. Empty + * segments (consecutive slashes, trailing slash) are tolerated; the + * caller-side path materialization will normalize them. + */ +static int validate_workdir(const char *s) +{ + if (!s || s[0] != '/') + return -1; + const char *p = s + 1; + while (*p) { + const char *next = strchr(p, '/'); + size_t seg_len = next ? (size_t) (next - p) : strlen(p); + if (seg_len == 2 && p[0] == '.' && p[1] == '.') + return -1; + if (!next) + break; + p = next + 1; + } + return 0; +} + +/* Resolve credentials from CLI --user override, then image User, then + * host inheritance. Both sources route through oci_user_lookup so the + * numeric / symbolic / mixed shapes are handled uniformly; the only + * difference between the two paths is the diagnostic prefix so a caller + * can tell whether the bad value came from their CLI flag or from the + * pulled image. Symbolic resolution requires flags->rootfs_for_nss; a + * NULL rootfs causes the helper to reject any symbolic token, preserving + * the "pure data" contract for callers that have not unpacked a rootfs. + */ +static int resolve_user(const oci_image_runtime_t *cfg, + const oci_runspec_flags_t *flags, + oci_runspec_t *out, + const char **err) +{ + if (flags->user_override) { + const char *lookup_err = NULL; + if (oci_user_lookup(flags->rootfs_for_nss, flags->user_override, + &out->uid, &out->gid, &lookup_err) < 0) { + int saved = errno; + set_err_fmt(err, "--user '%s': %s", flags->user_override, + lookup_err ? lookup_err : "lookup failed"); + errno = saved ? saved : EINVAL; + return -1; + } + out->has_creds = true; + return 0; + } + if (cfg && cfg->user && *cfg->user) { + const char *lookup_err = NULL; + if (oci_user_lookup(flags->rootfs_for_nss, cfg->user, &out->uid, + &out->gid, &lookup_err) < 0) { + int saved = errno; + set_err_fmt(err, "User '%s': %s", cfg->user, + lookup_err ? lookup_err : "lookup failed"); + errno = saved ? saved : EINVAL; + return -1; + } + out->has_creds = true; + return 0; + } + out->has_creds = false; + return 0; +} + +static int resolve_workdir(const oci_image_runtime_t *cfg, + const oci_runspec_flags_t *flags, + oci_runspec_t *out, + const char **err) +{ + const char *chosen = NULL; + if (flags->workdir_override) { + chosen = flags->workdir_override; + } else if (cfg && cfg->working_dir && *cfg->working_dir) { + chosen = cfg->working_dir; + } else { + chosen = "/"; + } + if (validate_workdir(chosen) < 0) { + set_err_fmt(err, "WorkingDir must be absolute and not contain '..': %s", + chosen); + errno = EINVAL; + return -1; + } + out->cwd = strdup(chosen); + if (!out->cwd) { + set_err_static(err, "out of memory allocating cwd"); + errno = ENOMEM; + return -1; + } + return 0; +} + +/* Argv build follows the override matrix from the Phase 3 plan. The + * matrix collapses to: --entrypoint clobbers everything (Cmd dropped, + * image Entrypoint dropped, [override] ++ CLI args); otherwise image + * Entrypoint and Cmd combine with the CLI tail using "CLI args drop Cmd + * but keep Entrypoint" precedence. + */ +static int build_argv(const oci_image_runtime_t *cfg, + const oci_runspec_flags_t *flags, + oci_runspec_t *out, + const char **err) +{ + strvec_t argv = {0}; + int rc = -1; + + if (flags->entrypoint_override) { + if (strvec_push_strdup(&argv, flags->entrypoint_override) < 0) + goto oom; + for (int i = 0; i < flags->positional_argc; i++) { + if (strvec_push_strdup(&argv, flags->positional_argv[i]) < 0) + goto oom; + } + } else if (cfg && runtime_arr_is_set(cfg->entrypoint)) { + for (char *const *p = cfg->entrypoint; *p; p++) { + if (strvec_push_strdup(&argv, *p) < 0) + goto oom; + } + if (flags->positional_argc > 0) { + for (int i = 0; i < flags->positional_argc; i++) { + if (strvec_push_strdup(&argv, flags->positional_argv[i]) < 0) + goto oom; + } + } else if (cfg && runtime_arr_is_set(cfg->cmd)) { + for (char *const *p = cfg->cmd; *p; p++) { + if (strvec_push_strdup(&argv, *p) < 0) + goto oom; + } + } + } else { + if (flags->positional_argc > 0) { + for (int i = 0; i < flags->positional_argc; i++) { + if (strvec_push_strdup(&argv, flags->positional_argv[i]) < 0) + goto oom; + } + } else if (cfg && runtime_arr_is_set(cfg->cmd)) { + for (char *const *p = cfg->cmd; *p; p++) { + if (strvec_push_strdup(&argv, *p) < 0) + goto oom; + } + } else { + set_err_static(err, + "image has no entrypoint or cmd; pass one on" + " the CLI"); + errno = EINVAL; + goto done; + } + } + + if (strvec_finalize(&argv, &out->argv) < 0) + goto oom; + rc = 0; + goto done; + +oom: + set_err_static(err, "out of memory building argv"); + errno = ENOMEM; +done: + strvec_free(&argv); + return rc; +} + +#define LINUX_DEFAULT_PATH \ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + +/* Apply the Env merge policy described in runspec.h. The build proceeds + * in stages so the merge order matches the spec: image Env first, CLI + * overrides second, then defaults (TERM, PATH, container). DYLD_ rejection + * runs before any allocation per CLI override so the failing key is + * reported with no half-applied state. + */ +static int build_envp(const oci_image_runtime_t *cfg, + const oci_runspec_flags_t *flags, + const char *const *host_environ, + oci_runspec_t *out, + const char **err) +{ + strvec_t env = {0}; + int rc = -1; + + if (cfg && cfg->env) { + for (char *const *p = cfg->env; *p; p++) { + if (env_set_strdup(&env, *p) < 0) + goto oom; + } + } + + for (size_t i = 0; i < flags->nenv_overrides; i++) { + const char *kv = flags->env_overrides[i]; + if (!kv) + continue; + if (strncmp(kv, "DYLD_", 5) == 0) { + size_t klen = env_key_len(kv); + char key_only[64]; + size_t copy = + klen < sizeof(key_only) - 1 ? klen : sizeof(key_only) - 1; + memcpy(key_only, kv, copy); + key_only[copy] = '\0'; + set_err_fmt(err, + "--env: refusing to set DYLD_* (macOS-only ABI):" + " %s", + key_only); + errno = EINVAL; + goto done; + } + const char *eq = strchr(kv, '='); + if (eq) { + if (env_set_strdup(&env, kv) < 0) + goto oom; + } else { + size_t klen = strlen(kv); + const char *hv = host_env_lookup(host_environ, kv, klen); + if (hv) { + if (env_set_pair(&env, kv, hv) < 0) + goto oom; + } + } + } + + if (env_lookup_idx(&env, "TERM", 4) < 0) { + const char *host_term = host_env_lookup(host_environ, "TERM", 4); + if (host_term) { + if (env_set_pair(&env, "TERM", host_term) < 0) + goto oom; + } + } + if (env_lookup_idx(&env, "PATH", 4) < 0) { + if (env_set_strdup(&env, LINUX_DEFAULT_PATH) < 0) + goto oom; + } + if (env_set_strdup(&env, "container=elfuse") < 0) + goto oom; + + if (strvec_finalize(&env, &out->envp) < 0) + goto oom; + rc = 0; + goto done; + +oom: + set_err_static(err, "out of memory building envp"); + errno = ENOMEM; +done: + strvec_free(&env); + return rc; +} + +int oci_runspec_build(const oci_image_runtime_t *cfg, + const oci_runspec_flags_t *flags, + const char *const *host_environ, + oci_runspec_t *out, + const char **err) +{ + if (err) + *err = NULL; + if (!flags || !out) { + set_err_static(err, "oci_runspec_build: NULL flags or out"); + errno = EINVAL; + return -1; + } + memset(out, 0, sizeof(*out)); + + if (resolve_user(cfg, flags, out, err) < 0) + goto fail; + if (resolve_workdir(cfg, flags, out, err) < 0) + goto fail; + if (build_argv(cfg, flags, out, err) < 0) + goto fail; + if (build_envp(cfg, flags, host_environ, out, err) < 0) + goto fail; + return 0; + +fail: + oci_runspec_free(out); + memset(out, 0, sizeof(*out)); + return -1; +} + +void oci_runspec_free(oci_runspec_t *spec) +{ + if (!spec) + return; + free(spec->cwd); + spec->cwd = NULL; + if (spec->argv) { + for (char **p = spec->argv; *p; p++) + free(*p); + free(spec->argv); + spec->argv = NULL; + } + if (spec->envp) { + for (char **p = spec->envp; *p; p++) + free(*p); + free(spec->envp); + spec->envp = NULL; + } + spec->uid = 0; + spec->gid = 0; + spec->has_creds = false; +} diff --git a/src/oci/runspec.h b/src/oci/runspec.h new file mode 100644 index 0000000..565f4bb --- /dev/null +++ b/src/oci/runspec.h @@ -0,0 +1,123 @@ +/* OCI launch-spec resolver for elfuse oci run + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Folds an image-config runtime block (User, WorkingDir, Entrypoint, Cmd, + * Env) together with elfuse oci run CLI overrides into the concrete launch + * bundle: guest cwd, argv, envp, and optional uid/gid credentials. The + * module is pure data: no filesystem access, no PATH lookup, no syscalls. + * That keeps the override matrix and the Env merge policy verifiable by a + * unit test that constructs an oci_image_runtime_t literal directly. + * + * PATH search and rootfs materialization happen in later Phase 3 modules + * (path-resolve, run). This builder treats argv[0] as opaque; the caller + * downstream is responsible for resolving the host path the guest will + * actually load. + * + * Override matrix (issue #31 acceptance 2) follows the table documented in + * the Phase 3 plan: --entrypoint replaces the image Entrypoint and the + * image Cmd is dropped whenever CLI positional arguments are provided or + * --entrypoint is set. The "image has neither Entrypoint nor Cmd and the + * CLI supplied no positional arguments" case is the only hard-fail in the + * argv assembly path. + * + * Env merge (issue #31 acceptance 3) starts from the image Env array, + * applies CLI -e overrides (KEY=VAL set-or-replace; bare KEY imports the + * matching host environ value when present and otherwise drops silently), + * auto-imports TERM from the host when the merged Env has no TERM, injects + * the Linux PAM-default PATH when no PATH key has been set, and finally + * forces container=elfuse so systemd-style sandbox detection works + * regardless of what the image declared. CLI overrides whose KEY starts + * with DYLD_ hard-fail with EINVAL: DYLD_* is a macOS-only loader contract + * and has no meaning inside the guest. Image-provided DYLD_* entries pass + * through (aarch64 Linux ignores them); reviewers can escalate to strip + * if needed. + * + * WorkingDir defaults to "/" when neither the image nor the CLI sets it. + * Relative paths and any path containing a ".." segment hard-fail with + * EINVAL; sysroot containment is enforced later by the path-resolve module + * and the syscall layer. + * + * User accepts the seven shapes the OCI image-spec defines: empty (no + * override), "uid", "uid:gid", "name", "name:group", "uid:group", and + * "name:gid". Symbolic forms (anything other than the two pure-numeric + * shapes) require flags->rootfs_for_nss; the resolver reads + * /etc/passwd and /etc/group through oci_user_lookup + * (see src/oci/user-lookup.h). When rootfs_for_nss is NULL the resolver + * still accepts the pure-numeric forms; a symbolic token then returns + * EINVAL so unit tests can hold runspec to its "pure data" contract by + * passing NULL. CLI --user takes precedence over the image User; both + * shapes go through the same lookup. + * + * Error reporting: on failure, the function writes a pointer into *err + * that names what went wrong. The pointer is valid until the next call + * from this thread. Static messages (the fixed strings) and dynamic + * messages (those that quote the bad value) share the same lifetime + * contract so the caller does not need to branch. + */ + +#pragma once + +#include +#include +#include + +#include "manifest.h" + +/* CLI overrides assembled by the oci run argument parser. NULL fields mean + * "no override; defer to the image config". env_overrides is the literal + * sequence of -e arguments (KEY=VAL or bare KEY); the merge logic below + * walks it in order. positional_argv is the IMAGE-trailing argv tail + * (everything after the image reference on the command line). + */ +typedef struct { + const char *entrypoint_override; + const char *const *env_overrides; + size_t nenv_overrides; + const char *workdir_override; + const char *user_override; + int positional_argc; + const char *const *positional_argv; + /* Directory the symbolic User resolver treats as "/" when reading + * /etc/passwd and /etc/group. NULL keeps the builder pure-data: any + * symbolic User token then returns EINVAL. + */ + const char *rootfs_for_nss; +} oci_runspec_flags_t; + +/* Resolved launch bundle. cwd is always set (defaults to "/"). argv and + * envp are NULL-terminated heap arrays of heap strings. has_creds is + * false when neither the image nor the CLI named a User; in that case the + * caller leaves the host uid/gid untouched and uid/gid in this struct are + * meaningless. + */ +typedef struct { + char *cwd; + char **argv; + char **envp; + uint32_t uid; + uint32_t gid; + bool has_creds; +} oci_runspec_t; + +/* Resolve image runtime config + CLI overrides into a launch bundle. + * + * cfg may be NULL (treated as "image config has no runtime block": all + * fields absent). flags must be non-NULL but may be zero-initialised. + * host_environ is a NULL-terminated environ-shaped pointer for KEY=VAL + * lookups; pass the process environ. out must be non-NULL; on entry it + * is overwritten verbatim. On success returns 0 and out owns all heap + * storage (call oci_runspec_free to release). On failure returns -1 + * with errno set (EINVAL for policy violations, ENOMEM for allocation + * failures), *out left zeroed (safe to pass to oci_runspec_free), and + * *err pointing at the diagnostic message. + */ +int oci_runspec_build(const oci_image_runtime_t *cfg, + const oci_runspec_flags_t *flags, + const char *const *host_environ, + oci_runspec_t *out, + const char **err); + +/* Release any heap fields. Safe on zero-initialised structs and on NULL. */ +void oci_runspec_free(oci_runspec_t *spec); diff --git a/src/oci/runtime-files.c b/src/oci/runtime-files.c new file mode 100644 index 0000000..f7baa8a --- /dev/null +++ b/src/oci/runtime-files.c @@ -0,0 +1,459 @@ +/* OCI per-run /etc host-truth injection + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * See runtime-files.h for the contract. Implementation walks the + * three target leaves under /etc/ in turn, unlinking any + * pre-existing inode first so a symlink left by the image (the + * common case for /etc/resolv.conf) cannot dangle past the + * injection. + * + * The nameserver list for resolv.conf is harvested by spawning + * /usr/sbin/scutil --dns and line-walking its stdout for the + * " nameserver[N] : " pattern. The reader uses posix_spawnp + * with a pipe, mirroring src/core/sysroot.c's spawn_capture_stdout + * helper; duplicating the ~40 lines here keeps the oci module's + * lifetime story self-contained instead of hoisting a host-only + * helper into a public header. + */ + +#include "runtime-files.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern char **environ; + +static _Thread_local char rf_err_buf[512]; + +static void set_err_static(const char **err, const char *msg) +{ + if (err) + *err = msg; +} + +static void set_err_fmt(const char **err, const char *fmt, ...) + __attribute__((format(printf, 2, 3))); + +#include + +static void set_err_fmt(const char **err, const char *fmt, ...) +{ + va_list ap; + va_start(ap, fmt); + vsnprintf(rf_err_buf, sizeof(rf_err_buf), fmt, ap); + va_end(ap); + if (err) + *err = rf_err_buf; +} + +/* Loop-write the full payload; EINTR retries and short writes are + * folded back into another write call. Returns 0 on success, -1 with + * errno preserved on the failing call. + */ +static int write_full(int fd, const char *buf, size_t n) +{ + size_t off = 0; + while (off < n) { + ssize_t w = write(fd, buf + off, n - off); + if (w < 0) { + if (errno == EINTR) + continue; + return -1; + } + off += (size_t) w; + } + return 0; +} + +/* mkdir /etc 0755; tolerate EEXIST iff it is a directory. */ +static int ensure_etc_dir(const char *run_dir, const char **err) +{ + char etc_path[4096]; + int n = snprintf(etc_path, sizeof(etc_path), "%s/etc", run_dir); + if (n < 0 || (size_t) n >= sizeof(etc_path)) { + set_err_static(err, "runtime-files: /etc path overflow"); + errno = ENAMETOOLONG; + return -1; + } + if (mkdir(etc_path, 0755) == 0) + return 0; + if (errno != EEXIST) { + int saved = errno; + set_err_fmt(err, "runtime-files: mkdir %s failed: %s", etc_path, + strerror(saved)); + errno = saved; + return -1; + } + struct stat st; + if (lstat(etc_path, &st) < 0) { + int saved = errno; + set_err_fmt(err, "runtime-files: lstat %s failed: %s", etc_path, + strerror(saved)); + errno = saved; + return -1; + } + if (!S_ISDIR(st.st_mode)) { + set_err_fmt(err, "runtime-files: %s exists but is not a directory", + etc_path); + errno = ENOTDIR; + return -1; + } + return 0; +} + +/* Build /etc/, remove any pre-existing inode (file or + * symlink) at that path, then open a fresh file for write. Returns + * the new fd on success, -1 with *err set and errno preserved on + * failure. + */ +static int open_etc_overwrite(const char *run_dir, + const char *leaf, + const char **err) +{ + char path[4096]; + int n = snprintf(path, sizeof(path), "%s/etc/%s", run_dir, leaf); + if (n < 0 || (size_t) n >= sizeof(path)) { + set_err_fmt(err, "runtime-files: path for /etc/%s too long", leaf); + errno = ENAMETOOLONG; + return -1; + } + if (unlink(path) < 0 && errno != ENOENT) { + int saved = errno; + set_err_fmt(err, "runtime-files: unlink %s failed: %s", path, + strerror(saved)); + errno = saved; + return -1; + } + int fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (fd < 0) { + int saved = errno; + set_err_fmt(err, "runtime-files: open %s failed: %s", path, + strerror(saved)); + errno = saved; + return -1; + } + return fd; +} + +/* Append a nameserver string to the dedup list. Returns 0 on success, + * -1 with errno=ENOMEM on allocation failure. Duplicates are + * silently dropped so resolver #1's primary appears once even when + * scutil prints "nameserver[0]" twice for IPv4 + IPv6 of the same + * interface. + */ +static int push_nameserver(char ***list, + size_t *n, + size_t *cap, + const char *ip, + size_t iplen) +{ + for (size_t i = 0; i < *n; i++) { + size_t e = strlen((*list)[i]); + if (e == iplen && memcmp((*list)[i], ip, iplen) == 0) + return 0; + } + if (*n + 1 > *cap) { + size_t newcap = *cap ? *cap * 2 : 4; + char **np = realloc(*list, newcap * sizeof(*np)); + if (!np) { + errno = ENOMEM; + return -1; + } + *list = np; + *cap = newcap; + } + char *dup = malloc(iplen + 1); + if (!dup) { + errno = ENOMEM; + return -1; + } + memcpy(dup, ip, iplen); + dup[iplen] = '\0'; + (*list)[(*n)++] = dup; + return 0; +} + +/* Parse one line of scutil --dns output. The relevant shape is: + * + * " nameserver[0] : 192.168.1.1" + * " nameserver[1] : 2001:db8::1" + * + * Anything else is ignored. The IP literal is the trailing token + * after the colon, with surrounding whitespace stripped. Validation + * is intentionally loose: scutil is the source of truth, so any + * non-empty token after "nameserver[N] :" is taken at face value. + */ +static void parse_scutil_line(const char *line, + size_t len, + char ***list, + size_t *n, + size_t *cap) +{ + const char *p = line; + const char *end = line + len; + while (p < end && isspace((unsigned char) *p)) + p++; + static const char prefix[] = "nameserver["; + size_t plen = sizeof(prefix) - 1; + if ((size_t) (end - p) < plen) + return; + if (memcmp(p, prefix, plen) != 0) + return; + p += plen; + while (p < end && *p != ']') + p++; + if (p >= end || *p != ']') + return; + p++; + while (p < end && isspace((unsigned char) *p)) + p++; + if (p >= end || *p != ':') + return; + p++; + while (p < end && isspace((unsigned char) *p)) + p++; + const char *ip = p; + while (p < end && !isspace((unsigned char) *p)) + p++; + if (p == ip) + return; + (void) push_nameserver(list, n, cap, ip, (size_t) (p - ip)); +} + +/* Run /usr/sbin/scutil --dns, capture stdout, harvest nameservers. + * Returns 0 on success with *out_list / *out_n populated (out_list + * may be NULL when out_n == 0). Returns -1 on any spawn / read / + * wait failure so the caller can fall back to a known-good set. + */ +static int scutil_collect_nameservers(char ***out_list, size_t *out_n) +{ + *out_list = NULL; + *out_n = 0; + + int pipefd[2] = {-1, -1}; + if (pipe(pipefd) < 0) + return -1; + + posix_spawn_file_actions_t actions; + if (posix_spawn_file_actions_init(&actions) != 0) { + close(pipefd[0]); + close(pipefd[1]); + return -1; + } + posix_spawn_file_actions_adddup2(&actions, pipefd[1], STDOUT_FILENO); + posix_spawn_file_actions_addclose(&actions, pipefd[0]); + posix_spawn_file_actions_addclose(&actions, pipefd[1]); + + char *const argv[] = {(char *) "/usr/sbin/scutil", (char *) "--dns", NULL}; + pid_t pid = -1; + int spawn_ret = posix_spawn(&pid, argv[0], &actions, NULL, argv, environ); + posix_spawn_file_actions_destroy(&actions); + close(pipefd[1]); + if (spawn_ret != 0) { + close(pipefd[0]); + errno = spawn_ret; + return -1; + } + + /* Drain stdout. scutil --dns output is bounded by the number of + * resolver entries macOS keeps; 16 KiB is comfortable for any + * realistic host. + */ + char buf[16384]; + size_t off = 0; + bool drained = true; + while (off + 1 < sizeof(buf)) { + ssize_t r = read(pipefd[0], buf + off, sizeof(buf) - 1 - off); + if (r < 0) { + if (errno == EINTR) + continue; + drained = false; + break; + } + if (r == 0) + break; + off += (size_t) r; + } + /* Best-effort drain past the cap so the child does not block on + * SIGPIPE; the parser only sees the first 16 KiB anyway. + */ + if (drained) { + char dump[4096]; + while (read(pipefd[0], dump, sizeof(dump)) > 0) { + } + } + close(pipefd[0]); + + int status = 0; + while (waitpid(pid, &status, 0) < 0) { + if (errno != EINTR) + return -1; + } + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) + return -1; + + char **list = NULL; + size_t n = 0; + size_t cap = 0; + buf[off] = '\0'; + char *line = buf; + char *p = buf; + while (p < buf + off) { + if (*p == '\n') { + parse_scutil_line(line, (size_t) (p - line), &list, &n, &cap); + line = p + 1; + } + p++; + } + if (line < buf + off) + parse_scutil_line(line, (size_t) ((buf + off) - line), &list, &n, &cap); + + *out_list = list; + *out_n = n; + return 0; +} + +static int write_resolv_conf(const char *run_dir, const char **err) +{ + int fd = open_etc_overwrite(run_dir, "resolv.conf", err); + if (fd < 0) + return -1; + + char **ns_list = NULL; + size_t ns_n = 0; + bool used_fallback = false; + if (scutil_collect_nameservers(&ns_list, &ns_n) < 0 || ns_n == 0) + used_fallback = true; + + int rc = 0; + if (used_fallback) { + static const char fallback[] = + "nameserver 8.8.8.8\n" + "nameserver 1.1.1.1\n"; + if (write_full(fd, fallback, sizeof(fallback) - 1) < 0) + rc = -1; + } else { + for (size_t i = 0; i < ns_n; i++) { + char line[128]; + int n = snprintf(line, sizeof(line), "nameserver %s\n", ns_list[i]); + if (n < 0 || (size_t) n >= sizeof(line)) { + /* Skip oversized entries silently; scutil should never + * print one but the loop must not abort on a bad + * resolver string. + */ + continue; + } + if (write_full(fd, line, (size_t) n) < 0) { + rc = -1; + break; + } + } + } + + int saved = errno; + for (size_t i = 0; i < ns_n; i++) + free(ns_list[i]); + free(ns_list); + + if (rc < 0) { + set_err_fmt(err, "runtime-files: write resolv.conf failed: %s", + strerror(saved)); + close(fd); + errno = saved; + return -1; + } + if (close(fd) < 0) { + int e = errno; + set_err_fmt(err, "runtime-files: close resolv.conf failed: %s", + strerror(e)); + errno = e; + return -1; + } + return 0; +} + +static int write_hosts(const char *run_dir, const char **err) +{ + static const char body[] = + "127.0.0.1 localhost\n" + "::1 localhost ip6-localhost " + "ip6-loopback\n" + "ff02::1 ip6-allnodes\n" + "ff02::2 ip6-allrouters\n" + "127.0.0.1 host.elfuse.internal\n"; + int fd = open_etc_overwrite(run_dir, "hosts", err); + if (fd < 0) + return -1; + if (write_full(fd, body, sizeof(body) - 1) < 0) { + int saved = errno; + set_err_fmt(err, "runtime-files: write hosts failed: %s", + strerror(saved)); + close(fd); + errno = saved; + return -1; + } + if (close(fd) < 0) { + int e = errno; + set_err_fmt(err, "runtime-files: close hosts failed: %s", strerror(e)); + errno = e; + return -1; + } + return 0; +} + +static int write_hostname(const char *run_dir, const char **err) +{ + static const char body[] = "elfuse\n"; + int fd = open_etc_overwrite(run_dir, "hostname", err); + if (fd < 0) + return -1; + if (write_full(fd, body, sizeof(body) - 1) < 0) { + int saved = errno; + set_err_fmt(err, "runtime-files: write hostname failed: %s", + strerror(saved)); + close(fd); + errno = saved; + return -1; + } + if (close(fd) < 0) { + int e = errno; + set_err_fmt(err, "runtime-files: close hostname failed: %s", + strerror(e)); + errno = e; + return -1; + } + return 0; +} + +int oci_runtime_files_inject(const char *run_dir, const char **err) +{ + if (err) + *err = NULL; + if (!run_dir || !*run_dir) { + set_err_static(err, "runtime-files: NULL/empty run_dir"); + errno = EINVAL; + return -1; + } + if (ensure_etc_dir(run_dir, err) < 0) + return -1; + if (write_resolv_conf(run_dir, err) < 0) + return -1; + if (write_hosts(run_dir, err) < 0) + return -1; + if (write_hostname(run_dir, err) < 0) + return -1; + return 0; +} diff --git a/src/oci/runtime-files.h b/src/oci/runtime-files.h new file mode 100644 index 0000000..23c707c --- /dev/null +++ b/src/oci/runtime-files.h @@ -0,0 +1,46 @@ +/* OCI per-run /etc host-truth injection + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Phase 4 F4.2 (/etc/resolv.conf) and F4.3 (/etc/hosts, /etc/hostname) + * ask the runtime to overlay three files on the per-run clone-rootfs + * so guest libc lookups (getaddrinfo, gethostname, /etc/hosts walks) + * see values matching the macOS host rather than the image's + * containerd defaults. The entry point is invoked from src/oci/run.c + * after oci_clone_rootfs and before the manifest parse. + */ + +#ifndef ELFUSE_OCI_RUNTIME_FILES_H +#define ELFUSE_OCI_RUNTIME_FILES_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* Write /etc/{resolv.conf,hosts,hostname} into /etc/. + * + * Creates the /etc/ directory at mode 0755 if it is missing. + * Overwrites any pre-existing file or symlink at the three target + * paths (image distros often ship /etc/resolv.conf as a symlink to + * /run/systemd/resolve/stub-resolv.conf, which would otherwise + * dangle inside the guest). + * + * /etc/resolv.conf is built from "nameserver " entries reported + * by scutil --dns; on any scutil failure or zero hits the helper + * falls back to 8.8.8.8 / 1.1.1.1. /etc/hosts is a fixed five-line + * block with localhost, ip6-loopback aliases, link-local multicast + * names, and host.elfuse.internal. /etc/hostname is the literal + * string "elfuse\n". + * + * Returns 0 on success, -1 on the first irrecoverable error with + * *err pointing at a static diagnostic. errno is preserved on the + * failing syscall. + */ +int oci_runtime_files_inject(const char *run_dir, const char **err); + +#ifdef __cplusplus +} +#endif + +#endif /* ELFUSE_OCI_RUNTIME_FILES_H */ diff --git a/src/oci/status.c b/src/oci/status.c new file mode 100644 index 0000000..3c8ae46 --- /dev/null +++ b/src/oci/status.c @@ -0,0 +1,807 @@ +/* Store-wide OCI status report + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Single walker over both store sources (pins via index.json + optional + * unpacked sysroots) plus three disk sweeps (blobs/, layers/, layers/stacks/). + * The pin walker resolves each manifest down to its image-config diff_ids, + * accumulates reachable diff_id and ChainID prefix sets into shared + * oci_digest_set_t accumulators, and records per-pin status / sizes / mtime + * for the CLI render. Unpacked sysroots feed the same accumulators from + * .elfuse-origin.json so no blob read is needed. + * + * After the walk, every diff_id in the reachable set is probed against + * /layers/// to compute the raw cache populate ratio; the + * same probe runs against /layers/stacks/// for the ChainID + * accumulator. The store sweeps under blobs//, layers//, and + * layers/stacks// count every entry (regardless of reachability) so the + * STORE TOTALS section can report the disk footprint. + * + * Failure policy: any per-entry error (missing manifest blob, malformed + * origin sidecar, bad image-config JSON) is recorded as the entry's status + * code rather than aborting. Fatal cases are reserved for the few states + * where no snapshot is possible: a NULL store, a failure walking + * /blobs/, or any other IO error during the directory sweeps. + * + * Duplication note: slurp_blob / sum_tree_size / resolve_config_digest / + * load_diff_ids / accumulate_chain are pattern-duplicates of helpers in + * src/oci/dedup-metrics.c and src/oci/store.c. The Plan 3 memory documents + * the lift threshold (deferred until a fourth caller of the diff_id walker + * pattern appears); status.c is the third caller, still below the cutoff. + */ + +#include "status.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "blob-store.h" +#include "digest-set.h" +#include "digest.h" +#include "manifest.h" +#include "origin-meta.h" +#include "volume.h" + +#define STATUS_PATH_MAX 4096 + +/* Largest blob this helper will read into a heap buffer. Mirrors the + * 64 MiB cap used by dedup-metrics.c so manifest / image-config parse + * failure modes stay uniform across the OCI module set. + */ +#define STATUS_BLOB_MAX ((size_t) 64 * 1024 * 1024) + +/* ── Helpers duplicated from dedup-metrics.c / store.c ───────────────── */ + +/* Slurp a blob into a fresh heap buffer, NUL-terminated for parser ergonomics. + * Returns 0 on success and writes the body + length; -1 with errno preserved + * on failure. Caller frees *out_body. + */ +static int slurp_blob(oci_blob_store_t *blobs, + const char *digest_str, + char **out_body, + size_t *out_len) +{ + oci_digest_algo_t algo; + char hex[OCI_DIGEST_HEX_MAX + 1]; + if (!oci_digest_parse(digest_str, &algo, hex)) { + errno = EINVAL; + return -1; + } + char path[STATUS_PATH_MAX]; + int n = oci_blob_store_path(blobs, algo, hex, path, sizeof(path)); + if (n < 0 || (size_t) n >= sizeof(path)) { + errno = ENAMETOOLONG; + return -1; + } + int fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + return -1; + struct stat st; + if (fstat(fd, &st) < 0) { + int saved = errno; + close(fd); + errno = saved; + return -1; + } + if (st.st_size < 0 || (uintmax_t) st.st_size > STATUS_BLOB_MAX) { + close(fd); + errno = EFBIG; + return -1; + } + size_t want = (size_t) st.st_size; + char *buf = malloc(want + 1); + if (!buf) { + close(fd); + errno = ENOMEM; + return -1; + } + size_t off = 0; + while (off < want) { + ssize_t r = read(fd, buf + off, want - off); + if (r < 0) { + if (errno == EINTR) + continue; + int saved = errno; + free(buf); + close(fd); + errno = saved; + return -1; + } + if (r == 0) + break; + off += (size_t) r; + } + close(fd); + if (off != want) { + free(buf); + errno = EIO; + return -1; + } + buf[want] = '\0'; + *out_body = buf; + *out_len = want; + return 0; +} + +/* Stat manifest blob to capture size + mtime for the pin row. Returns 0 on + * hit; -1 with errno=ENOENT on miss (caller treats as MISSING_MANIFEST) or + * with errno preserved on other failures. + */ +static int stat_blob(oci_blob_store_t *blobs, + const char *digest_str, + uint64_t *out_size, + int64_t *out_mtime) +{ + oci_digest_algo_t algo; + char hex[OCI_DIGEST_HEX_MAX + 1]; + if (!oci_digest_parse(digest_str, &algo, hex)) { + errno = EINVAL; + return -1; + } + char path[STATUS_PATH_MAX]; + int n = oci_blob_store_path(blobs, algo, hex, path, sizeof(path)); + if (n < 0 || (size_t) n >= sizeof(path)) { + errno = ENAMETOOLONG; + return -1; + } + struct stat st; + if (stat(path, &st) < 0) + return -1; + if (out_size) + *out_size = (uint64_t) (st.st_size < 0 ? 0 : st.st_size); + if (out_mtime) + *out_mtime = (int64_t) st.st_mtime; + return 0; +} + +/* Recursive st_size sum over a path tree. Returns the accumulated total on + * success or 0 when the entry is absent / unreadable (a missing layer cache + * entry simply contributes zero bytes). Symlinks are skipped via lstat so a + * stray symlink can never inflate the count. + */ +static uint64_t sum_tree_size(const char *path) +{ + struct stat st; + if (lstat(path, &st) < 0) + return 0; + if (S_ISREG(st.st_mode)) + return (uint64_t) st.st_size; + if (!S_ISDIR(st.st_mode)) + return 0; + DIR *d = opendir(path); + if (!d) + return 0; + uint64_t total = 0; + struct dirent *de; + while ((de = readdir(d)) != NULL) { + if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) + continue; + char child[STATUS_PATH_MAX]; + int n = snprintf(child, sizeof(child), "%s/%s", path, de->d_name); + if (n < 0 || (size_t) n >= sizeof(child)) + continue; + total += sum_tree_size(child); + } + closedir(d); + return total; +} + +/* Free a NULL-terminated char ** array (strdup'd entries plus the array). */ +static void free_strv(char **v) +{ + if (!v) + return; + for (size_t i = 0; v[i]; i++) + free(v[i]); + free((void *) v); +} + +/* Resolve a manifest blob to its image-config digest. Walks one level of + * image-index indirection (linux/arm64 pick). Returns a heap-allocated + * ":" config digest on success (caller frees) or NULL with errno + * set on failure: + * ENOENT - manifest blob missing OR index has no linux/arm64 entry + * EINVAL - manifest blob neither parseable as manifest nor index + * other - propagated from slurp_blob + * + * out_status_code, when non-NULL, receives a hint matching the pin status + * enum so the caller can map ENOENT-from-missing-sub-manifest to + * MISSING_MANIFEST (rather than INDEX_NO_ARM64) and so on. + */ +static char *resolve_config_digest(oci_store_t *store, + const char *manifest_digest, + oci_status_pin_code_t *out_status_code) +{ + oci_blob_store_t *blobs = oci_store_blobs(store); + char *body = NULL; + size_t body_len = 0; + if (slurp_blob(blobs, manifest_digest, &body, &body_len) < 0) { + if (out_status_code) + *out_status_code = (errno == ENOENT) + ? OCI_STATUS_PIN_MISSING_MANIFEST + : OCI_STATUS_PIN_CORRUPT_MANIFEST; + return NULL; + } + + oci_manifest_t mf = {0}; + if (oci_manifest_parse(body, body_len, &mf, NULL) == 0) { + char *cfg = strdup(mf.config.digest_str); + oci_manifest_free(&mf); + free(body); + if (!cfg) { + errno = ENOMEM; + if (out_status_code) + *out_status_code = OCI_STATUS_PIN_CORRUPT_CONFIG; + return NULL; + } + return cfg; + } + + oci_index_t idx = {0}; + if (oci_index_parse(body, body_len, &idx, NULL) < 0) { + free(body); + errno = EINVAL; + if (out_status_code) + *out_status_code = OCI_STATUS_PIN_CORRUPT_MANIFEST; + return NULL; + } + free(body); + + const oci_index_entry_t *picked = oci_index_pick_linux_arm64(&idx); + if (!picked) { + oci_index_free(&idx); + errno = ENOENT; + if (out_status_code) + *out_status_code = OCI_STATUS_PIN_INDEX_NO_ARM64; + return NULL; + } + char *sub_digest = strdup(picked->desc.digest_str); + oci_index_free(&idx); + if (!sub_digest) { + errno = ENOMEM; + if (out_status_code) + *out_status_code = OCI_STATUS_PIN_CORRUPT_CONFIG; + return NULL; + } + /* Recurse on the picked sub-manifest. Status code on a sub-manifest miss + * is MISSING_MANIFEST rather than INDEX_NO_ARM64: the index itself was + * fine, the sub-manifest blob just is not on disk. + */ + oci_status_pin_code_t sub_code = OCI_STATUS_PIN_OK; + char *cfg = resolve_config_digest(store, sub_digest, &sub_code); + free(sub_digest); + if (!cfg && out_status_code) + *out_status_code = sub_code; + return cfg; +} + +/* Load the rootfs.diff_ids array from an image-config blob. Returns a fresh + * NULL-terminated char ** on success (free via free_strv); NULL with errno + * set on missing / unparseable / OOM. The caller maps the failure to the + * CORRUPT_CONFIG pin status. + */ +static char **load_diff_ids(oci_store_t *store, const char *config_digest) +{ + oci_blob_store_t *blobs = oci_store_blobs(store); + char *body = NULL; + size_t body_len = 0; + if (slurp_blob(blobs, config_digest, &body, &body_len) < 0) + return NULL; + + oci_image_config_t cfg = {0}; + if (oci_image_config_parse(body, body_len, &cfg, NULL) < 0) { + free(body); + errno = EINVAL; + return NULL; + } + free(body); + + size_t n = 0; + while (cfg.rootfs_diff_ids[n]) + n++; + char **copy = (char **) calloc(n + 1, sizeof(*copy)); + if (!copy) { + oci_image_config_free(&cfg); + errno = ENOMEM; + return NULL; + } + for (size_t i = 0; i < n; i++) { + copy[i] = strdup(cfg.rootfs_diff_ids[i]); + if (!copy[i]) { + free_strv(copy); + oci_image_config_free(&cfg); + errno = ENOMEM; + return NULL; + } + } + oci_image_config_free(&cfg); + return copy; +} + +/* Add every diff_id and the per-layer ChainID prefix into the accumulators. + * Returns 0 on success, -1 with errno set on chainid or set-add failure. + * Mid-walk failures leave partial entries in the sets; status.c treats the + * caller as "this image's contribution is corrupt" and lets the partial + * entries stand because they cannot inflate the populated-count beyond + * reality (the union shape of digest_set means later images naturally dedup). + */ +static int accumulate_chain(char *const *diff_ids, + oci_digest_set_t *diff_acc, + oci_digest_set_t *chain_acc) +{ + char prev[OCI_DIGEST_HEX_MAX + 16] = ""; + for (size_t i = 0; diff_ids[i]; i++) { + if (oci_digest_set_add(diff_acc, diff_ids[i]) < 0) + return -1; + char chain[OCI_DIGEST_HEX_MAX + 16]; + const char *prev_arg = (i == 0) ? NULL : prev; + if (oci_chainid_compute(prev_arg, diff_ids[i], chain, sizeof(chain)) < + 0) + return -1; + memcpy(prev, chain, strlen(chain) + 1); + if (oci_digest_set_add(chain_acc, chain) < 0) + return -1; + } + return 0; +} + +/* ── Disk sweeps for STORE TOTALS ─────────────────────────────────────── */ + +/* Names whose ascii-hex shape matches the supplied algorithm. Mirrors the + * filter store.c uses to reject foreign state under blobs//. + */ +static bool entry_name_is_digest_hex(oci_digest_algo_t algo, const char *name) +{ + if (!name) + return false; + size_t want = oci_digest_hex_len(algo); + if (strlen(name) != want) + return false; + return oci_digest_hex_valid(algo, name); +} + +/* Sweep /blobs// and increment count + bytes for every well-formed + * blob entry. Returns 0 on success; -1 with errno set on opendir failure + * (except ENOENT, which is the empty-store case). When opts->skip_disk_usage + * is true the byte total is left at zero but the count still accumulates. + */ +static int sweep_blob_algo(const char *root, + oci_digest_algo_t algo, + bool skip_bytes, + size_t *count, + uint64_t *bytes) +{ + const char *algo_name = oci_digest_algo_name(algo); + if (!algo_name) + return 0; + char dir[STATUS_PATH_MAX]; + int n = snprintf(dir, sizeof(dir), "%s/blobs/%s", root, algo_name); + if (n < 0 || (size_t) n >= sizeof(dir)) { + errno = ENAMETOOLONG; + return -1; + } + DIR *d = opendir(dir); + if (!d) { + if (errno == ENOENT) + return 0; + return -1; + } + struct dirent *de; + while ((de = readdir(d)) != NULL) { + if (de->d_name[0] == '.') + continue; + if (!entry_name_is_digest_hex(algo, de->d_name)) + continue; + char child[STATUS_PATH_MAX]; + int cn = snprintf(child, sizeof(child), "%s/%s", dir, de->d_name); + if (cn < 0 || (size_t) cn >= sizeof(child)) + continue; + struct stat st; + if (lstat(child, &st) < 0) + continue; + if (!S_ISREG(st.st_mode)) + continue; + (*count)++; + if (!skip_bytes) + *bytes += (uint64_t) (st.st_size < 0 ? 0 : st.st_size); + } + closedir(d); + return 0; +} + +/* Sweep a content-addressed directory family rooted at /// + * where each well-formed child is itself a directory whose recursive + * st_size sum contributes to bytes. Used for layers// and + * layers/stacks//. Missing /// is the empty case + * (count == 0), not an error. + */ +static int sweep_tree_family(const char *root, + const char *base, + oci_digest_algo_t algo, + bool skip_bytes, + size_t *count, + uint64_t *bytes) +{ + const char *algo_name = oci_digest_algo_name(algo); + if (!algo_name) + return 0; + char dir[STATUS_PATH_MAX]; + int n = snprintf(dir, sizeof(dir), "%s/%s/%s", root, base, algo_name); + if (n < 0 || (size_t) n >= sizeof(dir)) { + errno = ENAMETOOLONG; + return -1; + } + DIR *d = opendir(dir); + if (!d) { + if (errno == ENOENT) + return 0; + return -1; + } + struct dirent *de; + while ((de = readdir(d)) != NULL) { + if (de->d_name[0] == '.') + continue; + if (!entry_name_is_digest_hex(algo, de->d_name)) + continue; + char child[STATUS_PATH_MAX]; + int cn = snprintf(child, sizeof(child), "%s/%s", dir, de->d_name); + if (cn < 0 || (size_t) cn >= sizeof(child)) + continue; + struct stat st; + if (lstat(child, &st) < 0) + continue; + if (!S_ISDIR(st.st_mode)) + continue; + (*count)++; + if (!skip_bytes) + *bytes += sum_tree_size(child); + } + closedir(d); + return 0; +} + +/* Algorithms swept by the disk pass. Mirrors PRUNE_ALGOS in store.c so the + * sweep stays consistent with what the GC walker considers a candidate. + */ +static const oci_digest_algo_t STATUS_ALGOS[] = { + OCI_DIGEST_SHA256, + OCI_DIGEST_SHA512, +}; +#define STATUS_ALGOS_LEN (sizeof(STATUS_ALGOS) / sizeof(STATUS_ALGOS[0])) + +/* Probe whether //// exists as a directory. Used to + * compute the populate ratios over the reachable digest sets. + */ +static bool tree_entry_exists(const char *root, + const char *base, + const char *digest_str) +{ + oci_digest_algo_t algo; + char hex[OCI_DIGEST_HEX_MAX + 1]; + if (!oci_digest_parse(digest_str, &algo, hex)) + return false; + const char *algo_name = oci_digest_algo_name(algo); + if (!algo_name) + return false; + char path[STATUS_PATH_MAX]; + int n = + snprintf(path, sizeof(path), "%s/%s/%s/%s", root, base, algo_name, hex); + if (n < 0 || (size_t) n >= sizeof(path)) + return false; + struct stat st; + if (stat(path, &st) < 0) + return false; + return S_ISDIR(st.st_mode); +} + +/* ── Per-pin walk ─────────────────────────────────────────────────────── */ + +/* Populate one pin entry. Returns 0 on every code path (pin entries record + * their own status; a per-pin failure is never fatal). diff_acc / chain_acc + * accumulate the reachable layer / chain sets when the pin resolves cleanly. + */ +static void walk_pin_entry(oci_store_t *store, + const oci_pin_entry_t *pin, + oci_status_pin_entry_t *out, + oci_digest_set_t *diff_acc, + oci_digest_set_t *chain_acc) +{ + memset(out, 0, sizeof(*out)); + out->last_seen_mtime = -1; + out->name = strdup(pin->name ? pin->name : ""); + out->digest = strdup(pin->digest ? pin->digest : ""); + if (!out->name || !out->digest) { + /* Allocation failure for the row identity strings is rare enough + * that surfacing it as MISSING_MANIFEST gives the operator a hint + * without complicating the failure model. + */ + out->status = OCI_STATUS_PIN_MISSING_MANIFEST; + return; + } + + oci_blob_store_t *blobs = oci_store_blobs(store); + + /* Manifest blob size / mtime first. A miss here short-circuits all later + * steps to MISSING_MANIFEST without polluting the per-pin row. + */ + if (stat_blob(blobs, out->digest, &out->manifest_size, + &out->last_seen_mtime) < 0) { + out->manifest_size = 0; + out->last_seen_mtime = -1; + out->status = OCI_STATUS_PIN_MISSING_MANIFEST; + return; + } + + oci_status_pin_code_t resolve_code = OCI_STATUS_PIN_OK; + char *config_digest = + resolve_config_digest(store, out->digest, &resolve_code); + if (!config_digest) { + out->status = resolve_code; + return; + } + + /* Capture image-config blob size when present. A missing config blob is + * the CORRUPT_CONFIG sentinel: the manifest was readable but the image + * is structurally incomplete. */ + uint64_t cfg_size = 0; + if (stat_blob(blobs, config_digest, &cfg_size, NULL) < 0) { + free(config_digest); + out->status = OCI_STATUS_PIN_CORRUPT_CONFIG; + return; + } + out->config_size = cfg_size; + + char **diff_ids = load_diff_ids(store, config_digest); + free(config_digest); + if (!diff_ids) { + out->status = OCI_STATUS_PIN_CORRUPT_CONFIG; + return; + } + size_t layer_count = 0; + while (diff_ids[layer_count]) + layer_count++; + out->layer_count = layer_count; + + /* Accumulate union sets so the populate ratios can run later. A failure + * inside accumulate_chain is treated as CORRUPT_CONFIG for the row but + * the partial entries stay in the accumulators: the union shape means + * a partial walk only ever inflates "reachable" honestly without + * over-reporting "populated". + */ + if (accumulate_chain(diff_ids, diff_acc, chain_acc) < 0) { + free_strv(diff_ids); + out->status = OCI_STATUS_PIN_CORRUPT_CONFIG; + return; + } + free_strv(diff_ids); + out->status = OCI_STATUS_PIN_OK; +} + +/* ── Per-unpacked-tree walk ───────────────────────────────────────────── */ + +static void walk_unpacked_entry(const char *tree_path, + bool skip_disk_usage, + oci_status_unpacked_entry_t *out, + oci_digest_set_t *diff_acc, + oci_digest_set_t *chain_acc) +{ + memset(out, 0, sizeof(*out)); + out->path = strdup(tree_path ? tree_path : ""); + if (!out->path) { + out->status = OCI_STATUS_UNPACKED_MISSING_ORIGIN; + return; + } + if (!skip_disk_usage) + out->tree_bytes = sum_tree_size(tree_path); + + oci_origin_t origin = {0}; + if (oci_origin_read(tree_path, &origin, NULL) < 0) { + out->status = (errno == ENOENT) ? OCI_STATUS_UNPACKED_MISSING_ORIGIN + : OCI_STATUS_UNPACKED_CORRUPT_ORIGIN; + return; + } + if (origin.manifest_digest) { + out->manifest_digest = strdup(origin.manifest_digest); + /* strdup failure for the digest is rare; surface as corrupt-origin + * so the row is still listed. + */ + if (!out->manifest_digest) { + oci_origin_free(&origin); + out->status = OCI_STATUS_UNPACKED_CORRUPT_ORIGIN; + return; + } + } + if (origin.layer_diffids) { + size_t n = 0; + while (origin.layer_diffids[n]) + n++; + out->layer_count = n; + if (n > 0 && + accumulate_chain(origin.layer_diffids, diff_acc, chain_acc) < 0) { + oci_origin_free(&origin); + out->status = OCI_STATUS_UNPACKED_CORRUPT_ORIGIN; + return; + } + } + oci_origin_free(&origin); + out->status = OCI_STATUS_UNPACKED_OK; +} + +/* ── Public entry point ───────────────────────────────────────────────── */ + +void oci_status_free(oci_status_t *out) +{ + if (!out) + return; + if (out->pins) { + for (size_t i = 0; i < out->pin_count; i++) { + free(out->pins[i].name); + free(out->pins[i].digest); + } + free(out->pins); + } + if (out->unpacked) { + for (size_t i = 0; i < out->unpacked_count; i++) { + free(out->unpacked[i].path); + free(out->unpacked[i].manifest_digest); + } + free(out->unpacked); + } + memset(out, 0, sizeof(*out)); +} + +int oci_status_compute(oci_store_t *store, + const oci_status_options_t *opts, + oci_status_t *out, + const char **err) +{ + static const char *dummy_err; + if (!err) + err = &dummy_err; + *err = NULL; + + if (!store || !out) { + *err = "status_compute: NULL argument"; + errno = EINVAL; + return -1; + } + memset(out, 0, sizeof(*out)); + + bool skip_du = opts && opts->skip_disk_usage; + out->disk_usage_skipped = skip_du; + const char *volume_root = opts ? opts->volume_root : NULL; + + oci_digest_set_t diff_acc = {0}; + oci_digest_set_t chain_acc = {0}; + + /* 1. Pin walk. An unreadable index.json (or a missing one) is treated + * as the empty case here: the rest of the snapshot still has value + * (store totals, layer caches) and surfacing the failure would be a + * regression vs the rest of the OCI CLI which all treat missing + * pins as empty. + */ + oci_pin_list_t pins = {0}; + const char *list_err = NULL; + if (oci_store_list_refs(store, &pins, &list_err) == 0 && pins.count > 0) { + out->pins = calloc(pins.count, sizeof(*out->pins)); + if (!out->pins) { + oci_pin_list_free(&pins); + oci_digest_set_free(&diff_acc); + oci_digest_set_free(&chain_acc); + *err = "status_compute: out of memory allocating pin rows"; + errno = ENOMEM; + return -1; + } + out->pin_count = pins.count; + for (size_t i = 0; i < pins.count; i++) { + walk_pin_entry(store, &pins.items[i], &out->pins[i], &diff_acc, + &chain_acc); + } + } + oci_pin_list_free(&pins); + + /* 2. Unpacked sysroots when volume_root is provided. */ + if (volume_root) { + oci_volume_list_t trees = {0}; + if (oci_volume_list_unpacked(volume_root, &trees, NULL) == 0 && + trees.count > 0) { + out->unpacked = calloc(trees.count, sizeof(*out->unpacked)); + if (!out->unpacked) { + oci_volume_list_free(&trees); + oci_digest_set_free(&diff_acc); + oci_digest_set_free(&chain_acc); + oci_status_free(out); + *err = "status_compute: out of memory allocating unpacked rows"; + errno = ENOMEM; + return -1; + } + out->unpacked_count = trees.count; + for (size_t i = 0; i < trees.count; i++) { + walk_unpacked_entry(trees.items[i], skip_du, &out->unpacked[i], + &diff_acc, &chain_acc); + } + } + oci_volume_list_free(&trees); + } + + /* 3. Store totals (counts always; bytes only when not skipped). */ + const char *root = oci_store_root(store); + if (!root) { + oci_digest_set_free(&diff_acc); + oci_digest_set_free(&chain_acc); + oci_status_free(out); + *err = "status_compute: store has no root path"; + errno = EINVAL; + return -1; + } + + for (size_t i = 0; i < STATUS_ALGOS_LEN; i++) { + if (sweep_blob_algo(root, STATUS_ALGOS[i], skip_du, &out->blob_count, + &out->blob_bytes_total) < 0) { + int saved = errno; + oci_digest_set_free(&diff_acc); + oci_digest_set_free(&chain_acc); + oci_status_free(out); + *err = "status_compute: blob sweep failed"; + errno = saved; + return -1; + } + } + for (size_t i = 0; i < STATUS_ALGOS_LEN; i++) { + if (sweep_tree_family(root, "layers", STATUS_ALGOS[i], skip_du, + &out->layer_cache_count, + &out->layer_cache_bytes_total) < 0) { + int saved = errno; + oci_digest_set_free(&diff_acc); + oci_digest_set_free(&chain_acc); + oci_status_free(out); + *err = "status_compute: layers/ sweep failed"; + errno = saved; + return -1; + } + } + for (size_t i = 0; i < STATUS_ALGOS_LEN; i++) { + if (sweep_tree_family(root, "layers/stacks", STATUS_ALGOS[i], skip_du, + &out->stack_cache_count, + &out->stack_cache_bytes_total) < 0) { + int saved = errno; + oci_digest_set_free(&diff_acc); + oci_digest_set_free(&chain_acc); + oci_status_free(out); + *err = "status_compute: layers/stacks/ sweep failed"; + errno = saved; + return -1; + } + } + + /* 4. Populate ratios. Iterate the reachable diff_id and ChainID sets + * and probe each against its respective cache family. Each entry is at + * most one stat(2) so the cost is O(R) where R = reachable count. + */ + out->diff_ids_reachable = oci_digest_set_size(&diff_acc); + for (size_t i = 0; i < out->diff_ids_reachable; i++) { + const char *d = oci_digest_set_at(&diff_acc, i); + if (d && tree_entry_exists(root, "layers", d)) + out->diff_ids_populated++; + } + out->chain_ids_reachable = oci_digest_set_size(&chain_acc); + for (size_t i = 0; i < out->chain_ids_reachable; i++) { + const char *c = oci_digest_set_at(&chain_acc, i); + if (c && tree_entry_exists(root, "layers/stacks", c)) + out->chain_ids_populated++; + } + + oci_digest_set_free(&diff_acc); + oci_digest_set_free(&chain_acc); + (void) list_err; /* swallowed: missing index.json is the empty case */ + return 0; +} diff --git a/src/oci/status.h b/src/oci/status.h new file mode 100644 index 0000000..39d113b --- /dev/null +++ b/src/oci/status.h @@ -0,0 +1,160 @@ +/* Store-wide OCI status report + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Aggregates a single point-in-time snapshot of every pin in index.json plus + * every unpacked sysroot under volume_root, into a flat struct the CLI can + * render or serialise to JSON. The walker is informational rather than a GC + * keep-set: a malformed manifest or unreadable origin sidecar surfaces as a + * per-entry status code without aborting the rest of the walk, so an operator + * with one corrupt pin still sees the healthy rows. + * + * Layer-cache populate ratios (raw + stack) drill into reachable diff_ids and + * ChainID prefixes the same way oci_dedup_metrics_compute does, except the + * aggregation is store-wide rather than per-target. The two values are + * separate metrics (raw vs stack) because the Plan 3 caches dedup along + * different axes; collapsing them into one number would hide which family + * needs operator attention. + * + * Plan 4 C4.1 first consumer. Reusable bits to revisit if Plan 4 C4.2 or any + * later store-wide reporter lands a fourth caller: the diff_id / ChainID + * walker duplication this file inherits from dedup-metrics.c and store.c + * crosses the lift threshold once a fourth caller appears; see + * project_oci_plan3 memory for the pattern. + */ + +#pragma once + +#include +#include +#include + +#include "store.h" + +/* Per-pin entry status. ok == 0 means every field below is populated; + * non-zero codes name which step failed so the renderer can print a sentinel + * row instead of suppressing the pin entirely. + */ +typedef enum { + OCI_STATUS_PIN_OK = 0, + OCI_STATUS_PIN_MISSING_MANIFEST = 1, /* manifest blob not on disk */ + OCI_STATUS_PIN_CORRUPT_MANIFEST = + 2, /* blob unparseable as manifest or index */ + OCI_STATUS_PIN_CORRUPT_CONFIG = + 3, /* image-config blob missing or unparseable */ + OCI_STATUS_PIN_INDEX_NO_ARM64 = + 4, /* image-index has no linux/arm64 entry */ +} oci_status_pin_code_t; + +typedef struct { + char *name; /* canonical "/:" */ + char *digest; /* pinned manifest digest, ":" */ + uint64_t manifest_size; /* st_size of manifest blob; 0 on missing */ + uint64_t config_size; /* st_size of image-config blob; 0 on missing */ + size_t layer_count; /* rootfs.diff_ids length; 0 on resolve failure */ + int64_t last_seen_mtime; /* st_mtime of manifest blob (epoch sec); -1 if + missing */ + oci_status_pin_code_t status; +} oci_status_pin_entry_t; + +/* Per-unpacked-sysroot entry status. */ +typedef enum { + OCI_STATUS_UNPACKED_OK = 0, + OCI_STATUS_UNPACKED_MISSING_ORIGIN = 1, + OCI_STATUS_UNPACKED_CORRUPT_ORIGIN = 2, +} oci_status_unpacked_code_t; + +typedef struct { + char *path; /* absolute path to /images/sha256-/ */ + char *manifest_digest; /* origin.manifest_digest; NULL on read failure */ + size_t layer_count; /* origin.layer_diffids length; 0 on read failure */ + uint64_t tree_bytes; /* recursive st_size sum; 0 under skip_disk_usage */ + oci_status_unpacked_code_t status; +} oci_status_unpacked_entry_t; + +/* Inputs to oci_status_compute. */ +typedef struct { + /* When non-NULL, the unpacked-tree walker scans /images/. + * NULL skips that source entirely; pins-only mode still populates every + * blob and cache total. Missing or unreadable volume_root is the empty + * case (unpacked_count = 0), not an error. + */ + const char *volume_root; + + /* When true, every byte counter is left at zero and the rendered output + * notes that disk usage was skipped. This is the operator escape hatch + * for stores large enough that walking layers/sha256/ and + * layers/stacks/sha256/ trees would be too slow. + */ + bool skip_disk_usage; +} oci_status_options_t; + +/* The aggregated report. Owned by the caller; release via oci_status_free. + * Array fields are NULL with count == 0 when empty so the renderer can + * branch on count alone. + */ +typedef struct { + /* Pins: always populated from index.json (best-effort; an unreadable + * index.json is the same as an empty store, not a fatal error so the + * renderer can still print store totals). + */ + oci_status_pin_entry_t *pins; + size_t pin_count; + + /* Unpacked sysroots: empty when volume_root was NULL or the + * /images/ directory was missing. + */ + oci_status_unpacked_entry_t *unpacked; + size_t unpacked_count; + + /* Store-wide disk totals. Counts are always populated. Byte totals are + * zero when skip_disk_usage is true. + */ + size_t blob_count; + uint64_t blob_bytes_total; + size_t layer_cache_count; + uint64_t layer_cache_bytes_total; + size_t stack_cache_count; + uint64_t stack_cache_bytes_total; + + /* Reachable-set populate ratios. Numerator counts entries the union of + * reachable diff_ids / ChainIDs that are actually present on disk under + * /layers/// or /layers/stacks///. + * A diff_id or ChainID is reachable if some healthy pin or unpacked + * sysroot named it; corrupt pins contribute nothing here so the ratios + * are not skewed by unreadable manifests. + */ + size_t diff_ids_reachable; + size_t diff_ids_populated; + size_t chain_ids_reachable; + size_t chain_ids_populated; + + /* Mirrors options.skip_disk_usage so the renderer does not need the + * options struct in scope. + */ + bool disk_usage_skipped; +} oci_status_t; + +/* Compute a store-wide status snapshot. + * + * Failure model: fatal only on bad arguments or a store-open / index.json + * lock failure where no useful snapshot can be produced. Per-pin and + * per-tree failures surface as the entry's status code; the walker keeps + * going. + * + * On entry *out is reset; on success it is fully populated and the caller + * must release it via oci_status_free. On failure *out is left in the + * freed-empty state and *err (when non-NULL) points at a static description. + * + * opts may be NULL for "pin-only, include disk usage" defaults. + */ +int oci_status_compute(oci_store_t *store, + const oci_status_options_t *opts, + oci_status_t *out, + const char **err); + +/* Release every owned heap field in *out and zero the struct. Safe on a + * zero-initialised struct and on NULL. + */ +void oci_status_free(oci_status_t *out); diff --git a/src/oci/store.c b/src/oci/store.c new file mode 100644 index 0000000..0539e2f --- /dev/null +++ b/src/oci/store.c @@ -0,0 +1,3429 @@ +/* Local OCI image store: blobs + tag-to-digest pinning + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Pin discipline: + * + * - index.json is the only pin store. A pin is one descriptor in + * manifests[] keyed by org.opencontainers.image.ref.name. + * - Writers serialize via flock(/index.json.lock, LOCK_EX) and + * publish via tmp + rename. The lock file is independent of index.json + * itself so that rename(2) replacing the inode does not invalidate the + * advisory lock identity for concurrent writers. + * - Readers parse the file lock-free: rename is atomic on a POSIX + * filesystem and cJSON consumes the document in one shot. + * - Re-pinning the same canonical name replaces the existing manifests[] + * entry in place; pull-by-tag with a moved tag updates rather than + * accumulating duplicates. + * + * Blob store layout: + * + * - The blob layer below this module keeps its link(2) discipline because + * content-addressed blobs are immutable; tag pins use rename(2) because + * pulling alpine:3.20 today may resolve to a different digest tomorrow + * and overwriting the pin is the correct semantic. + * + * Image-layout marker: + * + * - /oci-layout advertises the directory as a standards-compliant + * OCI image-layout so skopeo, umoci, and crane can consume the store + * directly. Writing the marker is idempotent: it is only created when + * missing and existing markers are never rewritten so a third party + * that bumped the imageLayoutVersion is not stomped. + * + * Pre-C2.2 stores wrote pin files under refs/// + * instead of index.json. C2.3 migrates older stores on open by recursively + * scanning refs/ and rebuilding index.json under the same flock that + * oci_store_put_ref takes, so a concurrent first-open and first-put cannot + * double-write. refs/ is left in place for one release so a downgrade still + * finds the legacy data. Migration is suppressed when ELFUSE_OCI_NO_MIGRATE + * is set in the environment; in that mode an older store appears empty to + * oci_store_get_ref / oci_store_list_refs until the env var is cleared on a + * subsequent open. + */ + +#include "store.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../externals/cjson/cJSON.h" +#include "digest-set.h" +#include "digest.h" +#include "manifest.h" +#include "origin-meta.h" +#include "volume.h" + +/* Largest path the store materializes. Comfortably above PATH_MAX so snprintf + * truncation surfaces as ENAMETOOLONG instead of a silent corruption. + */ +#define STORE_PATH_MAX 4096 + +/* Conservative ceiling for a single manifest body. Real OCI manifests run + * a few KiB; index.json itself is bounded by O(pin count * descriptor size) + * and stays well under this. Anything larger is treated as a corrupted or + * hostile blob and rejected at parse time. + */ +#define MAX_MANIFEST_BYTES (4 * 1024 * 1024) + +/* OCI annotation key under which pin names live in manifests[] descriptors. */ +static const char ANNOT_REF_NAME[] = "org.opencontainers.image.ref.name"; + +/* OCI media types used when filling the manifests[] descriptor. The actual + * mediaType is read from the manifest blob when present; these constants + * are the fallbacks used when the blob omits the JSON field (an older + * Docker manifest, for instance). + */ +static const char MT_OCI_IMAGE_INDEX[] = + "application/vnd.oci.image.index.v1+json"; +static const char MT_OCI_IMAGE_MANIFEST[] = + "application/vnd.oci.image.manifest.v1+json"; + +struct oci_store { + char *root; + oci_blob_store_t *blobs; +}; + +/* OCI image-layout 1.0.0 marker payload. The spec wants a JSON object with + * exactly one field: imageLayoutVersion = "1.0.0". The trailing newline is + * conventional and matches what umoci / skopeo write. + */ +static const char OCI_LAYOUT_BODY[] = "{\"imageLayoutVersion\":\"1.0.0\"}\n"; + +/* Environment variable that disables C2.3 auto-migration of pre-index.json + * stores. When set to any non-empty value, oci_store_open leaves refs/ and + * the absent index.json alone so a downgrade test or recovery workflow can + * inspect the legacy layout without the daemon helpfully rewriting it. + */ +static const char NO_MIGRATE_ENV[] = "ELFUSE_OCI_NO_MIGRATE"; + +/* Walks /refs/ recursively, rebuilds /index.json with one + * descriptor per discovered pin file, and writes it via tmp + rename. The + * caller must already hold an LOCK_EX on /index.json.lock. Returns 0 + * on success (including the no-pins-found case), -1 with errno preserved on + * an unrecoverable IO error. Individual pins whose manifest blob is missing + * from blobs/ are skipped with a stderr warning so a single dangling pin + * does not block migration for the rest of the store. + */ +static int migrate_legacy_refs(struct oci_store *s); + +/* Plan 3 C3.3b: probe /layers/.schema and migrate v1 stores to v2. + * + * Behaviour matrix at oci_store_open time: + * + * - marker present + schemaVersion == 2: no-op. + * - marker present + other schemaVersion or unparseable JSON: fail with + * errno=EINVAL so a forward-incompatible store does not get silently + * repopulated under the wrong shape. + * - marker absent + ELFUSE_OCI_NO_MIGRATE set: no-op (inspection mode; + * analogous to the C2.3 refs/ -> index.json gate). + * - marker absent + /layers/sha256/ empty: write v2 marker. + * - marker absent + /layers/sha256/ populated: wipe every direct + * child entry under layers/sha256/ (C3.2 cumulative-by-diff_id entries + * are not v2-compatible; reachability is recomputable from manifests + * at unpack time) and then write the v2 marker. + * + * The wipe + write runs under flock(/index.json.lock, LOCK_EX) and + * re-stats the marker under hold so a concurrent opener does not double + * migrate. The wipe is scoped to /layers/sha256/ children only; + * blobs/, images/, tmp/, refs/, index.json, and layers/.staging/ are + * never touched. + */ +static int ensure_layer_schema_marker(const char *root); + +/* Idempotently write /oci-layout. Returns 0 on success or when the + * marker already exists, -1 on any unexpected IO failure. The write uses a + * pid + counter-suffixed tmp file plus link(2) so a concurrent opener never + * observes a partial JSON document. link(2) is preferred over rename(2) for + * the publish step so that two racing openers cannot replace an external + * tool's bumped marker with our own; EEXIST is the happy path. + */ +static unsigned long layout_seq(void) +{ + static unsigned long n = 0; + return __sync_add_and_fetch(&n, 1); +} + +/* Ensure /layers/sha256/, /layers/stacks/sha256/, and + * /layers/.staging/ exist on open. The Plan 3 layer caches depend on + * three subtrees: layers/sha256/ holds committed per-layer raw entries + * (C3.3c), layers/stacks/sha256/ holds committed ChainID-keyed assembled + * stack snapshots (C3.3c), and layers/.staging/ is the shared in-flight + * staging area for clonefile(2) writers in both families. The blob store + * already created itself (oci_blob_store_open mkdirs the root + * tree), so this helper only adds the layers/ subtree. mkdir EEXIST is + * benign so reopens are idempotent. + */ +static int ensure_layer_dirs(const char *root) +{ + static const char *const subdirs[] = { + "layers", "layers/sha256", + "layers/stacks", "layers/stacks/sha256", + "layers/.staging", + }; + for (size_t i = 0; i < sizeof(subdirs) / sizeof(subdirs[0]); i++) { + char path[STORE_PATH_MAX]; + int n = snprintf(path, sizeof(path), "%s/%s", root, subdirs[i]); + if (n < 0 || (size_t) n >= sizeof(path)) { + errno = ENAMETOOLONG; + return -1; + } + if (mkdir(path, 0755) < 0 && errno != EEXIST) + return -1; + } + return 0; +} + +static int ensure_oci_layout_marker(const char *root) +{ + char path[STORE_PATH_MAX]; + int n = snprintf(path, sizeof(path), "%s/oci-layout", root); + if (n < 0 || (size_t) n >= sizeof(path)) { + errno = ENAMETOOLONG; + return -1; + } + struct stat st; + if (stat(path, &st) == 0) { + if (!S_ISREG(st.st_mode)) { + errno = ENOTDIR; + return -1; + } + return 0; + } + if (errno != ENOENT) + return -1; + + char tmp[STORE_PATH_MAX]; + n = snprintf(tmp, sizeof(tmp), "%s.tmp-%d-%lu", path, (int) getpid(), + layout_seq()); + if (n < 0 || (size_t) n >= sizeof(tmp)) { + errno = ENAMETOOLONG; + return -1; + } + + int fd = open(tmp, O_WRONLY | O_CREAT | O_EXCL, 0644); + if (fd < 0) + return -1; + size_t body_len = sizeof(OCI_LAYOUT_BODY) - 1; + if (write(fd, OCI_LAYOUT_BODY, body_len) != (ssize_t) body_len) { + int saved = errno; + close(fd); + unlink(tmp); + errno = saved; + return -1; + } + if (fsync(fd) < 0) { + int saved = errno; + close(fd); + unlink(tmp); + errno = saved; + return -1; + } + if (close(fd) < 0) { + int saved = errno; + unlink(tmp); + errno = saved; + return -1; + } + if (link(tmp, path) < 0) { + int saved = errno; + unlink(tmp); + if (saved == EEXIST) + return 0; + errno = saved; + return -1; + } + unlink(tmp); + return 0; +} + +oci_store_t *oci_store_open(const char *root) +{ + if (!root || !*root) { + errno = EINVAL; + return NULL; + } + oci_blob_store_t *blobs = oci_blob_store_open(root); + if (!blobs) + return NULL; + + if (ensure_oci_layout_marker(root) < 0) { + int saved = errno; + oci_blob_store_close(blobs); + errno = saved; + return NULL; + } + if (ensure_layer_dirs(root) < 0) { + int saved = errno; + oci_blob_store_close(blobs); + errno = saved; + return NULL; + } + if (ensure_layer_schema_marker(root) < 0) { + int saved = errno; + oci_blob_store_close(blobs); + errno = saved; + return NULL; + } + + oci_store_t *s = calloc(1, sizeof(*s)); + if (!s) { + oci_blob_store_close(blobs); + errno = ENOMEM; + return NULL; + } + s->root = strdup(root); + if (!s->root) { + free(s); + oci_blob_store_close(blobs); + errno = ENOMEM; + return NULL; + } + s->blobs = blobs; + + /* C2.3 auto-migration: detect a pre-index.json store (refs/ tree without + * an index.json) and rebuild index.json under the same flock that + * oci_store_put_ref takes. Suppressed by ELFUSE_OCI_NO_MIGRATE so a + * downgrade test or recovery workflow can inspect the legacy layout + * without it being silently rewritten. + */ + const char *no_migrate = getenv(NO_MIGRATE_ENV); + if (!no_migrate || !*no_migrate) { + if (migrate_legacy_refs(s) < 0) { + int saved = errno; + oci_store_close(s); + errno = saved; + return NULL; + } + } + return s; +} + +void oci_store_close(oci_store_t *s) +{ + if (!s) + return; + oci_blob_store_close(s->blobs); + free(s->root); + free(s); +} + +const char *oci_store_root(const oci_store_t *s) +{ + return s ? s->root : NULL; +} + +oci_blob_store_t *oci_store_blobs(oci_store_t *s) +{ + return s ? s->blobs : NULL; +} + +char *oci_store_default_root(void) +{ + const char *xdg = getenv("XDG_DATA_HOME"); + if (xdg && *xdg) { + size_t n = strlen(xdg) + sizeof("/elfuse/store"); + char *r = malloc(n); + if (!r) { + errno = ENOMEM; + return NULL; + } + snprintf(r, n, "%s/elfuse/store", xdg); + return r; + } + const char *home = getenv("HOME"); + if (!home || !*home) { + errno = ENOENT; + return NULL; + } + static const char SUFFIX[] = "/Library/Application Support/elfuse/store"; + size_t n = strlen(home) + sizeof(SUFFIX); + char *r = malloc(n); + if (!r) { + errno = ENOMEM; + return NULL; + } + snprintf(r, n, "%s%s", home, SUFFIX); + return r; +} + +/* Resolve the on-disk path of a manifest blob keyed by ":". The + * digest string has already been validated by oci_digest_parse, so the hex + * length is bounded and snprintf cannot truncate within STORE_PATH_MAX. + */ +static int blob_path_for_digest(const oci_store_t *s, + const char *digest_str, + char *out, + size_t cap) +{ + oci_digest_algo_t algo; + char hex[OCI_DIGEST_HEX_MAX + 1]; + if (!oci_digest_parse(digest_str, &algo, hex)) { + errno = EINVAL; + return -1; + } + int n = oci_blob_store_path(s->blobs, algo, hex, out, cap); + if (n < 0 || (size_t) n >= cap) { + errno = ENAMETOOLONG; + return -1; + } + return 0; +} + +/* stat the manifest blob and return its size. The caller has already + * validated the digest shape; ENOENT here means the caller forgot to + * persist the blob before pinning it, which is a programmer error in the + * pull / fixture path rather than user input. + */ +static int blob_size(const oci_store_t *s, + const char *digest_str, + int64_t *out_size) +{ + char path[STORE_PATH_MAX]; + if (blob_path_for_digest(s, digest_str, path, sizeof(path)) < 0) + return -1; + struct stat st; + if (stat(path, &st) < 0) + return -1; + if (!S_ISREG(st.st_mode)) { + errno = EINVAL; + return -1; + } + *out_size = (int64_t) st.st_size; + return 0; +} + +/* Best-effort read of the manifest blob's mediaType. Returns a heap-allocated + * string on success. When the blob omits the JSON mediaType field (older + * Docker manifests), sniff the shape: a top-level manifests array means an + * image-index, a layers array means an image-manifest. Falls back to the + * OCI image-manifest media type when the JSON is unrecognized so the + * descriptor stays schema-valid. Returns NULL on IO or parse failure with + * errno preserved. + */ +static char *infer_manifest_media_type(const oci_store_t *s, + const char *digest_str) +{ + char path[STORE_PATH_MAX]; + if (blob_path_for_digest(s, digest_str, path, sizeof(path)) < 0) + return NULL; + + int fd = open(path, O_RDONLY); + if (fd < 0) + return NULL; + struct stat st; + if (fstat(fd, &st) < 0) { + int saved = errno; + close(fd); + errno = saved; + return NULL; + } + if (st.st_size <= 0 || st.st_size > (off_t) MAX_MANIFEST_BYTES) { + close(fd); + errno = EINVAL; + return NULL; + } + size_t len = (size_t) st.st_size; + char *body = malloc(len + 1); + if (!body) { + close(fd); + errno = ENOMEM; + return NULL; + } + size_t off = 0; + while (off < len) { + ssize_t got = read(fd, body + off, len - off); + if (got < 0) { + int saved = errno; + free(body); + close(fd); + errno = saved; + return NULL; + } + if (got == 0) + break; + off += (size_t) got; + } + close(fd); + body[off] = '\0'; + + cJSON *root = cJSON_Parse(body); + free(body); + if (!root) { + errno = EINVAL; + return NULL; + } + + const char *mt = NULL; + const cJSON *mt_field = cJSON_GetObjectItemCaseSensitive(root, "mediaType"); + if (cJSON_IsString(mt_field) && mt_field->valuestring) + mt = mt_field->valuestring; + + char *dup = NULL; + if (mt) { + dup = strdup(mt); + } else if (cJSON_IsArray( + cJSON_GetObjectItemCaseSensitive(root, "manifests"))) { + dup = strdup(MT_OCI_IMAGE_INDEX); + } else { + dup = strdup(MT_OCI_IMAGE_MANIFEST); + } + cJSON_Delete(root); + if (!dup) { + errno = ENOMEM; + return NULL; + } + return dup; +} + +/* Read /index.json as a parsed cJSON tree. Returns NULL with errno=ENOENT + * when the file is missing (the empty-store happy path), NULL with another + * errno on IO failure, or NULL with errno=EINVAL on a parse error. The caller + * owns the returned tree and must cJSON_Delete it. + */ +static cJSON *read_index_json(const char *root, const char **err_msg) +{ + char path[STORE_PATH_MAX]; + int n = snprintf(path, sizeof(path), "%s/index.json", root); + if (n < 0 || (size_t) n >= sizeof(path)) { + errno = ENAMETOOLONG; + if (err_msg) + *err_msg = "index.json path exceeds STORE_PATH_MAX"; + return NULL; + } + int fd = open(path, O_RDONLY); + if (fd < 0) { + if (err_msg && errno != ENOENT) + *err_msg = "failed to open index.json"; + return NULL; + } + struct stat st; + if (fstat(fd, &st) < 0) { + int saved = errno; + close(fd); + errno = saved; + if (err_msg) + *err_msg = "fstat on index.json failed"; + return NULL; + } + if (st.st_size < 0 || st.st_size > (off_t) MAX_MANIFEST_BYTES) { + close(fd); + errno = EINVAL; + if (err_msg) + *err_msg = "index.json is empty or implausibly large"; + return NULL; + } + size_t len = (size_t) st.st_size; + char *body = malloc(len + 1); + if (!body) { + close(fd); + errno = ENOMEM; + if (err_msg) + *err_msg = "out of memory reading index.json"; + return NULL; + } + size_t off = 0; + while (off < len) { + ssize_t got = read(fd, body + off, len - off); + if (got < 0) { + int saved = errno; + free(body); + close(fd); + errno = saved; + if (err_msg) + *err_msg = "read on index.json failed"; + return NULL; + } + if (got == 0) + break; + off += (size_t) got; + } + close(fd); + body[off] = '\0'; + + cJSON *root_json = cJSON_Parse(body); + free(body); + if (!root_json) { + errno = EINVAL; + if (err_msg) + *err_msg = "index.json is not valid JSON"; + return NULL; + } + return root_json; +} + +/* Build an empty OCI image-index skeleton. Returns NULL on alloc failure. */ +static cJSON *new_empty_index(void) +{ + cJSON *root = cJSON_CreateObject(); + if (!root) + return NULL; + if (!cJSON_AddNumberToObject(root, "schemaVersion", 2) || + !cJSON_AddStringToObject(root, "mediaType", MT_OCI_IMAGE_INDEX)) { + cJSON_Delete(root); + return NULL; + } + cJSON *manifests = cJSON_CreateArray(); + if (!manifests) { + cJSON_Delete(root); + return NULL; + } + if (!cJSON_AddItemToObject(root, "manifests", manifests)) { + cJSON_Delete(manifests); + cJSON_Delete(root); + return NULL; + } + return root; +} + +/* Walk the manifests[] array, return the index of the descriptor whose + * annotations. equals name, or -1 if not found. + */ +static int find_manifest_index(const cJSON *manifests, const char *name) +{ + if (!cJSON_IsArray(manifests)) + return -1; + int n = cJSON_GetArraySize(manifests); + for (int i = 0; i < n; i++) { + const cJSON *entry = cJSON_GetArrayItem(manifests, i); + if (!cJSON_IsObject(entry)) + continue; + const cJSON *annots = + cJSON_GetObjectItemCaseSensitive(entry, "annotations"); + if (!cJSON_IsObject(annots)) + continue; + const cJSON *got = + cJSON_GetObjectItemCaseSensitive(annots, ANNOT_REF_NAME); + if (cJSON_IsString(got) && got->valuestring && + strcmp(got->valuestring, name) == 0) + return i; + } + return -1; +} + +/* Build a manifests[] descriptor object for (name, media_type, digest, size). + * Returns a newly-allocated cJSON node owned by the caller. NULL on alloc. + */ +static cJSON *build_descriptor(const char *name, + const char *media_type, + const char *digest_str, + int64_t size) +{ + cJSON *desc = cJSON_CreateObject(); + if (!desc) + return NULL; + if (!cJSON_AddStringToObject(desc, "mediaType", media_type) || + !cJSON_AddStringToObject(desc, "digest", digest_str) || + !cJSON_AddNumberToObject(desc, "size", (double) size)) + goto fail; + cJSON *annots = cJSON_CreateObject(); + if (!annots) + goto fail; + if (!cJSON_AddItemToObject(desc, "annotations", annots)) { + cJSON_Delete(annots); + goto fail; + } + if (!cJSON_AddStringToObject(annots, ANNOT_REF_NAME, name)) + goto fail; + return desc; + +fail: + cJSON_Delete(desc); + return NULL; +} + +static unsigned long pin_seq(void) +{ + static unsigned long n = 0; + return __sync_add_and_fetch(&n, 1); +} + +/* fsync the directory containing path so a rename(2) that publishes a new + * entry is durable across a crash: fsync on the file persists its contents + * but not the parent directory entry. Best-effort -- the tmp-file fsync is + * the primary guarantee and some filesystems reject a directory fsync, so a + * failure here must not fail the publish. + */ +static void fsync_parent_dir(const char *path) +{ + const char *slash = strrchr(path, '/'); + char dir[STORE_PATH_MAX]; + if (!slash) { + dir[0] = '.'; + dir[1] = '\0'; + } else if (slash == path) { + dir[0] = '/'; + dir[1] = '\0'; + } else { + size_t n = (size_t) (slash - path); + if (n >= sizeof(dir)) + return; + memcpy(dir, path, n); + dir[n] = '\0'; + } + int dfd = open(dir, O_RDONLY | O_DIRECTORY | O_CLOEXEC); + if (dfd < 0) + return; + (void) fsync(dfd); + (void) close(dfd); +} + +/* Serialize root_json to /index.json via tmp + rename. The publish is + * atomic with respect to readers: an open() either sees the previous inode + * or the new one, never a half-written file. fsync the tmp file before + * rename, and the parent directory after, so a crash leaves the pin update + * durable rather than reverting to the prior index.json. + */ +static int write_index_json(const char *root, + const cJSON *root_json, + const char **err_msg) +{ + char path[STORE_PATH_MAX]; + int n = snprintf(path, sizeof(path), "%s/index.json", root); + if (n < 0 || (size_t) n >= sizeof(path)) { + errno = ENAMETOOLONG; + if (err_msg) + *err_msg = "index.json path exceeds STORE_PATH_MAX"; + return -1; + } + char tmp[STORE_PATH_MAX]; + n = snprintf(tmp, sizeof(tmp), "%s.tmp-%d-%lu", path, (int) getpid(), + pin_seq()); + if (n < 0 || (size_t) n >= sizeof(tmp)) { + errno = ENAMETOOLONG; + if (err_msg) + *err_msg = "index.json tmp path exceeds STORE_PATH_MAX"; + return -1; + } + + char *body = cJSON_PrintUnformatted(root_json); + if (!body) { + errno = ENOMEM; + if (err_msg) + *err_msg = "failed to serialize index.json"; + return -1; + } + size_t body_len = strlen(body); + + int fd = open(tmp, O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (fd < 0) { + int saved = errno; + free(body); + errno = saved; + if (err_msg) + *err_msg = "failed to create index.json tmp file"; + return -1; + } + + /* Append a trailing newline so external tools that line-print the file + * (jq, cat) render cleanly. cJSON_PrintUnformatted does not include it. + */ + const char nl = '\n'; + if (write(fd, body, body_len) != (ssize_t) body_len || + write(fd, &nl, 1) != 1) { + int saved = errno; + close(fd); + unlink(tmp); + free(body); + errno = saved; + if (err_msg) + *err_msg = "failed to write index.json tmp file"; + return -1; + } + free(body); + if (fsync(fd) < 0) { + int saved = errno; + close(fd); + unlink(tmp); + errno = saved; + if (err_msg) + *err_msg = "fsync on index.json tmp file failed"; + return -1; + } + if (close(fd) < 0) { + int saved = errno; + unlink(tmp); + errno = saved; + if (err_msg) + *err_msg = "close on index.json tmp file failed"; + return -1; + } + if (rename(tmp, path) < 0) { + int saved = errno; + unlink(tmp); + errno = saved; + if (err_msg) + *err_msg = "rename of index.json tmp file failed"; + return -1; + } + /* Persist the directory entry the rename just swapped in so a crash does + * not silently roll the tag->digest pins back to the previous index.json. + */ + fsync_parent_dir(path); + return 0; +} + +/* Acquire LOCK_EX on /index.json.lock. The lock file is created when + * missing; failures to create it (full disk, permission) surface immediately + * so a writer never proceeds without coordination. Returns the lock fd on + * success; the caller must close() it to release the lock (POSIX advisory + * lock semantics tie lifetime to the fd). + */ +static int acquire_index_lock(const char *root, const char **err_msg) +{ + char path[STORE_PATH_MAX]; + int n = snprintf(path, sizeof(path), "%s/index.json.lock", root); + if (n < 0 || (size_t) n >= sizeof(path)) { + errno = ENAMETOOLONG; + if (err_msg) + *err_msg = "index.json.lock path exceeds STORE_PATH_MAX"; + return -1; + } + int fd = open(path, O_RDWR | O_CREAT | O_CLOEXEC, 0644); + if (fd < 0) { + if (err_msg) + *err_msg = "failed to open index.json.lock"; + return -1; + } + if (flock(fd, LOCK_EX) < 0) { + int saved = errno; + close(fd); + errno = saved; + if (err_msg) + *err_msg = "flock on index.json.lock failed"; + return -1; + } + return fd; +} + +int oci_store_put_ref(oci_store_t *s, + const oci_ref_t *ref, + const char *digest_str, + const char **err_msg) +{ + if (!s || !ref || !digest_str || !ref->registry || !ref->repository) { + if (err_msg) + *err_msg = "invalid arguments"; + errno = EINVAL; + return -1; + } + if (!ref->tag) { + if (err_msg) + *err_msg = "ref has no tag; digest-only refs are self-pinning"; + errno = EINVAL; + return -1; + } + + /* Validate digest shape so a corrupt caller cannot poison the pin + * descriptor with arbitrary bytes that later defeat oci_store_get_ref. + */ + oci_digest_algo_t algo; + char hex[OCI_DIGEST_HEX_MAX + 1]; + if (!oci_digest_parse(digest_str, &algo, hex)) { + if (err_msg) + *err_msg = "digest must be lowercase :"; + errno = EINVAL; + return -1; + } + + int64_t size = 0; + if (blob_size(s, digest_str, &size) < 0) { + if (err_msg) + *err_msg = "manifest blob is not present in the local store"; + return -1; + } + char *media_type = infer_manifest_media_type(s, digest_str); + if (!media_type) { + if (err_msg) + *err_msg = "failed to determine manifest mediaType from blob"; + return -1; + } + + char *name = oci_ref_canonical_name(ref); + if (!name) { + int saved = errno; + free(media_type); + errno = saved; + if (err_msg) + *err_msg = "failed to render canonical ref name"; + return -1; + } + + int rc = -1; + int lock_fd = acquire_index_lock(s->root, err_msg); + if (lock_fd < 0) + goto out_no_lock; + + const char *read_err = NULL; + cJSON *root_json = read_index_json(s->root, &read_err); + if (!root_json) { + if (errno != ENOENT) { + if (err_msg) + *err_msg = read_err ? read_err : "failed to read index.json"; + goto out; + } + root_json = new_empty_index(); + if (!root_json) { + errno = ENOMEM; + if (err_msg) + *err_msg = "out of memory building empty index.json"; + goto out; + } + } + + cJSON *manifests = cJSON_GetObjectItemCaseSensitive(root_json, "manifests"); + if (!cJSON_IsArray(manifests)) { + /* Corrupt or hand-edited index: rebuild the array so writes still + * make progress. The old contents are discarded. + */ + cJSON_DeleteItemFromObject(root_json, "manifests"); + manifests = cJSON_CreateArray(); + if (!manifests || + !cJSON_AddItemToObject(root_json, "manifests", manifests)) { + cJSON_Delete(manifests); + errno = ENOMEM; + if (err_msg) + *err_msg = "out of memory rebuilding manifests array"; + goto out; + } + } + + cJSON *desc = build_descriptor(name, media_type, digest_str, size); + if (!desc) { + errno = ENOMEM; + if (err_msg) + *err_msg = "out of memory building pin descriptor"; + goto out; + } + + int existing = find_manifest_index(manifests, name); + if (existing >= 0) { + /* Replace in place so concurrent re-pulls of the same tag do not + * accumulate duplicate descriptors. + */ + if (!cJSON_ReplaceItemInArray(manifests, existing, desc)) { + cJSON_Delete(desc); + errno = EIO; + if (err_msg) + *err_msg = "failed to replace existing pin descriptor"; + goto out; + } + } else if (!cJSON_AddItemToArray(manifests, desc)) { + cJSON_Delete(desc); + errno = ENOMEM; + if (err_msg) + *err_msg = "failed to append pin descriptor"; + goto out; + } + + if (write_index_json(s->root, root_json, err_msg) < 0) + goto out; + + rc = 0; + +out: + cJSON_Delete(root_json); + /* close releases the flock per POSIX advisory-lock semantics. */ + close(lock_fd); +out_no_lock: + free(name); + free(media_type); + return rc; +} + +int oci_store_get_ref(oci_store_t *s, + const oci_ref_t *ref, + char **out_digest, + const char **err_msg) +{ + if (!s || !ref || !out_digest || !ref->registry || !ref->repository) { + if (err_msg) + *err_msg = "invalid arguments"; + errno = EINVAL; + return -1; + } + *out_digest = NULL; + if (!ref->tag) { + if (err_msg) + *err_msg = "ref has no tag"; + errno = EINVAL; + return -1; + } + + char *name = oci_ref_canonical_name(ref); + if (!name) { + if (err_msg) + *err_msg = "failed to render canonical ref name"; + return -1; + } + + const char *read_err = NULL; + cJSON *root_json = read_index_json(s->root, &read_err); + if (!root_json) { + free(name); + if (errno == ENOENT && err_msg) + *err_msg = "ref not pinned in local store"; + else if (err_msg) + *err_msg = read_err ? read_err : "failed to read index.json"; + return -1; + } + + cJSON *manifests = cJSON_GetObjectItemCaseSensitive(root_json, "manifests"); + int idx = find_manifest_index(manifests, name); + free(name); + if (idx < 0) { + cJSON_Delete(root_json); + errno = ENOENT; + if (err_msg) + *err_msg = "ref not pinned in local store"; + return -1; + } + + const cJSON *entry = cJSON_GetArrayItem(manifests, idx); + const cJSON *digest_field = + cJSON_GetObjectItemCaseSensitive(entry, "digest"); + if (!cJSON_IsString(digest_field) || !digest_field->valuestring) { + cJSON_Delete(root_json); + errno = EINVAL; + if (err_msg) + *err_msg = "pin descriptor is missing digest field"; + return -1; + } + + /* Re-validate the digest shape so a hand-edited index.json cannot smuggle + * a malformed digest back to a caller that trusts the store output. + */ + oci_digest_algo_t algo; + char hex[OCI_DIGEST_HEX_MAX + 1]; + if (!oci_digest_parse(digest_field->valuestring, &algo, hex)) { + cJSON_Delete(root_json); + errno = EINVAL; + if (err_msg) + *err_msg = "pin descriptor digest has invalid shape"; + return -1; + } + + char *copy = strdup(digest_field->valuestring); + cJSON_Delete(root_json); + if (!copy) { + errno = ENOMEM; + if (err_msg) + *err_msg = "out of memory"; + return -1; + } + *out_digest = copy; + return 0; +} + +int oci_store_list_refs(oci_store_t *s, + oci_pin_list_t *out, + const char **err_msg) +{ + if (!s || !out) { + if (err_msg) + *err_msg = "invalid arguments"; + errno = EINVAL; + return -1; + } + out->items = NULL; + out->count = 0; + + const char *read_err = NULL; + cJSON *root_json = read_index_json(s->root, &read_err); + if (!root_json) { + if (errno == ENOENT) + return 0; + if (err_msg) + *err_msg = read_err ? read_err : "failed to read index.json"; + return -1; + } + + cJSON *manifests = cJSON_GetObjectItemCaseSensitive(root_json, "manifests"); + if (!cJSON_IsArray(manifests)) { + cJSON_Delete(root_json); + return 0; + } + int n = cJSON_GetArraySize(manifests); + if (n <= 0) { + cJSON_Delete(root_json); + return 0; + } + + oci_pin_entry_t *items = calloc((size_t) n, sizeof(*items)); + if (!items) { + cJSON_Delete(root_json); + errno = ENOMEM; + if (err_msg) + *err_msg = "out of memory allocating pin list"; + return -1; + } + size_t filled = 0; + for (int i = 0; i < n; i++) { + const cJSON *entry = cJSON_GetArrayItem(manifests, i); + if (!cJSON_IsObject(entry)) + continue; + const cJSON *annots = + cJSON_GetObjectItemCaseSensitive(entry, "annotations"); + const cJSON *name_field = + cJSON_IsObject(annots) + ? cJSON_GetObjectItemCaseSensitive(annots, ANNOT_REF_NAME) + : NULL; + const cJSON *digest_field = + cJSON_GetObjectItemCaseSensitive(entry, "digest"); + if (!cJSON_IsString(name_field) || !name_field->valuestring || + !cJSON_IsString(digest_field) || !digest_field->valuestring) { + /* Skip schema-incomplete entries: a third-party tool may have + * inserted a manifest without the ref-name annotation, in which + * case it is not a pin from elfuse's perspective. + */ + continue; + } + char *name_copy = strdup(name_field->valuestring); + char *digest_copy = strdup(digest_field->valuestring); + if (!name_copy || !digest_copy) { + free(name_copy); + free(digest_copy); + for (size_t k = 0; k < filled; k++) { + free(items[k].name); + free(items[k].digest); + } + free(items); + cJSON_Delete(root_json); + errno = ENOMEM; + if (err_msg) + *err_msg = "out of memory copying pin entry"; + return -1; + } + items[filled].name = name_copy; + items[filled].digest = digest_copy; + filled++; + } + cJSON_Delete(root_json); + + if (filled == 0) { + free(items); + return 0; + } + + out->items = items; + out->count = filled; + return 0; +} + +void oci_pin_list_free(oci_pin_list_t *list) +{ + if (!list) + return; + if (list->items) { + for (size_t i = 0; i < list->count; i++) { + free(list->items[i].name); + free(list->items[i].digest); + } + free(list->items); + } + list->items = NULL; + list->count = 0; +} + +/* Slurp the manifest-class blob at digest_str into a heap buffer. The + * caller frees *out_body. Mirrors the size and bounds checks of + * infer_manifest_media_type so a corrupt or hostile blob does not + * trigger a multi-GB malloc here. Returns 0 on success or -1 with + * errno preserved on failure. + */ +static int load_manifest_blob(const oci_store_t *s, + const char *digest_str, + char **out_body, + size_t *out_len) +{ + char path[STORE_PATH_MAX]; + if (blob_path_for_digest(s, digest_str, path, sizeof(path)) < 0) + return -1; + int fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + return -1; + struct stat st; + if (fstat(fd, &st) < 0) { + int saved = errno; + close(fd); + errno = saved; + return -1; + } + if (st.st_size <= 0 || st.st_size > (off_t) MAX_MANIFEST_BYTES) { + close(fd); + errno = EINVAL; + return -1; + } + size_t len = (size_t) st.st_size; + char *body = malloc(len + 1); + if (!body) { + close(fd); + errno = ENOMEM; + return -1; + } + size_t off = 0; + while (off < len) { + ssize_t got = read(fd, body + off, len - off); + if (got < 0) { + if (errno == EINTR) + continue; + int saved = errno; + free(body); + close(fd); + errno = saved; + return -1; + } + if (got == 0) + break; + off += (size_t) got; + } + close(fd); + if (off != len) { + free(body); + errno = EIO; + return -1; + } + body[len] = '\0'; + *out_body = body; + *out_len = len; + return 0; +} + +/* True when blobs// for digest_str exists on disk. Errors + * other than ENOENT (permission, ENAMETOOLONG) propagate as "missing" + * because the caller's failure path treats either as fatal for the + * keep-set walk; the distinction is academic. + */ +static bool manifest_blob_exists(const oci_store_t *s, const char *digest_str) +{ + oci_digest_algo_t algo; + char hex[OCI_DIGEST_HEX_MAX + 1]; + if (!oci_digest_parse(digest_str, &algo, hex)) + return false; + return oci_blob_store_has(s->blobs, algo, hex); +} + +/* Recursive expander: ensure digest_str is in out and, if its blob is + * a manifest or image-index, also add every descriptor it references. + * Recursion terminates because oci_digest_set_add is a no-op for any + * digest already in the set, so a cycle (theoretical: an image-index + * pointing at itself) is bounded. + * + * Returns 0 on success, -1 on fatal failure (missing or unparseable + * blob) with errno set and *err populated. + */ +static int expand_manifest_digest(oci_store_t *s, + const char *digest_str, + oci_digest_set_t *out, + const char **err) +{ + if (oci_digest_set_contains(out, digest_str)) + return 0; + if (oci_digest_set_add(out, digest_str) < 0) { + if (err) + *err = "collect_roots: digest_set_add failed"; + return -1; + } + + char *body = NULL; + size_t body_len = 0; + if (load_manifest_blob(s, digest_str, &body, &body_len) < 0) { + if (err) + *err = + "collect_roots: referenced manifest blob is missing or " + "unreadable"; + return -1; + } + + oci_manifest_t manifest = {0}; + const char *perr = NULL; + if (oci_manifest_parse(body, body_len, &manifest, &perr) == 0) { + int rc = 0; + if (oci_digest_set_add(out, manifest.config.digest_str) < 0) { + if (err) + *err = "collect_roots: digest_set_add for config failed"; + rc = -1; + goto manifest_done; + } + for (size_t i = 0; i < manifest.nlayers; i++) { + if (oci_digest_set_add(out, manifest.layers[i].digest_str) < 0) { + if (err) + *err = "collect_roots: digest_set_add for layer failed"; + rc = -1; + goto manifest_done; + } + } + manifest_done: + oci_manifest_free(&manifest); + free(body); + return rc; + } + memset(&manifest, 0, sizeof(manifest)); + + /* Not an image-manifest. Try image-index: a multi-arch index + * references one sub-manifest descriptor per platform. + */ + oci_index_t index = {0}; + const char *ierr = NULL; + if (oci_index_parse(body, body_len, &index, &ierr) < 0) { + free(body); + if (err) + *err = + "collect_roots: blob is neither image-manifest nor " + "image-index"; + errno = EINVAL; + return -1; + } + free(body); + + for (size_t i = 0; i < index.nentries; i++) { + const char *sub = index.entries[i].desc.digest_str; + /* Record the sub-manifest descriptor digest even when the + * blob is not on disk: a multi-arch index legitimately + * references blobs for other platforms that pull never + * fetched, and a sweep must not delete the platforms that + * did materialise. When the blob is present recurse so its + * config + layers join the keep set; when absent the index + * descriptor alone is enough because there is no blob to + * delete. + */ + if (manifest_blob_exists(s, sub)) { + if (expand_manifest_digest(s, sub, out, err) < 0) { + oci_index_free(&index); + return -1; + } + } else if (oci_digest_set_add(out, sub) < 0) { + if (err) + *err = "collect_roots: digest_set_add for sub-manifest failed"; + oci_index_free(&index); + return -1; + } + } + oci_index_free(&index); + return 0; +} + +int oci_store_collect_roots(oci_store_t *s, + oci_digest_set_t *out, + const char *volume_root, + const char **err) +{ + static const char *dummy_err; + if (!err) + err = &dummy_err; + *err = NULL; + if (!s || !out) { + *err = "collect_roots: NULL argument"; + errno = EINVAL; + return -1; + } + oci_digest_set_init(out); + + /* Source 1: pins in index.json. list_refs handles the empty case + * (no index.json yet) without surfacing an error, so a fresh + * store contributes zero entries from this source. + */ + oci_pin_list_t pins = {0}; + const char *list_err = NULL; + if (oci_store_list_refs(s, &pins, &list_err) < 0) { + if (err) + *err = list_err ? list_err + : "collect_roots: oci_store_list_refs failed"; + return -1; + } + for (size_t i = 0; i < pins.count; i++) { + if (expand_manifest_digest(s, pins.items[i].digest, out, err) < 0) { + oci_pin_list_free(&pins); + oci_digest_set_free(out); + return -1; + } + } + oci_pin_list_free(&pins); + + /* Source 2: unpacked image trees under /images/. + * A NULL volume_root skips this source entirely (callers that + * only need the pin contribution). A missing images/ directory + * is treated as zero contribution by oci_volume_list_unpacked. + */ + if (volume_root) { + oci_volume_list_t trees = {0}; + const char *vlerr = NULL; + if (oci_volume_list_unpacked(volume_root, &trees, &vlerr) < 0) { + if (err) + *err = vlerr ? vlerr + : "collect_roots: volume_list_unpacked failed"; + oci_digest_set_free(out); + return -1; + } + for (size_t i = 0; i < trees.count; i++) { + oci_origin_t origin = {0}; + const char *oerr = NULL; + if (oci_origin_read(trees.items[i], &origin, &oerr) < 0) { + if (err) + *err = oerr ? oerr + : "collect_roots: origin sidecar read failed"; + oci_volume_list_free(&trees); + oci_digest_set_free(out); + return -1; + } + if (expand_manifest_digest(s, origin.manifest_digest, out, err) < + 0) { + oci_origin_free(&origin); + oci_volume_list_free(&trees); + oci_digest_set_free(out); + return -1; + } + oci_origin_free(&origin); + } + oci_volume_list_free(&trees); + } + return 0; +} + +/* Read a legacy pin file at path. The format is a single line of + * ":" optionally followed by \n or \r\n. Trims trailing whitespace + * and validates digest shape. Returns a heap-allocated digest string on + * success, NULL on IO or schema failure with errno preserved. + */ +static char *read_legacy_pin_file(const char *path) +{ + int fd = open(path, O_RDONLY); + if (fd < 0) + return NULL; + char buf[OCI_DIGEST_HEX_MAX + 32]; + ssize_t got = read(fd, buf, sizeof(buf) - 1); + int saved_errno = errno; + close(fd); + if (got < 0) { + errno = saved_errno; + return NULL; + } + if (got == 0) { + errno = EINVAL; + return NULL; + } + buf[got] = '\0'; + /* Trim trailing newline / carriage return so a Windows-edited pin file + * does not feed a stray byte into the digest validator. + */ + while (got > 0 && (buf[got - 1] == '\n' || buf[got - 1] == '\r' || + buf[got - 1] == ' ' || buf[got - 1] == '\t')) { + buf[--got] = '\0'; + } + if (got == 0) { + errno = EINVAL; + return NULL; + } + oci_digest_algo_t algo; + char hex[OCI_DIGEST_HEX_MAX + 1]; + if (!oci_digest_parse(buf, &algo, hex)) { + errno = EINVAL; + return NULL; + } + char *copy = strdup(buf); + if (!copy) { + errno = ENOMEM; + return NULL; + } + return copy; +} + +/* Synthesize a pin descriptor for one legacy refs/ leaf and insert it into + * manifests[]. (name, digest_str) describe the pin; the manifest blob must + * exist on disk so mediaType + size can be derived. Returns 0 on success, + * +1 if the pin should be skipped (missing blob or other recoverable hole), + * -1 with errno preserved on an unrecoverable failure. + */ +static int migrate_append_descriptor(const oci_store_t *s, + cJSON *manifests, + const char *name, + const char *digest_str) +{ + int64_t size = 0; + if (blob_size(s, digest_str, &size) < 0) { + if (errno == ENOENT) { + fprintf(stderr, + "elfuse oci: migration skipping pin %s: manifest blob " + "%s missing from blobs/\n", + name, digest_str); + return 1; + } + return -1; + } + char *media_type = infer_manifest_media_type(s, digest_str); + if (!media_type) + return -1; + cJSON *desc = build_descriptor(name, media_type, digest_str, size); + free(media_type); + if (!desc) { + errno = ENOMEM; + return -1; + } + /* Pre-existing index entry with the same name should not happen during + * a fresh migration, but defend against a partially-migrated store + * inheriting from an earlier crash: a later refs/ leaf with the same + * canonical name simply replaces the earlier one. */ + int existing = find_manifest_index(manifests, name); + if (existing >= 0) { + if (!cJSON_ReplaceItemInArray(manifests, existing, desc)) { + cJSON_Delete(desc); + errno = EIO; + return -1; + } + } else if (!cJSON_AddItemToArray(manifests, desc)) { + cJSON_Delete(desc); + errno = ENOMEM; + return -1; + } + return 0; +} + +/* Recursive walk of refs/. depth counts directory levels below refs/: + * depth 0 = registry, depth 1.. = repository components, leaf file = tag. + * head and head_len accumulate the relative path so the leaf callback can + * split it into registry/repo/tag. *migrated and *skipped track totals for + * the final log line. Returns 0 on success, -1 on unrecoverable failure. + */ +static int scan_refs_dir(const oci_store_t *s, + cJSON *manifests, + const char *dir_abs, + const char *rel_head, + size_t rel_head_len, + size_t depth, + size_t *migrated, + size_t *skipped) +{ + DIR *dp = opendir(dir_abs); + if (!dp) + return -1; + + int rc = 0; + struct dirent *de; + while ((de = readdir(dp)) != NULL) { + const char *name = de->d_name; + if (name[0] == '.' && + (name[1] == '\0' || (name[1] == '.' && name[2] == '\0'))) + continue; + /* Hidden files (e.g. .DS_Store) are ignored: legacy pins always used + * an unprefixed registry / tag name so a dotfile cannot represent a + * pin and we skip it without surfacing as a migration warning. */ + if (name[0] == '.') + continue; + + size_t name_len = strlen(name); + char child_abs[STORE_PATH_MAX]; + int n = snprintf(child_abs, sizeof(child_abs), "%s/%s", dir_abs, name); + if (n < 0 || (size_t) n >= sizeof(child_abs)) { + errno = ENAMETOOLONG; + rc = -1; + break; + } + char child_rel[STORE_PATH_MAX]; + int rn = rel_head_len == 0 + ? snprintf(child_rel, sizeof(child_rel), "%s", name) + : snprintf(child_rel, sizeof(child_rel), "%s/%s", rel_head, + name); + if (rn < 0 || (size_t) rn >= sizeof(child_rel)) { + errno = ENAMETOOLONG; + rc = -1; + break; + } + + struct stat st; + if (lstat(child_abs, &st) < 0) { + rc = -1; + break; + } + if (S_ISDIR(st.st_mode)) { + if (scan_refs_dir(s, manifests, child_abs, child_rel, (size_t) rn, + depth + 1, migrated, skipped) < 0) { + rc = -1; + break; + } + continue; + } + if (!S_ISREG(st.st_mode)) + continue; + + /* Leaf file. Need at least one repository component plus the tag, so + * total depth must be >= 2 (registry / repo / tag). A leaf that lands + * directly under refs// has no repository component and is + * not a well-formed legacy pin; skip it with a warning so the store + * is not silently lossy. + */ + if (depth < 2) { + fprintf(stderr, + "elfuse oci: migration skipping refs/%s: not deep enough " + "for //\n", + child_rel); + (*skipped)++; + continue; + } + (void) name_len; + + /* Split child_rel = "/(/)*" + * "/". First '/' separates registry; last '/' separates tag. + */ + const char *first_slash = strchr(child_rel, '/'); + const char *last_slash = strrchr(child_rel, '/'); + if (!first_slash || !last_slash || first_slash == last_slash) { + fprintf(stderr, + "elfuse oci: migration skipping refs/%s: malformed path\n", + child_rel); + (*skipped)++; + continue; + } + size_t reg_len = (size_t) (first_slash - child_rel); + size_t repo_len = (size_t) (last_slash - first_slash - 1); + const char *repo_start = first_slash + 1; + const char *tag_start = last_slash + 1; + size_t tag_len = strlen(tag_start); + if (reg_len == 0 || repo_len == 0 || tag_len == 0) { + fprintf(stderr, + "elfuse oci: migration skipping refs/%s: empty path " + "component\n", + child_rel); + (*skipped)++; + continue; + } + + char *digest_str = read_legacy_pin_file(child_abs); + if (!digest_str) { + fprintf(stderr, + "elfuse oci: migration skipping refs/%s: pin file " + "unreadable or malformed\n", + child_rel); + (*skipped)++; + continue; + } + + /* Build canonical "/:" inline (cannot + * borrow oci_ref_canonical_name without round-tripping through the + * parser, which would reject repository components that the legacy + * code path happened to accept). */ + size_t total = reg_len + 1 + repo_len + 1 + tag_len + 1; + char *canon = malloc(total); + if (!canon) { + free(digest_str); + errno = ENOMEM; + rc = -1; + break; + } + char *wp = canon; + memcpy(wp, child_rel, reg_len); + wp += reg_len; + *wp++ = '/'; + memcpy(wp, repo_start, repo_len); + wp += repo_len; + *wp++ = ':'; + memcpy(wp, tag_start, tag_len); + wp += tag_len; + *wp = '\0'; + + int ar = migrate_append_descriptor(s, manifests, canon, digest_str); + free(canon); + free(digest_str); + if (ar < 0) { + rc = -1; + break; + } + if (ar > 0) { + (*skipped)++; + continue; + } + (*migrated)++; + } + closedir(dp); + return rc; +} + +static int migrate_legacy_refs(struct oci_store *s) +{ + char refs_path[STORE_PATH_MAX]; + int n = snprintf(refs_path, sizeof(refs_path), "%s/refs", s->root); + if (n < 0 || (size_t) n >= sizeof(refs_path)) { + errno = ENAMETOOLONG; + return -1; + } + struct stat st; + if (lstat(refs_path, &st) < 0) { + if (errno == ENOENT) + return 0; + return -1; + } + if (!S_ISDIR(st.st_mode)) + return 0; + + char index_path[STORE_PATH_MAX]; + n = snprintf(index_path, sizeof(index_path), "%s/index.json", s->root); + if (n < 0 || (size_t) n >= sizeof(index_path)) { + errno = ENAMETOOLONG; + return -1; + } + if (lstat(index_path, &st) == 0) + return 0; + if (errno != ENOENT) + return -1; + + /* Race window: between the unlocked check and acquire_index_lock a + * concurrent put_ref / open may have written index.json. Re-check under + * the lock and bail out if so, otherwise two migrations would race and + * the later write would partially overwrite a put_ref's descriptor. + */ + const char *lock_err = NULL; + int lock_fd = acquire_index_lock(s->root, &lock_err); + if (lock_fd < 0) + return -1; + + if (lstat(index_path, &st) == 0) { + close(lock_fd); + return 0; + } + if (errno != ENOENT) { + int saved = errno; + close(lock_fd); + errno = saved; + return -1; + } + + cJSON *root_json = new_empty_index(); + if (!root_json) { + close(lock_fd); + errno = ENOMEM; + return -1; + } + cJSON *manifests = cJSON_GetObjectItemCaseSensitive(root_json, "manifests"); + + size_t migrated = 0, skipped = 0; + int rc = + scan_refs_dir(s, manifests, refs_path, "", 0, 0, &migrated, &skipped); + if (rc < 0) { + int saved = errno; + cJSON_Delete(root_json); + close(lock_fd); + errno = saved; + return -1; + } + + /* Write even when migrated == 0 so a future open does not re-probe a + * refs/ tree that turned out to be empty or all-skipped. An empty + * index.json is the documented C2.2 happy-path shape (manifests: []). + */ + const char *write_err = NULL; + if (write_index_json(s->root, root_json, &write_err) < 0) { + int saved = errno; + cJSON_Delete(root_json); + close(lock_fd); + errno = saved; + return -1; + } + cJSON_Delete(root_json); + close(lock_fd); + + if (skipped > 0) { + fprintf(stderr, + "elfuse oci: migrated %zu pin(s) from refs/ to index.json " + "(refs/ kept for downgrade fallback; %zu pin(s) skipped)\n", + migrated, skipped); + } else { + fprintf(stderr, + "elfuse oci: migrated %zu pin(s) from refs/ to index.json " + "(refs/ kept for downgrade fallback)\n", + migrated); + } + return 0; +} + +/* --- Plan 3 C3.3d: layer + stack cache mark walker -------------------- */ + +/* Free a NULL-terminated heap-owned char ** array. */ +static void diff_id_strv_free(char **v) +{ + if (!v) + return; + for (size_t i = 0; v[i]; i++) + free(v[i]); + free((void *) v); +} + +/* Walk a directory tree summing the st_size of every regular file. Symlinks + * and other non-regular entries contribute zero (lstat does not follow). A + * missing entry (ENOENT) yields 0 so a concurrent rm cannot make the caller + * undercount what is still on disk. Other directory IO errors are treated as + * zero too because the prune sweep already counted the entry as a candidate + * and a partial size sum here would only shrink the reported reclaim figure; + * the recursive rm in apply_verdicts will surface the real failure. + * + * Duplicate of dedup-metrics.c::sum_tree_size; lift to a shared util when a + * third copy appears (rebuild-cache.c already carries its own rm_recursive + * for the same reason). + */ +static uint64_t dir_tree_size_sum(const char *path) +{ + struct stat st; + if (lstat(path, &st) < 0) + return 0; + if (S_ISREG(st.st_mode)) + return (uint64_t) st.st_size; + if (!S_ISDIR(st.st_mode)) + return 0; + DIR *d = opendir(path); + if (!d) + return 0; + uint64_t total = 0; + struct dirent *de; + while ((de = readdir(d)) != NULL) { + if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) + continue; + char child[STORE_PATH_MAX]; + int n = snprintf(child, sizeof(child), "%s/%s", path, de->d_name); + if (n < 0 || (size_t) n >= sizeof(child)) + continue; + total += dir_tree_size_sum(child); + } + closedir(d); + return total; +} + +/* Walk one image's manifest digest down to the linux/arm64 image-config and + * extract its rootfs.diff_ids as a heap-allocated NULL-terminated char **. + * + * Resolution path: + * - load + try image-manifest parse -> read config descriptor, load + parse + * image-config, return rootfs.diff_ids + * - else try image-index parse -> pick linux/arm64 sub-manifest, recurse + * if its blob is on disk, otherwise return a soft NO_LINUX_ARM64 result + * - else fatal: malformed blob + * + * Return discipline (see diff_id_resolve_t below): SOFT_NONE indicates "no + * linux/arm64 entry in this image-index" or "the picked sub-manifest blob is + * not on disk" and contributes zero to the keep set without surfacing as an + * error; HARD_FAIL indicates a corrupt or missing manifest / config blob and + * propagates as a fatal mark failure so prune cannot later delete reachable + * cache entries. + * + * Duplicate of dedup-metrics.c::resolve_config_digest + load_diff_ids (those + * helpers fold all failures into "skip the image"; the mark walker needs the + * fatal vs soft distinction). + */ +typedef enum { + DIFF_ID_RESOLVE_OK = 0, + DIFF_ID_RESOLVE_SOFT_NONE = 1, + DIFF_ID_RESOLVE_HARD_FAIL = 2, +} diff_id_resolve_t; + +static diff_id_resolve_t resolve_image_diff_ids(oci_store_t *s, + const char *manifest_digest, + char ***out_diff_ids, + const char **err) +{ + *out_diff_ids = NULL; + char *body = NULL; + size_t body_len = 0; + if (load_manifest_blob(s, manifest_digest, &body, &body_len) < 0) { + if (err) + *err = "collect_layer_roots: manifest blob missing or unreadable"; + return DIFF_ID_RESOLVE_HARD_FAIL; + } + + /* Image-manifest path: drill into its image-config. */ + oci_manifest_t mf = {0}; + if (oci_manifest_parse(body, body_len, &mf, NULL) == 0) { + char config_digest[OCI_DIGEST_HEX_MAX + 16]; + int dn = snprintf(config_digest, sizeof(config_digest), "%s", + mf.config.digest_str); + oci_manifest_free(&mf); + free(body); + if (dn < 0 || (size_t) dn >= sizeof(config_digest)) { + if (err) + *err = "collect_layer_roots: config digest overflow"; + errno = ENAMETOOLONG; + return DIFF_ID_RESOLVE_HARD_FAIL; + } + char *cfg_body = NULL; + size_t cfg_len = 0; + if (load_manifest_blob(s, config_digest, &cfg_body, &cfg_len) < 0) { + if (err) + *err = + "collect_layer_roots: image-config blob missing or " + "unreadable"; + return DIFF_ID_RESOLVE_HARD_FAIL; + } + oci_image_config_t cfg = {0}; + if (oci_image_config_parse(cfg_body, cfg_len, &cfg, NULL) < 0) { + free(cfg_body); + if (err) + *err = "collect_layer_roots: image-config blob unparseable"; + errno = EINVAL; + return DIFF_ID_RESOLVE_HARD_FAIL; + } + free(cfg_body); + /* Count and copy the diff_ids. Empty list yields a one-element NULL + * terminator so callers iterate uniformly. */ + size_t n = 0; + while (cfg.rootfs_diff_ids[n]) + n++; + char **copy = (char **) calloc(n + 1, sizeof(*copy)); + if (!copy) { + oci_image_config_free(&cfg); + if (err) + *err = "collect_layer_roots: diff_id strv alloc failed"; + errno = ENOMEM; + return DIFF_ID_RESOLVE_HARD_FAIL; + } + for (size_t i = 0; i < n; i++) { + copy[i] = strdup(cfg.rootfs_diff_ids[i]); + if (!copy[i]) { + diff_id_strv_free(copy); + oci_image_config_free(&cfg); + if (err) + *err = "collect_layer_roots: diff_id strdup failed"; + errno = ENOMEM; + return DIFF_ID_RESOLVE_HARD_FAIL; + } + } + oci_image_config_free(&cfg); + *out_diff_ids = copy; + return DIFF_ID_RESOLVE_OK; + } + memset(&mf, 0, sizeof(mf)); + + /* Image-index path: pick linux/arm64 and recurse. */ + oci_index_t idx = {0}; + if (oci_index_parse(body, body_len, &idx, NULL) < 0) { + free(body); + if (err) + *err = + "collect_layer_roots: blob is neither image-manifest nor " + "image-index"; + errno = EINVAL; + return DIFF_ID_RESOLVE_HARD_FAIL; + } + free(body); + const oci_index_entry_t *picked = oci_index_pick_linux_arm64(&idx); + if (!picked) { + oci_index_free(&idx); + return DIFF_ID_RESOLVE_SOFT_NONE; + } + char *sub_digest = strdup(picked->desc.digest_str); + oci_index_free(&idx); + if (!sub_digest) { + if (err) + *err = "collect_layer_roots: sub-manifest digest strdup failed"; + errno = ENOMEM; + return DIFF_ID_RESOLVE_HARD_FAIL; + } + if (!manifest_blob_exists(s, sub_digest)) { + /* Multi-arch pin where pull never fetched linux/arm64. Contribute + * nothing; the sub-manifest's layers are not on disk so there is + * nothing to keep. Matches expand_manifest_digest's soft policy for + * the same shape under blob mark. + */ + free(sub_digest); + return DIFF_ID_RESOLVE_SOFT_NONE; + } + diff_id_resolve_t rc = + resolve_image_diff_ids(s, sub_digest, out_diff_ids, err); + free(sub_digest); + return rc; +} + +/* Add every diff_id in the NULL-terminated list to *diff_set and every + * ChainID prefix (ChainID(L0..Lk) for k = 0..n-1) to *chain_set. The walker + * threads the running chain through a single buffer; oci_chainid_compute + * already handles the L0 passthrough case via a NULL prev argument. + * + * Returns 0 on success or -1 with errno set on allocation failure inside the + * digest set or chainid composition. err is populated on failure. + */ +static int add_diff_ids_and_chains(char *const *diff_ids, + oci_digest_set_t *diff_set, + oci_digest_set_t *chain_set, + const char **err) +{ + char prev[OCI_DIGEST_HEX_MAX + 16] = ""; + for (size_t i = 0; diff_ids[i]; i++) { + if (oci_digest_set_add(diff_set, diff_ids[i]) < 0) { + if (err) + *err = "collect_layer_roots: diff_id set add failed"; + return -1; + } + char chain[OCI_DIGEST_HEX_MAX + 16]; + const char *prev_arg = (i == 0) ? NULL : prev; + if (oci_chainid_compute(prev_arg, diff_ids[i], chain, sizeof(chain)) < + 0) { + if (err) + *err = "collect_layer_roots: chainid compute failed"; + return -1; + } + memcpy(prev, chain, strlen(chain) + 1); + if (oci_digest_set_add(chain_set, chain) < 0) { + if (err) + *err = "collect_layer_roots: chain set add failed"; + return -1; + } + } + return 0; +} + +int oci_store_collect_layer_roots(oci_store_t *s, + oci_digest_set_t *out_diff_ids, + oci_digest_set_t *out_chain_ids, + const char *volume_root, + const char **err) +{ + static const char *dummy_err; + if (!err) + err = &dummy_err; + *err = NULL; + if (!s || !out_diff_ids || !out_chain_ids) { + *err = "collect_layer_roots: NULL argument"; + errno = EINVAL; + return -1; + } + oci_digest_set_init(out_diff_ids); + oci_digest_set_init(out_chain_ids); + + /* Source 1: pins in index.json. */ + oci_pin_list_t pins = {0}; + const char *list_err = NULL; + if (oci_store_list_refs(s, &pins, &list_err) < 0) { + *err = list_err ? list_err + : "collect_layer_roots: oci_store_list_refs failed"; + oci_digest_set_free(out_diff_ids); + oci_digest_set_free(out_chain_ids); + return -1; + } + for (size_t i = 0; i < pins.count; i++) { + char **diff_ids = NULL; + diff_id_resolve_t rr = + resolve_image_diff_ids(s, pins.items[i].digest, &diff_ids, err); + if (rr == DIFF_ID_RESOLVE_HARD_FAIL) { + oci_pin_list_free(&pins); + oci_digest_set_free(out_diff_ids); + oci_digest_set_free(out_chain_ids); + return -1; + } + if (rr == DIFF_ID_RESOLVE_SOFT_NONE) + continue; + int ac = + add_diff_ids_and_chains(diff_ids, out_diff_ids, out_chain_ids, err); + diff_id_strv_free(diff_ids); + if (ac < 0) { + oci_pin_list_free(&pins); + oci_digest_set_free(out_diff_ids); + oci_digest_set_free(out_chain_ids); + return -1; + } + } + oci_pin_list_free(&pins); + + /* Source 2: unpacked image trees under /images/. The + * origin sidecar already carries the resolved diff_id list so no blob + * read is required here. + */ + if (volume_root) { + oci_volume_list_t trees = {0}; + const char *vlerr = NULL; + if (oci_volume_list_unpacked(volume_root, &trees, &vlerr) < 0) { + *err = vlerr ? vlerr + : "collect_layer_roots: volume_list_unpacked failed"; + oci_digest_set_free(out_diff_ids); + oci_digest_set_free(out_chain_ids); + return -1; + } + for (size_t i = 0; i < trees.count; i++) { + oci_origin_t origin = {0}; + const char *oerr = NULL; + if (oci_origin_read(trees.items[i], &origin, &oerr) < 0) { + *err = oerr ? oerr + : "collect_layer_roots: origin sidecar read failed"; + oci_volume_list_free(&trees); + oci_digest_set_free(out_diff_ids); + oci_digest_set_free(out_chain_ids); + return -1; + } + if (origin.layer_diffids) { + if (add_diff_ids_and_chains(origin.layer_diffids, out_diff_ids, + out_chain_ids, err) < 0) { + oci_origin_free(&origin); + oci_volume_list_free(&trees); + oci_digest_set_free(out_diff_ids); + oci_digest_set_free(out_chain_ids); + return -1; + } + } + oci_origin_free(&origin); + } + oci_volume_list_free(&trees); + } + return 0; +} + +/* Algorithm set this build expects to find under blobs/. Other algorithm + * subdirectories (a future operator hand-created sha384/, for instance) + * are left untouched: sweep only inspects directories it recognises. + */ +static const oci_digest_algo_t PRUNE_ALGOS[] = { + OCI_DIGEST_SHA256, + OCI_DIGEST_SHA512, +}; + +/* One dangling-blob entry produced by the classify phase and consumed + * by the apply phase. path is heap-owned. verdict starts at PRUNE and + * may be flipped to SKIP by the older-than veto or the keep-bytes + * budget. size is the on-disk byte count (st_size at classify time); + * mtime is st_mtime, used as the sort key for the LRU budget and the + * comparison source for the older-than cutoff. + */ +typedef enum { + PRUNE_VERDICT_PRUNE = 0, + PRUNE_VERDICT_SKIP = 1, +} prune_verdict_t; + +typedef struct { + char *path; + uint64_t size; + time_t mtime; + prune_verdict_t verdict; +} prune_candidate_t; + +typedef struct { + prune_candidate_t *items; + size_t count; + size_t cap; +} prune_candidate_list_t; + +/* Append one dangling-blob entry. Doubles cap from 32 so the realloc + * cost amortizes across a typical store's tens-to-hundreds of blobs. + * On alloc failure returns -1 with errno=ENOMEM; the caller is + * responsible for cleaning up entries staged so far via + * prune_candidate_list_free. + */ +static int prune_candidate_list_append(prune_candidate_list_t *list, + char *path, + uint64_t size, + time_t mtime) +{ + if (list->count == list->cap) { + size_t new_cap = list->cap ? list->cap * 2 : 32; + prune_candidate_t *grown = + realloc(list->items, new_cap * sizeof(*grown)); + if (!grown) { + errno = ENOMEM; + return -1; + } + list->items = grown; + list->cap = new_cap; + } + list->items[list->count].path = path; + list->items[list->count].size = size; + list->items[list->count].mtime = mtime; + list->items[list->count].verdict = PRUNE_VERDICT_PRUNE; + list->count++; + return 0; +} + +static void prune_candidate_list_free(prune_candidate_list_t *list) +{ + if (!list) + return; + for (size_t i = 0; i < list->count; i++) + free(list->items[i].path); + free(list->items); + list->items = NULL; + list->count = 0; + list->cap = 0; +} + +/* qsort comparator over an indirection array of candidate pointers + * (prune_candidate_t **). Sort key is ascending mtime with the path + * as tie-breaker so order stays deterministic on stores that + * materialize blobs in quick succession (the test suite needs + * stable LRU picks against a fixture). + */ +static int prune_candidate_ptr_cmp_mtime_asc(const void *a, const void *b) +{ + const prune_candidate_t *pa = *(const prune_candidate_t *const *) a; + const prune_candidate_t *pb = *(const prune_candidate_t *const *) b; + if (pa->mtime < pb->mtime) + return -1; + if (pa->mtime > pb->mtime) + return 1; + return strcmp(pa->path, pb->path); +} + +/* The three cache families share a single sweep pipeline. Family selects + * how apply_verdicts removes a PRUNE-verdict entry (unlink vs recursive rm) + * and informs diagnostics; the classify and filter passes operate uniformly + * on prune_candidate_list_t. + */ +typedef enum { + PRUNE_FAMILY_BLOB = 0, /* /blobs// regular files */ + PRUNE_FAMILY_TREE = 1, /* /layers/.../// directories */ +} prune_family_t; + +/* Classify one blobs// directory. For every regular file whose + * name is a valid lowercase hex digest of the right length, build the + * canonical ":" digest, look it up in the keep set, and + * either bump *out_kept (reachable) or append a candidate (dangling). + * lstat ENOENT mid-walk is treated as a concurrent prune and skipped + * silently. Subdirectories, dotfiles, and otherwise-shaped entries + * pass through untouched so the OCI image-layout spec's regular-blob + * convention is preserved without trampling foreign state. + * + * Returns 0 on success and -1 on unrecoverable IO failure with errno + * preserved. + */ +static int classify_algo_dir(oci_store_t *s, + oci_digest_algo_t algo, + const oci_digest_set_t *keep, + size_t *out_kept, + prune_candidate_list_t *list, + const char **err) +{ + const char *algo_name = oci_digest_algo_name(algo); + if (!algo_name) { + if (err) + *err = "prune: unknown digest algorithm"; + errno = EINVAL; + return -1; + } + + char dir_path[STORE_PATH_MAX]; + int n = + snprintf(dir_path, sizeof(dir_path), "%s/blobs/%s", s->root, algo_name); + if (n < 0 || (size_t) n >= sizeof(dir_path)) { + if (err) + *err = "prune: blobs/ path exceeds STORE_PATH_MAX"; + errno = ENAMETOOLONG; + return -1; + } + + DIR *dp = opendir(dir_path); + if (!dp) { + if (errno == ENOENT) + return 0; + if (err) + *err = "prune: opendir on blobs/ failed"; + return -1; + } + + int rc = 0; + struct dirent *de; + size_t hex_len = oci_digest_hex_len(algo); + while ((de = readdir(dp)) != NULL) { + const char *name = de->d_name; + if (name[0] == '.' && + (name[1] == '\0' || (name[1] == '.' && name[2] == '\0'))) + continue; + if (name[0] == '.') + continue; + /* Reject anything that is not the expected hex shape before + * paying for an lstat. This both filters subdirectories + * (whose names rarely happen to be 64 hex chars) and shields + * the digest_set lookup from non-blob filenames. + */ + if (strlen(name) != hex_len) + continue; + if (!oci_digest_hex_valid(algo, name)) + continue; + + char blob_path[STORE_PATH_MAX]; + int bn = + snprintf(blob_path, sizeof(blob_path), "%s/%s", dir_path, name); + if (bn < 0 || (size_t) bn >= sizeof(blob_path)) { + if (err) + *err = "prune: blob path exceeds STORE_PATH_MAX"; + errno = ENAMETOOLONG; + rc = -1; + break; + } + + struct stat st; + if (lstat(blob_path, &st) < 0) { + if (errno == ENOENT) + continue; + if (err) + *err = "prune: lstat on blob failed"; + rc = -1; + break; + } + if (!S_ISREG(st.st_mode)) + continue; + + char digest[OCI_DIGEST_HEX_MAX + 16]; + int dn = snprintf(digest, sizeof(digest), "%s:%s", algo_name, name); + if (dn < 0 || (size_t) dn >= sizeof(digest)) { + if (err) + *err = "prune: digest string buffer too small"; + errno = ENAMETOOLONG; + rc = -1; + break; + } + + if (oci_digest_set_contains(keep, digest)) { + (*out_kept)++; + continue; + } + + char *path_copy = strdup(blob_path); + if (!path_copy) { + if (err) + *err = "prune: strdup blob path failed"; + errno = ENOMEM; + rc = -1; + break; + } + if (prune_candidate_list_append(list, path_copy, (uint64_t) st.st_size, + st.st_mtime) < 0) { + free(path_copy); + if (err) + *err = "prune: candidate list grow failed"; + rc = -1; + break; + } + } + closedir(dp); + return rc; +} + +/* Classify one tree-shaped cache directory (layers// or + * layers/stacks//). The base_subpath argument is the relative path + * beneath the store root, e.g. "layers/sha256" or "layers/stacks/sha256"; + * it lets one helper drive both the raw layer cache and the ChainID-keyed + * stack cache without duplicating the dir-walk plumbing. + * + * For every immediate child whose name is a valid lowercase hex digest of + * the right length AND whose lstat reports a directory, compose the + * canonical ":" digest and look it up in the keep set. Misses + * (dangling cache entries) get appended to the candidate list together with + * the recursive size of the entry tree and the directory's own st_mtime + * (set by rename(2) at commit time, so newer entries sort newer). Hits + * bump *out_kept. + * + * Non-directory entries, dotfiles, sibling .schema / .staging markers, and + * malformed names are all skipped silently so the caller never deletes + * foreign state. Missing base directory (fresh store before any unpack) + * yields 0 with no entries. Other IO failures are fatal. + */ +static int classify_tree_cache_dir(oci_store_t *s, + const char *base_subpath, + oci_digest_algo_t algo, + const oci_digest_set_t *keep, + size_t *out_kept, + prune_candidate_list_t *list, + const char **err) +{ + const char *algo_name = oci_digest_algo_name(algo); + if (!algo_name) { + if (err) + *err = "prune: unknown digest algorithm"; + errno = EINVAL; + return -1; + } + + char dir_path[STORE_PATH_MAX]; + int n = + snprintf(dir_path, sizeof(dir_path), "%s/%s", s->root, base_subpath); + if (n < 0 || (size_t) n >= sizeof(dir_path)) { + if (err) + *err = "prune: tree-cache path exceeds STORE_PATH_MAX"; + errno = ENAMETOOLONG; + return -1; + } + + DIR *dp = opendir(dir_path); + if (!dp) { + if (errno == ENOENT) + return 0; + if (err) + *err = "prune: opendir on tree-cache dir failed"; + return -1; + } + + int rc = 0; + struct dirent *de; + size_t hex_len = oci_digest_hex_len(algo); + while ((de = readdir(dp)) != NULL) { + const char *name = de->d_name; + if (name[0] == '.' && + (name[1] == '\0' || (name[1] == '.' && name[2] == '\0'))) + continue; + if (name[0] == '.') + continue; + if (strlen(name) != hex_len) + continue; + if (!oci_digest_hex_valid(algo, name)) + continue; + + char entry_path[STORE_PATH_MAX]; + int en = + snprintf(entry_path, sizeof(entry_path), "%s/%s", dir_path, name); + if (en < 0 || (size_t) en >= sizeof(entry_path)) { + if (err) + *err = "prune: tree-cache entry path exceeds STORE_PATH_MAX"; + errno = ENAMETOOLONG; + rc = -1; + break; + } + + struct stat st; + if (lstat(entry_path, &st) < 0) { + if (errno == ENOENT) + continue; + if (err) + *err = "prune: lstat on tree-cache entry failed"; + rc = -1; + break; + } + if (!S_ISDIR(st.st_mode)) + continue; + + char digest[OCI_DIGEST_HEX_MAX + 16]; + int dn = snprintf(digest, sizeof(digest), "%s:%s", algo_name, name); + if (dn < 0 || (size_t) dn >= sizeof(digest)) { + if (err) + *err = "prune: digest string buffer too small"; + errno = ENAMETOOLONG; + rc = -1; + break; + } + + if (oci_digest_set_contains(keep, digest)) { + (*out_kept)++; + continue; + } + + uint64_t tree_bytes = dir_tree_size_sum(entry_path); + char *path_copy = strdup(entry_path); + if (!path_copy) { + if (err) + *err = "prune: strdup tree-cache path failed"; + errno = ENOMEM; + rc = -1; + break; + } + if (prune_candidate_list_append(list, path_copy, tree_bytes, + st.st_mtime) < 0) { + free(path_copy); + if (err) + *err = "prune: candidate list grow failed"; + rc = -1; + break; + } + } + closedir(dp); + return rc; +} + +/* Apply the C1.4 filter passes to the candidate list. Both passes + * mutate verdict only; nothing is unlinked here. The caller invokes + * apply_verdicts afterwards to count + (when commit) unlink. + * + * older-than veto (B1) inspects each candidate independently: when + * older_than_sec is non-zero and (now - mtime) is less than the + * cutoff, verdict flips to SKIP. now is provided by the caller so a + * single time(NULL) snapshot drives the whole filter pass (avoids + * the boundary case where a candidate flips between PRUNE and SKIP + * across two sequential time(NULL) reads). + * + * keep-bytes budget (B2) operates over the candidates still in PRUNE + * state after B1. Their pointers are gathered, sorted by mtime + * ascending, and walked newest-first. The newest candidates whose + * cumulative size fits keep_bytes flip to SKIP; the first candidate + * that does not fit terminates the walk so any older candidate stays + * in PRUNE even if its own size would have fit alone. This matches + * LRU semantics: oldest evicted first, regardless of size. + */ +static int apply_filters(oci_store_prune_options_t *opts, + prune_candidate_list_t *list, + time_t now, + const char **err) +{ + if (opts->older_than_sec > 0) { + time_t cutoff = (time_t) opts->older_than_sec; + for (size_t i = 0; i < list->count; i++) { + if (list->items[i].verdict != PRUNE_VERDICT_PRUNE) + continue; + time_t age = now - list->items[i].mtime; + if (age < cutoff) + list->items[i].verdict = PRUNE_VERDICT_SKIP; + } + } + + if (opts->keep_bytes > 0 && list->count > 0) { + prune_candidate_t **active = + (prune_candidate_t **) malloc(list->count * sizeof(*active)); + if (!active) { + if (err) + *err = "prune: out of memory ranking candidates"; + errno = ENOMEM; + return -1; + } + size_t na = 0; + for (size_t i = 0; i < list->count; i++) { + if (list->items[i].verdict == PRUNE_VERDICT_PRUNE) + active[na++] = &list->items[i]; + } + if (na > 0) { + /* Sort the active subset through an indirection array so + * the candidate-list iteration order in apply_verdicts + * stays in insertion order (the test suite is easier to + * reason about when path verdicts read in the same order + * the classify phase produced them). + */ + qsort((void *) active, na, sizeof(*active), + prune_candidate_ptr_cmp_mtime_asc); + uint64_t running = 0; + for (ssize_t i = (ssize_t) na - 1; i >= 0; i--) { + /* Use unsigned arithmetic with an overflow guard so a + * pathological size never wraps the accumulator. + */ + uint64_t next = running + active[i]->size; + if (next < running) { + /* Overflow: cannot fit any more blobs under the + * budget, so stop reclassifying. + */ + break; + } + if (next <= opts->keep_bytes) { + active[i]->verdict = PRUNE_VERDICT_SKIP; + running = next; + } else { + break; + } + } + } + free((void *) active); + } + return 0; +} + +/* Forward declaration for the recursive rm helper used by the TREE family + * removal path below; the implementation lives with the layer cache helpers + * later in this file. + */ +static int layer_stage_rm(const char *path); + +/* Materialise filter verdicts onto disk and stats. Every candidate + * contributes to exactly one output bucket: SKIP -> skipped_*, PRUNE + * -> pruned_* (and removal when commit). The removal failure policy + * mirrors C1.3: ENOENT is treated as a concurrent prune and counted + * silently; any other errno is fatal so the caller's stats never + * report bytes we did not actually reclaim. family selects the + * removal primitive: BLOB uses unlink(2) on a regular file; TREE uses + * layer_stage_rm on a directory subtree so a populated cache entry is + * taken down in one call. The four output pointers let the caller + * route the counters into the per-family stats fields (kept lives in + * classify; this function only writes pruned + skipped). + */ +static int apply_verdicts(prune_candidate_list_t *list, + bool commit, + prune_family_t family, + size_t *out_pruned_count, + uint64_t *out_pruned_bytes, + size_t *out_skipped_count, + uint64_t *out_skipped_bytes, + const char **err) +{ + for (size_t i = 0; i < list->count; i++) { + if (list->items[i].verdict == PRUNE_VERDICT_SKIP) { + (*out_skipped_count)++; + *out_skipped_bytes += list->items[i].size; + continue; + } + (*out_pruned_count)++; + *out_pruned_bytes += list->items[i].size; + if (!commit) + continue; + if (family == PRUNE_FAMILY_BLOB) { + if (unlink(list->items[i].path) < 0) { + if (errno == ENOENT) + continue; + if (err) + *err = "prune: unlink on dangling blob failed"; + return -1; + } + } else { + /* TREE: recursive rm tolerates ENOENT internally via lstat + * but still returns -1 on any other failure mid-walk. The + * stats already count the entry as pruned so a partial + * teardown that succeeds for some children leaves the stats + * consistent with what was actually freed. + */ + if (layer_stage_rm(list->items[i].path) < 0) { + if (errno == ENOENT) + continue; + if (err) + *err = + "prune: recursive rm on dangling cache entry " + "failed"; + return -1; + } + } + } + return 0; +} + +int oci_store_prune(oci_store_t *s, + oci_store_prune_options_t *opts, + const char **err) +{ + static const char *dummy_err; + if (!err) + err = &dummy_err; + *err = NULL; + if (!s || !opts) { + *err = "prune: NULL argument"; + errno = EINVAL; + return -1; + } + + opts->kept_blobs = 0; + opts->pruned_blobs = 0; + opts->pruned_bytes = 0; + opts->skipped_blobs = 0; + opts->skipped_bytes = 0; + opts->kept_layers = 0; + opts->pruned_layers = 0; + opts->pruned_layer_bytes = 0; + opts->skipped_layers = 0; + opts->skipped_layer_bytes = 0; + opts->kept_stacks = 0; + opts->pruned_stacks = 0; + opts->pruned_stack_bytes = 0; + opts->skipped_stacks = 0; + opts->skipped_stack_bytes = 0; + + /* Serialize against oci_store_put_ref so a pull cannot publish a + * new pin between the mark snapshot and the sweep. Mark and sweep + * for all three cache families share this single lock window so + * the blob keep set, the diff_id keep set, and the chain_id keep + * set are derived from one consistent view of pins + unpacked + * sysroots. + */ + int lock_fd = acquire_index_lock(s->root, err); + if (lock_fd < 0) + return -1; + + /* Mark phase: build three keep sets in one window. oci_digest_set_free + * is safe on zero-initialised structs so a partial mark still cleans + * up correctly via the single done: label below. + */ + oci_digest_set_t keep_blobs = {0}; + oci_digest_set_t keep_diff_ids = {0}; + oci_digest_set_t keep_chain_ids = {0}; + int rc = 0; + if (oci_store_collect_roots(s, &keep_blobs, opts->volume_root, err) < 0) { + rc = -1; + goto done; + } + if (oci_store_collect_layer_roots(s, &keep_diff_ids, &keep_chain_ids, + opts->volume_root, err) < 0) { + rc = -1; + goto done; + } + + /* Sweep phase: each family classifies, filters, and applies independently + * against its own keep set and candidate list. The filter passes use the + * same opts->older_than_sec / opts->keep_bytes inputs but each family + * runs its own keep-bytes budget so a fat blob cannot crowd a layer + * eviction (or vice versa) off a shared global budget. + */ + time_t now = time(NULL); + + /* Family 1: blobs */ + prune_candidate_list_t blob_candidates = {0}; + for (size_t i = 0; i < sizeof(PRUNE_ALGOS) / sizeof(PRUNE_ALGOS[0]); i++) { + if (classify_algo_dir(s, PRUNE_ALGOS[i], &keep_blobs, &opts->kept_blobs, + &blob_candidates, err) < 0) { + prune_candidate_list_free(&blob_candidates); + rc = -1; + goto done; + } + } + if (apply_filters(opts, &blob_candidates, now, err) < 0) { + prune_candidate_list_free(&blob_candidates); + rc = -1; + goto done; + } + if (apply_verdicts(&blob_candidates, opts->commit, PRUNE_FAMILY_BLOB, + &opts->pruned_blobs, &opts->pruned_bytes, + &opts->skipped_blobs, &opts->skipped_bytes, err) < 0) { + prune_candidate_list_free(&blob_candidates); + rc = -1; + goto done; + } + prune_candidate_list_free(&blob_candidates); + + /* Family 2: layers/// raw cache directories */ + prune_candidate_list_t layer_candidates = {0}; + for (size_t i = 0; i < sizeof(PRUNE_ALGOS) / sizeof(PRUNE_ALGOS[0]); i++) { + const char *algo_name = oci_digest_algo_name(PRUNE_ALGOS[i]); + char base[STORE_PATH_MAX]; + int bn = snprintf(base, sizeof(base), "layers/%s", algo_name); + if (bn < 0 || (size_t) bn >= sizeof(base)) { + *err = "prune: layers/ subpath overflow"; + errno = ENAMETOOLONG; + prune_candidate_list_free(&layer_candidates); + rc = -1; + goto done; + } + if (classify_tree_cache_dir(s, base, PRUNE_ALGOS[i], &keep_diff_ids, + &opts->kept_layers, &layer_candidates, + err) < 0) { + prune_candidate_list_free(&layer_candidates); + rc = -1; + goto done; + } + } + if (apply_filters(opts, &layer_candidates, now, err) < 0) { + prune_candidate_list_free(&layer_candidates); + rc = -1; + goto done; + } + if (apply_verdicts(&layer_candidates, opts->commit, PRUNE_FAMILY_TREE, + &opts->pruned_layers, &opts->pruned_layer_bytes, + &opts->skipped_layers, &opts->skipped_layer_bytes, + err) < 0) { + prune_candidate_list_free(&layer_candidates); + rc = -1; + goto done; + } + prune_candidate_list_free(&layer_candidates); + + /* Family 3: layers/stacks/// ChainID-keyed snapshots */ + prune_candidate_list_t stack_candidates = {0}; + for (size_t i = 0; i < sizeof(PRUNE_ALGOS) / sizeof(PRUNE_ALGOS[0]); i++) { + const char *algo_name = oci_digest_algo_name(PRUNE_ALGOS[i]); + char base[STORE_PATH_MAX]; + int bn = snprintf(base, sizeof(base), "layers/stacks/%s", algo_name); + if (bn < 0 || (size_t) bn >= sizeof(base)) { + *err = "prune: layers/stacks/ subpath overflow"; + errno = ENAMETOOLONG; + prune_candidate_list_free(&stack_candidates); + rc = -1; + goto done; + } + if (classify_tree_cache_dir(s, base, PRUNE_ALGOS[i], &keep_chain_ids, + &opts->kept_stacks, &stack_candidates, + err) < 0) { + prune_candidate_list_free(&stack_candidates); + rc = -1; + goto done; + } + } + if (apply_filters(opts, &stack_candidates, now, err) < 0) { + prune_candidate_list_free(&stack_candidates); + rc = -1; + goto done; + } + if (apply_verdicts(&stack_candidates, opts->commit, PRUNE_FAMILY_TREE, + &opts->pruned_stacks, &opts->pruned_stack_bytes, + &opts->skipped_stacks, &opts->skipped_stack_bytes, + err) < 0) { + prune_candidate_list_free(&stack_candidates); + rc = -1; + goto done; + } + prune_candidate_list_free(&stack_candidates); + +done:; + int saved = errno; + oci_digest_set_free(&keep_chain_ids); + oci_digest_set_free(&keep_diff_ids); + oci_digest_set_free(&keep_blobs); + close(lock_fd); + errno = saved; + return rc; +} + +/* --- Plan 3 C3.2: layer cache helpers ---------------------------------- */ + +/* Parse a ":" digest into its components and the lowercase + * algorithm name used as the cache subdir. Shared by the per-layer raw + * cache (keyed by diff_id) and the ChainID-keyed stack cache (C3.3c), + * both of which materialise as /layers/.../// on disk. + * Validation matches the digest library; oci_digest_parse already rejects + * unknown algos and bad hex. + */ +static int parse_digest_for_cache_dir(const char *digest_str, + oci_digest_algo_t *out_algo, + char *out_hex, + const char **out_algo_name) +{ + if (!digest_str || !*digest_str) { + errno = EINVAL; + return -1; + } + oci_digest_algo_t algo; + char hex[OCI_DIGEST_HEX_MAX + 1]; + if (!oci_digest_parse(digest_str, &algo, hex)) { + errno = EINVAL; + return -1; + } + const char *name = oci_digest_algo_name(algo); + if (!name) { + errno = EINVAL; + return -1; + } + *out_algo = algo; + memcpy(out_hex, hex, strlen(hex) + 1); + *out_algo_name = name; + return 0; +} + +int oci_store_layer_has(oci_store_t *s, const char *diff_id) +{ + if (!s) { + errno = EINVAL; + return -1; + } + oci_digest_algo_t algo; + char hex[OCI_DIGEST_HEX_MAX + 1]; + const char *algo_name = NULL; + if (parse_digest_for_cache_dir(diff_id, &algo, hex, &algo_name) < 0) + return -1; + + char path[STORE_PATH_MAX]; + int n = snprintf(path, sizeof(path), "%s/layers/%s/%s", s->root, algo_name, + hex); + if (n < 0 || (size_t) n >= sizeof(path)) { + errno = ENAMETOOLONG; + return -1; + } + struct stat st; + if (stat(path, &st) < 0) { + if (errno == ENOENT) + return 0; + return -1; + } + if (!S_ISDIR(st.st_mode)) { + errno = ENOTDIR; + return -1; + } + return 1; +} + +int oci_store_layer_resolve(oci_store_t *s, + const char *diff_id, + char *out, + size_t cap) +{ + if (!s || !out || cap == 0) { + errno = EINVAL; + return -1; + } + oci_digest_algo_t algo; + char hex[OCI_DIGEST_HEX_MAX + 1]; + const char *algo_name = NULL; + if (parse_digest_for_cache_dir(diff_id, &algo, hex, &algo_name) < 0) + return -1; + int n = snprintf(out, cap, "%s/layers/%s/%s/", s->root, algo_name, hex); + if (n < 0 || (size_t) n >= cap) { + errno = ENAMETOOLONG; + return -1; + } + return 0; +} + +/* Produce 12 lowercase hex chars from 6 random bytes. Mirrors the local + * rand_hex helpers in src/oci/unpack.c and src/oci/clone-rootfs.c; kept + * static here so store.c stays self-contained. + */ +static int layer_stage_rand_suffix(char out[13]) +{ + uint8_t raw[6]; + if (getentropy(raw, sizeof(raw)) < 0) + return -1; + static const char hex[] = "0123456789abcdef"; + for (size_t i = 0; i < sizeof(raw); i++) { + out[i * 2] = hex[raw[i] >> 4]; + out[i * 2 + 1] = hex[raw[i] & 0xf]; + } + out[12] = '\0'; + return 0; +} + +int oci_store_layer_stage_path(oci_store_t *s, + const char *diff_id, + char *out, + size_t cap) +{ + if (!s || !out || cap == 0) { + errno = EINVAL; + return -1; + } + oci_digest_algo_t algo; + char hex[OCI_DIGEST_HEX_MAX + 1]; + const char *algo_name = NULL; + if (parse_digest_for_cache_dir(diff_id, &algo, hex, &algo_name) < 0) + return -1; + char rand_suffix[13]; + if (layer_stage_rand_suffix(rand_suffix) < 0) + return -1; + int n = snprintf(out, cap, "%s/layers/.staging/%s-%s-%s", s->root, + algo_name, hex, rand_suffix); + if (n < 0 || (size_t) n >= cap) { + errno = ENAMETOOLONG; + return -1; + } + return 0; +} + +/* Recursively rm a path (file, symlink, or directory). Mirrors the discipline + * in src/oci/clone-rootfs.c so the layer stage abort path does not shell out. + * Returns 0 on success or when path was already absent; -1 with errno set on + * any unexpected IO error. Designed for staging cleanup, not as a general- + * purpose rm. + */ +static int layer_stage_rm(const char *path) +{ + struct stat st; + if (lstat(path, &st) < 0) { + if (errno == ENOENT) + return 0; + return -1; + } + if (!S_ISDIR(st.st_mode)) + return unlink(path); + DIR *d = opendir(path); + if (!d) + return -1; + struct dirent *de; + int rc = 0; + while ((de = readdir(d))) { + if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) + continue; + char child[STORE_PATH_MAX]; + int n = snprintf(child, sizeof(child), "%s/%s", path, de->d_name); + if (n < 0 || (size_t) n >= sizeof(child)) { + errno = ENAMETOOLONG; + rc = -1; + break; + } + if (layer_stage_rm(child) < 0) { + rc = -1; + break; + } + } + closedir(d); + if (rc == 0 && rmdir(path) < 0) + rc = -1; + return rc; +} + +int oci_store_layer_commit(oci_store_t *s, + const char *stage_path, + const char *diff_id, + const char **err) +{ + static const char *dummy_err; + if (!err) + err = &dummy_err; + *err = NULL; + if (!s || !stage_path || !*stage_path) { + *err = "layer_commit: NULL argument"; + errno = EINVAL; + return -1; + } + oci_digest_algo_t algo; + char hex[OCI_DIGEST_HEX_MAX + 1]; + const char *algo_name = NULL; + if (parse_digest_for_cache_dir(diff_id, &algo, hex, &algo_name) < 0) { + *err = "layer_commit: invalid diff_id"; + return -1; + } + char dest[STORE_PATH_MAX]; + int n = snprintf(dest, sizeof(dest), "%s/layers/%s/%s", s->root, algo_name, + hex); + if (n < 0 || (size_t) n >= sizeof(dest)) { + *err = "layer_commit: dest path exceeds STORE_PATH_MAX"; + errno = ENAMETOOLONG; + return -1; + } + if (rename(stage_path, dest) == 0) + return 0; + int saved = errno; + if (saved == EEXIST || saved == ENOTEMPTY) { + /* Concurrent writer landed the same entry first; drop the loser's + * staging tree and treat this as a benign success. The cache content + * is content-addressed so the winning entry is byte-equivalent. + */ + (void) layer_stage_rm(stage_path); + errno = 0; + return 0; + } + *err = "layer_commit: rename to layers/// failed"; + errno = saved; + return -1; +} + +/* --- Plan 3 C3.3c: ChainID stack cache helpers ------------------------- */ + +int oci_store_stack_has(oci_store_t *s, const char *chain_id) +{ + if (!s) { + errno = EINVAL; + return -1; + } + oci_digest_algo_t algo; + char hex[OCI_DIGEST_HEX_MAX + 1]; + const char *algo_name = NULL; + if (parse_digest_for_cache_dir(chain_id, &algo, hex, &algo_name) < 0) + return -1; + + char path[STORE_PATH_MAX]; + int n = snprintf(path, sizeof(path), "%s/layers/stacks/%s/%s", s->root, + algo_name, hex); + if (n < 0 || (size_t) n >= sizeof(path)) { + errno = ENAMETOOLONG; + return -1; + } + struct stat st; + if (stat(path, &st) < 0) { + if (errno == ENOENT) + return 0; + return -1; + } + if (!S_ISDIR(st.st_mode)) { + errno = ENOTDIR; + return -1; + } + return 1; +} + +int oci_store_stack_resolve(oci_store_t *s, + const char *chain_id, + char *out, + size_t cap) +{ + if (!s || !out || cap == 0) { + errno = EINVAL; + return -1; + } + oci_digest_algo_t algo; + char hex[OCI_DIGEST_HEX_MAX + 1]; + const char *algo_name = NULL; + if (parse_digest_for_cache_dir(chain_id, &algo, hex, &algo_name) < 0) + return -1; + int n = + snprintf(out, cap, "%s/layers/stacks/%s/%s/", s->root, algo_name, hex); + if (n < 0 || (size_t) n >= cap) { + errno = ENAMETOOLONG; + return -1; + } + return 0; +} + +int oci_store_stack_stage_path(oci_store_t *s, + const char *chain_id, + char *out, + size_t cap) +{ + if (!s || !out || cap == 0) { + errno = EINVAL; + return -1; + } + oci_digest_algo_t algo; + char hex[OCI_DIGEST_HEX_MAX + 1]; + const char *algo_name = NULL; + if (parse_digest_for_cache_dir(chain_id, &algo, hex, &algo_name) < 0) + return -1; + char rand_suffix[13]; + if (layer_stage_rand_suffix(rand_suffix) < 0) + return -1; + /* The "stack-" prefix keeps stack stage paths visually distinct from + * per-layer raw cache stage paths inside the shared .staging/ dir. + * Commit publishes to a different destination tree so the prefix is + * purely a debug aid; the rename is what actually disambiguates the + * two artifact families. + */ + int n = snprintf(out, cap, "%s/layers/.staging/stack-%s-%s-%s", s->root, + algo_name, hex, rand_suffix); + if (n < 0 || (size_t) n >= cap) { + errno = ENAMETOOLONG; + return -1; + } + return 0; +} + +int oci_store_stack_commit(oci_store_t *s, + const char *stage_path, + const char *chain_id, + const char **err) +{ + static const char *dummy_err; + if (!err) + err = &dummy_err; + *err = NULL; + if (!s || !stage_path || !*stage_path) { + *err = "stack_commit: NULL argument"; + errno = EINVAL; + return -1; + } + oci_digest_algo_t algo; + char hex[OCI_DIGEST_HEX_MAX + 1]; + const char *algo_name = NULL; + if (parse_digest_for_cache_dir(chain_id, &algo, hex, &algo_name) < 0) { + *err = "stack_commit: invalid chain_id"; + return -1; + } + char dest[STORE_PATH_MAX]; + int n = snprintf(dest, sizeof(dest), "%s/layers/stacks/%s/%s", s->root, + algo_name, hex); + if (n < 0 || (size_t) n >= sizeof(dest)) { + *err = "stack_commit: dest path exceeds STORE_PATH_MAX"; + errno = ENAMETOOLONG; + return -1; + } + if (rename(stage_path, dest) == 0) + return 0; + int saved = errno; + if (saved == EEXIST || saved == ENOTEMPTY) { + /* Concurrent writer landed the same entry first; drop the loser's + * staging tree and treat this as a benign success. Stack snapshots + * are content-addressed via ChainID so the winning entry is byte- + * equivalent. + */ + (void) layer_stage_rm(stage_path); + errno = 0; + return 0; + } + *err = "stack_commit: rename to layers/stacks/// failed"; + errno = saved; + return -1; +} + +/* --- Plan 3 C3.3b: layer cache schema marker --------------------------- */ + +/* Relative path of the schema marker beneath the store root. */ +static const char LAYER_SCHEMA_REL_PATH[] = "layers/.schema"; + +/* Schema version this build writes and accepts. v1 was the implicit + * C3.2 cumulative-by-diff_id layout (no marker). v2 will be the C3.3 + * raw per-layer payload plus ChainID stack cache. C3.3b lands the + * marker + migration; C3.3c will rewrite the unpack assembly to + * populate and consume the v2 cache. + */ +#define LAYER_SCHEMA_VERSION_CURRENT 2 + +/* Body written on first migration to v2. The description field is + * informational; readers only key on schemaVersion. Trailing newline + * matches the oci-layout marker convention. */ +static const char LAYER_SCHEMA_V2_BODY[] = + "{\"schemaVersion\":2," + "\"description\":\"raw per-layer payload + ChainID stack cache\"}\n"; + +/* Upper bound on the marker file size. The expected body is ~80 bytes; + * anything larger is treated as a malformed marker rather than parsed. + */ +#define LAYER_SCHEMA_MAX_BYTES 4096 + +/* Counter used for the marker's tmp file suffix; kept distinct from the + * oci-layout counter so the two helpers do not contend on the same + * monotonic source. + */ +static unsigned long layer_schema_seq(void) +{ + static unsigned long n = 0; + return __sync_add_and_fetch(&n, 1); +} + +/* Read /layers/.schema and extract the schemaVersion field. On + * success returns 0 and writes the parsed integer to *out_version. On + * failure returns -1 with errno set (ENOENT when the marker is absent; + * EINVAL for unparseable JSON, missing schemaVersion field, wrong type, + * non-regular file, or size out of range; other errno values propagated + * from open / read / fstat). When non-NULL, *out_reason is populated on + * the -1 paths with a static description for the caller to surface via + * stderr. + */ +static int read_layer_schema_version(const char *root, + int *out_version, + const char **out_reason) +{ + if (out_reason) + *out_reason = NULL; + char path[STORE_PATH_MAX]; + int n = snprintf(path, sizeof(path), "%s/%s", root, LAYER_SCHEMA_REL_PATH); + if (n < 0 || (size_t) n >= sizeof(path)) { + if (out_reason) + *out_reason = "layers/.schema path exceeds STORE_PATH_MAX"; + errno = ENAMETOOLONG; + return -1; + } + int fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) { + if (errno == ENOENT) { + if (out_reason) + *out_reason = "layers/.schema absent"; + return -1; + } + if (out_reason) + *out_reason = "open layers/.schema failed"; + return -1; + } + struct stat st; + if (fstat(fd, &st) < 0) { + int saved = errno; + close(fd); + if (out_reason) + *out_reason = "fstat layers/.schema failed"; + errno = saved; + return -1; + } + if (!S_ISREG(st.st_mode)) { + close(fd); + if (out_reason) + *out_reason = "layers/.schema is not a regular file"; + errno = EINVAL; + return -1; + } + if (st.st_size <= 0 || st.st_size > LAYER_SCHEMA_MAX_BYTES) { + close(fd); + if (out_reason) + *out_reason = "layers/.schema size out of range"; + errno = EINVAL; + return -1; + } + char buf[LAYER_SCHEMA_MAX_BYTES + 1]; + ssize_t got = read(fd, buf, (size_t) st.st_size); + close(fd); + if (got != (ssize_t) st.st_size) { + if (out_reason) + *out_reason = "read layers/.schema failed"; + errno = EIO; + return -1; + } + buf[got] = '\0'; + cJSON *json = cJSON_Parse(buf); + if (!json) { + if (out_reason) + *out_reason = "layers/.schema JSON parse failed"; + errno = EINVAL; + return -1; + } + cJSON *v = cJSON_GetObjectItemCaseSensitive(json, "schemaVersion"); + if (!cJSON_IsNumber(v)) { + cJSON_Delete(json); + if (out_reason) + *out_reason = + "layers/.schema schemaVersion missing or not a number"; + errno = EINVAL; + return -1; + } + int version = v->valueint; + cJSON_Delete(json); + *out_version = version; + return 0; +} + +/* Recursively remove every direct child of /layers/sha256/, leaving + * the layers/sha256/ directory itself in place. Each child is dispatched + * through layer_stage_rm so symlinks, regular files, and nested + * directories are all handled identically. On the first IO failure the + * call returns -1 with errno preserved; *out_removed reflects the entry + * count successfully removed before the error. layer_stage_rm is + * ENOENT-tolerant so a partial wipe resumes cleanly on a later open. + */ +static int wipe_layers_sha256(const char *root, size_t *out_removed) +{ + *out_removed = 0; + char dir_path[STORE_PATH_MAX]; + int n = snprintf(dir_path, sizeof(dir_path), "%s/layers/sha256", root); + if (n < 0 || (size_t) n >= sizeof(dir_path)) { + errno = ENAMETOOLONG; + return -1; + } + DIR *d = opendir(dir_path); + if (!d) { + if (errno == ENOENT) + return 0; + return -1; + } + int rc = 0; + struct dirent *de; + while ((de = readdir(d))) { + if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) + continue; + char child[STORE_PATH_MAX]; + int cn = snprintf(child, sizeof(child), "%s/%s", dir_path, de->d_name); + if (cn < 0 || (size_t) cn >= sizeof(child)) { + errno = ENAMETOOLONG; + rc = -1; + break; + } + if (layer_stage_rm(child) < 0) { + rc = -1; + break; + } + (*out_removed)++; + } + closedir(d); + return rc; +} + +/* Write /layers/.schema atomically. The body is materialized into a + * pid + counter-suffixed tmp file, fsynced, and renamed into place. The + * caller must hold flock(/index.json.lock, LOCK_EX) so two openers + * cannot race the rename. Returns 0 on success, -1 with errno preserved + * on any IO failure (the tmp file is removed before returning). + */ +static int write_layer_schema_v2(const char *root) +{ + char path[STORE_PATH_MAX]; + int n = snprintf(path, sizeof(path), "%s/%s", root, LAYER_SCHEMA_REL_PATH); + if (n < 0 || (size_t) n >= sizeof(path)) { + errno = ENAMETOOLONG; + return -1; + } + char tmp[STORE_PATH_MAX]; + n = snprintf(tmp, sizeof(tmp), "%s.tmp-%d-%lu", path, (int) getpid(), + layer_schema_seq()); + if (n < 0 || (size_t) n >= sizeof(tmp)) { + errno = ENAMETOOLONG; + return -1; + } + int fd = open(tmp, O_WRONLY | O_CREAT | O_EXCL, 0644); + if (fd < 0) + return -1; + size_t body_len = sizeof(LAYER_SCHEMA_V2_BODY) - 1; + if (write(fd, LAYER_SCHEMA_V2_BODY, body_len) != (ssize_t) body_len) { + int saved = errno; + close(fd); + unlink(tmp); + errno = saved; + return -1; + } + if (fsync(fd) < 0) { + int saved = errno; + close(fd); + unlink(tmp); + errno = saved; + return -1; + } + if (close(fd) < 0) { + int saved = errno; + unlink(tmp); + errno = saved; + return -1; + } + if (rename(tmp, path) < 0) { + int saved = errno; + unlink(tmp); + errno = saved; + return -1; + } + return 0; +} + +static int ensure_layer_schema_marker(const char *root) +{ + /* First probe is lock-free so the marker-present fast path does not + * pay the flock cost on every open. */ + int version = 0; + const char *reason = NULL; + int rc = read_layer_schema_version(root, &version, &reason); + if (rc == 0) { + if (version == LAYER_SCHEMA_VERSION_CURRENT) + return 0; + fprintf(stderr, + "elfuse oci: unsupported layers schema version %d at %s; " + "this build understands up to %d\n", + version, root, LAYER_SCHEMA_VERSION_CURRENT); + errno = EINVAL; + return -1; + } + if (errno != ENOENT) { + int saved = errno; + fprintf(stderr, "elfuse oci: layers/.schema unreadable at %s: %s\n", + root, reason ? reason : "unknown error"); + errno = saved; + return -1; + } + + /* Marker absent. ELFUSE_OCI_NO_MIGRATE leaves the store untouched so + * a downgrade test or recovery workflow can inspect any pre-existing + * v1 entries without the daemon helpfully rewriting state. */ + const char *no_migrate = getenv(NO_MIGRATE_ENV); + if (no_migrate && *no_migrate) + return 0; + + const char *lock_err = NULL; + int lock_fd = acquire_index_lock(root, &lock_err); + if (lock_fd < 0) { + int saved = errno; + fprintf(stderr, "elfuse oci: %s\n", + lock_err ? lock_err : "failed to acquire index.json.lock"); + errno = saved; + return -1; + } + + /* Re-stat under hold: a racing opener may already have migrated. */ + rc = read_layer_schema_version(root, &version, &reason); + if (rc == 0) { + close(lock_fd); + if (version == LAYER_SCHEMA_VERSION_CURRENT) + return 0; + fprintf(stderr, + "elfuse oci: unsupported layers schema version %d at %s; " + "this build understands up to %d\n", + version, root, LAYER_SCHEMA_VERSION_CURRENT); + errno = EINVAL; + return -1; + } + if (errno != ENOENT) { + int saved = errno; + close(lock_fd); + fprintf(stderr, "elfuse oci: layers/.schema unreadable at %s: %s\n", + root, reason ? reason : "unknown error"); + errno = saved; + return -1; + } + + /* Marker still absent under hold: wipe pre-existing v1 entries and + * publish the v2 marker. */ + size_t removed = 0; + if (wipe_layers_sha256(root, &removed) < 0) { + int saved = errno; + close(lock_fd); + fprintf(stderr, + "elfuse oci: failed to migrate layer cache schema at %s: " + "wipe partial (%zu entr%s removed before error)\n", + root, removed, removed == 1 ? "y" : "ies"); + errno = saved; + return -1; + } + if (removed > 0) { + fprintf(stderr, + "elfuse oci: layer cache schema v1 detected; cleared %zu " + "entr%s from %s/layers/sha256 to migrate to v2 " + "(raw + ChainID stack)\n", + removed, removed == 1 ? "y" : "ies", root); + } + + if (write_layer_schema_v2(root) < 0) { + int saved = errno; + close(lock_fd); + fprintf(stderr, "elfuse oci: failed to write layers/.schema at %s\n", + root); + errno = saved; + return -1; + } + close(lock_fd); + return 0; +} diff --git a/src/oci/store.h b/src/oci/store.h new file mode 100644 index 0000000..805b628 --- /dev/null +++ b/src/oci/store.h @@ -0,0 +1,532 @@ +/* Local OCI image store: blobs + tag-to-digest pinning + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + * + * Wraps the slice-2 content-addressable blob store with a tag-to-digest pin + * table so that elfuse oci pull / inspect can reproduce a pull by name. The + * on-disk layout under follows the OCI image-layout spec (v1.0.0) so + * external tools (skopeo, umoci, crane) can consume the store directly: + * + * oci-layout OCI image-layout 1.0.0 marker + * index.json OCI image-index of all pins + * index.json.lock flock target for serializing + * writers blobs// finalized blob (immutable) + * tmp/blob---XXXXXX in-flight blob staging + * + * Each pin is one descriptor in index.json's manifests[] array. The pin name + * (canonical "/:") is stored in the descriptor's + * org.opencontainers.image.ref.name annotation. The descriptor's mediaType, + * digest, and size mirror the manifest blob in blobs//. Writers + * serialize through flock(/index.json.lock, LOCK_EX) and publish via + * tmp + rename so a concurrent reader always observes a complete document. + * Readers parse the snapshot lock-free: rename is atomic and cJSON consumes + * the file in one open + read. + * + * Pre-C2.2 stores used a refs/// flat-file layout + * instead of index.json. oci_store_open auto-migrates such stores: refs/ is + * scanned recursively, every well-formed pin file becomes a manifests[] + * descriptor in a freshly written index.json, and refs/ is kept on disk for + * one release so a downgrade still finds the legacy data. Pins whose + * manifest blob is missing from blobs/ are skipped with a warning rather + * than aborting the open. Migration is suppressed when the environment + * variable ELFUSE_OCI_NO_MIGRATE is set to any non-empty value, which lets + * a downgrade test or recovery workflow inspect the legacy layout. + * + * Phase 1 keeps as a plain directory. The sparse case-sensitive APFS + * volume bootstrap (oci-roadmap Q1) is a Phase 2 concern; the volume mount + * point will sit at the same default path so this API does not change. + */ + +#pragma once + +#include +#include +#include + +#include "blob-store.h" +#include "digest-set.h" +#include "ref.h" + +typedef struct oci_store oci_store_t; + +/* One pin entry produced by oci_store_list_refs. name is the canonical + * "/:" string captured from the descriptor's + * org.opencontainers.image.ref.name annotation; digest is the manifest + * digest in ":" form. Both fields are heap-allocated and owned + * by the enclosing oci_pin_list_t. + */ +typedef struct { + char *name; + char *digest; +} oci_pin_entry_t; + +typedef struct { + oci_pin_entry_t *items; + size_t count; +} oci_pin_list_t; + +/* Open or create the store rooted at `root`. Ensures blobs//, tmp/, + * and the OCI image-layout 1.0.0 marker exist. Marker writes are idempotent: + * a pre-existing oci-layout file is never rewritten so a third party that + * bumped the imageLayoutVersion is preserved. The index.json file is not + * materialized until the first oci_store_put_ref so an empty store stays + * literally empty on disk. + * + * If the root contains a pre-C2.2 refs/ tree but no index.json, this call + * also rebuilds index.json from refs/ contents under flock(index.json.lock, + * LOCK_EX) so a concurrent put_ref cannot race the migration. refs/ is + * preserved on disk for a downgrade fallback. Set ELFUSE_OCI_NO_MIGRATE to + * any non-empty value to skip the migration on this open. Migration failure + * is propagated through this call (NULL return with errno preserved); per-pin + * issues (missing blob, malformed pin file) are logged to stderr and skipped + * without failing the open. + * + * The Plan 3 C3.3b layer cache schema marker at /layers/.schema is + * also written or validated here. A v1 store (no marker but an existing + * layers/sha256/ subtree from C3.2) has its layers/sha256/ children wiped + * before the v2 marker is published; the wipe is scoped to layers/sha256/ + * only so blobs/, images/, refs/, index.json, and layers/.staging/ are + * never touched. A marker whose schemaVersion is unknown to this build + * (forward incompatibility, corruption, or an experimental schema) is + * fatal and returns NULL with errno=EINVAL. ELFUSE_OCI_NO_MIGRATE gates + * this migration too: when set and the marker is absent, the wipe and + * the marker write are both skipped so a downgrade test can inspect any + * v1 entries on disk. + * + * Returns NULL on failure with errno preserved. + */ +oci_store_t *oci_store_open(const char *root); + +/* Close the store handle. Does not delete on-disk state. Safe on NULL. */ +void oci_store_close(oci_store_t *s); + +/* Return the store root path. The returned pointer is owned by the store and + * is valid until oci_store_close. + */ +const char *oci_store_root(const oci_store_t *s); + +/* Return the underlying blob store handle. The returned pointer is owned by + * the store; do not close it directly. + */ +oci_blob_store_t *oci_store_blobs(oci_store_t *s); + +/* Return the default store root for the current user. macOS XDG-ish: + * $XDG_DATA_HOME/elfuse/store when XDG_DATA_HOME is set + * $HOME/Library/Application Support/elfuse/store otherwise + * Returns a heap-allocated string the caller must free, or NULL on env miss + * (errno=ENOENT) or oom (errno=ENOMEM). + */ +char *oci_store_default_root(void); + +/* Upsert a tag-to-digest pin for ref. ref->tag must be set; digest-only refs + * are self-pinning by their digest field and putting a pin for them is an + * EINVAL. digest_str is the canonical ":" form of the manifest + * digest captured at pull time; the manifest blob must already be present + * under /blobs// so the descriptor's size and mediaType + * can be derived from the on-disk blob (mediaType is read from the JSON + * body and inferred from structure when absent). + * + * Concurrency: the write is serialized by flock(/index.json.lock, + * LOCK_EX) and published via tmp + rename so a concurrent reader never + * observes a partial index.json. Re-pinning the same canonical name + * replaces the existing descriptor in place rather than appending a + * duplicate entry. + * + * Returns 0 on success, -1 with errno preserved and *err_msg (when non-NULL) + * pointing at a static description on failure. + */ +int oci_store_put_ref(oci_store_t *s, + const oci_ref_t *ref, + const char *digest_str, + const char **err_msg); + +/* Read the pinned manifest digest for ref. ref->tag must be set; digest-only + * refs are self-pinning and trigger EINVAL. On hit returns 0 and writes a + * heap-allocated ":" string into *out_digest (caller frees). On + * miss returns -1 with errno=ENOENT and *out_digest=NULL. Other IO errors + * return -1 with errno preserved. *err_msg (when non-NULL) is populated on + * any non-success path. + * + * The read is lock-free: tmp + rename in oci_store_put_ref makes the + * index.json switch atomic so a single open + read snapshots a complete + * document. + */ +int oci_store_get_ref(oci_store_t *s, + const oci_ref_t *ref, + char **out_digest, + const char **err_msg); + +/* Enumerate every pin currently recorded in index.json. On success returns + * 0 and populates *out with a heap-allocated array of (name, digest) + * entries; an empty store yields count == 0 and items == NULL. The caller + * releases the result via oci_pin_list_free. Missing index.json is treated + * as an empty store, not as an error. Other IO or schema errors return -1 + * with errno preserved and *err_msg (when non-NULL) populated. + * + * The order of returned entries matches the order in index.json; callers + * that need a stable sort must impose it themselves. + */ +int oci_store_list_refs(oci_store_t *s, + oci_pin_list_t *out, + const char **err_msg); + +/* Release every name / digest string in list and zero the struct. Safe on + * a zero-initialised list and on NULL. + */ +void oci_pin_list_free(oci_pin_list_t *list); + +/* Mark phase of the Plan 1 garbage collector: enumerate every blob + * digest still reachable from on-disk state and accumulate them in + * *out. Two sources are walked: + * + * 1. Pins in index.json. For each pin's manifest digest, the + * manifest blob is read and parsed; the config descriptor and + * every layer descriptor are added to the set. If the pinned + * blob is an OCI image-index instead of an image manifest, every + * sub-manifest descriptor digest is added, and for each + * sub-manifest whose blob is on disk the walk recurses into its + * config + layers. Sub-manifests not on disk (the multi-arch + * case where only one platform was fetched) are still added to + * the keep set so a sweep does not delete a sub-manifest blob + * that did materialise locally. + * + * 2. Unpacked image trees under /images/sha256-/. + * Each tree's .elfuse-origin.json is parsed; its manifest_digest + * drives the same manifest/config/layer expansion as a pin. + * + * Failure policy is fail-fast on anything that would let prune later + * delete a reachable blob: a missing manifest blob, an unparseable + * manifest, a missing or malformed .elfuse-origin.json, or a missing + * image-config blob all return -1 with *err populated so the operator + * can repair the store before retrying. A missing + * /images/ tree is treated as the fresh-store case + * (count == 0 contribution from that source) and not an error. + * + * volume_root may be NULL, in which case the unpacked-tree walk is + * skipped entirely. The pin walk runs unconditionally. + * + * Returns 0 on success; on failure returns -1 with errno preserved + * and *err (when non-NULL) pointing at a static description. On + * failure *out is left in a freed-empty state. + */ +int oci_store_collect_roots(oci_store_t *s, + oci_digest_set_t *out, + const char *volume_root, + const char **err); + +/* Plan 3 C3.3d mark walker for the layer + stack caches. Computes the + * reachable set of layer raw-cache and stack-cache entries from the + * same two sources oci_store_collect_roots reads: pins in index.json + * (resolved through one image-index level into a linux/arm64 + * sub-manifest as needed) and unpacked sysroots under + * /images/. For every image the walker can resolve: + * + * - Every layer's diff_id (from rootfs.diff_ids in the image-config + * blob for pinned images, or directly from .elfuse-origin.json's + * layer_diffids for unpacked sysroots) is added to *out_diff_ids. + * These name the entries under /layers///. + * + * - Every prefix ChainID through the layer list is added to + * *out_chain_ids. ChainID(L0) == DiffID(L0); ChainID(Li) == + * sha256(" "). oci_unpack writes one stack + * snapshot per prefix during the apply loop (src/oci/unpack.c + * line 1063), so a prune sweep must keep every prefix that maps + * to a reachable image, not only the terminating chain. These + * name the entries under /layers/stacks///. + * + * Both sets are populated by the same walker pass so they stay + * consistent across the two sources. + * + * Failure policy mirrors oci_store_collect_roots: a missing or + * unparseable image-config blob for a pinned image-manifest, a + * malformed origin sidecar, or a chainid_compute failure aborts the + * mark phase with -1 / errno set so prune cannot proceed to delete + * reachable cache entries. Soft cases (image-index pins whose + * linux/arm64 sub-manifest blob is not on disk, image-index pins + * with no linux/arm64 entry at all, a missing /images/ + * directory) contribute nothing without surfacing as errors so a + * multi-arch operator stays unblocked. + * + * volume_root may be NULL, in which case only the pin source is + * walked. On entry *out_diff_ids and *out_chain_ids are + * initialised so the caller may pass uninitialised structs. On + * failure both sets are freed back to empty. + */ +int oci_store_collect_layer_roots(oci_store_t *s, + oci_digest_set_t *out_diff_ids, + oci_digest_set_t *out_chain_ids, + const char *volume_root, + const char **err); + +/* Options + stats for oci_store_prune. Output fields are filled + * regardless of dry-run vs commit so callers can render a uniform + * report from the same struct. + * + * older_than_sec and keep_bytes shape which dangling entries survive + * the sweep. Both default to 0 with the documented meaning of "no + * filter" so the C1.3 behaviour (every dangling blob is pruned) is + * preserved when the caller does not opt in. + * + * older_than_sec > 0 vetoes per-entry: a dangling entry whose mtime + * is younger than (now - older_than_sec) is reported in the + * skipped_* family and left on disk. This is the grace window for a + * half-completed pull whose blob has been committed but whose + * put_ref has not landed yet, or for a layer/stack cache entry an + * unpack is still publishing. + * + * keep_bytes > 0 enforces a per-family LRU budget over the + * candidates that survive the older-than veto: candidates are + * sorted by mtime ascending and walked newest-first, the newest + * entries whose cumulative size fits under keep_bytes are + * reclassified as skipped, the rest stay pruned. A single + * candidate that does not fit the budget terminates the keep walk + * so older candidates are always evicted first even when an older + * entry would fit alone. The budget is applied independently to + * each cache family (blobs, layers, stacks) so a fat blob cannot + * crowd out a layer-cache eviction. + * + * The two filters compose by running older-than first and keep-bytes + * second per family: a transient just-pulled entry never enters the + * LRU budget computation so the grace window holds. + * + * The Plan 3 C3.3d extension introduces per-layer and per-stack + * counters that behave just like the blob counters: kept entries + * survive the mark phase, pruned entries are unreachable and (when + * commit is true) recursively removed, skipped entries were + * unreachable but the filters spared them. layer entries live under + * /layers/// and stack entries under + * /layers/stacks///; both are directory trees so + * the *_bytes counters are the recursive st_size sum of every + * regular file beneath the entry directory. + */ +typedef struct { + /* Inputs */ + bool commit; /* false (default) = dry-run; true = unlink */ + const char *volume_root; /* NULL = pin-only walk (see collect_roots) */ + uint64_t older_than_sec; /* 0 = no mtime filter */ + uint64_t keep_bytes; /* 0 = no size budget (no filter) */ + + /* Outputs - blobs */ + size_t kept_blobs; + size_t pruned_blobs; + uint64_t pruned_bytes; + size_t + skipped_blobs; /* dangling but spared by older_than_sec or keep_bytes */ + uint64_t skipped_bytes; /* sum of st_size for skipped_blobs */ + + /* Outputs - per-layer raw cache entries (C3.3d) */ + size_t kept_layers; + size_t pruned_layers; + uint64_t pruned_layer_bytes; + size_t skipped_layers; + uint64_t skipped_layer_bytes; + + /* Outputs - ChainID-keyed stack cache entries (C3.3d) */ + size_t kept_stacks; + size_t pruned_stacks; + uint64_t pruned_stack_bytes; + size_t skipped_stacks; + uint64_t skipped_stack_bytes; +} oci_store_prune_options_t; + +/* Garbage-collect dangling entries from three cache families under + * the store root. The mark phase pairs oci_store_collect_roots (blob + * keep set) with oci_store_collect_layer_roots (diff_id and ChainID + * keep sets) so all three sweeps share a single consistent snapshot + * of pins + unpacked sysroots: + * + * - /blobs// regular files (Plan 1) + * - /layers/// raw layer dirs (C3.3d) + * - /layers/stacks/// stack snapshot dirs (C3.3d) + * + * The sweep phase walks each family in turn, comparing every entry's + * : against its family's keep set; entries not reachable + * are counted into the family's pruned_* counters and (when + * opts->commit is true) removed (unlink for blobs, recursive rm for + * layer / stack directory trees). pruned_layer_bytes and + * pruned_stack_bytes hold the recursive sum of regular-file st_size + * beneath each removed directory. + * + * The whole operation runs under flock(/index.json.lock, + * LOCK_EX), which is the same write lock oci_store_put_ref holds. + * That bounds the race where a concurrent pull writes a new pin + * after the mark phase already snapshotted index.json. Layer / stack + * cache writers (oci_unpack / oci_rebuild_cache) do NOT take this + * lock so they may publish new cache entries concurrent with prune; + * those entries are reachable from their image's pin or unpacked + * sysroot, both of which the mark phase already captured, so their + * diff_id / ChainID is in the keep set even if the directory did + * not yet exist when sweep enumerated. The remaining window is the + * mid-pull case (a layer extracted before put_ref lands) which + * matches the C1.3 blob-mid-pull semantic: the operator retries. + * + * On entry every counter in opts is reset to zero so the caller does + * not have to memset between invocations. Entries whose name is not + * a valid lowercase hex digest for the enclosing algorithm subdir, + * non-directory entries inside layers/, and dotfiles are all skipped + * without surfacing as errors so any foreign state under the store + * root stays untouched. + * + * When opts->older_than_sec or opts->keep_bytes is set, each family + * gathers dangling candidates first and then applies the filters + * (older-than veto, then keep-bytes LRU budget) before any removal. + * The keep-bytes budget is independent per family: a 100 MiB budget + * means "keep up to 100 MiB of newest dangling blobs AND up to + * 100 MiB of newest dangling layer trees AND up to 100 MiB of + * newest dangling stack trees", so a fat blob cannot crowd a layer + * eviction off the budget. Candidates spared by either filter + * contribute to skipped_* rather than pruned_* so the caller can + * render a three-way kept/pruned/skipped split per family. + * + * Returns 0 on success and -1 on failure with errno preserved and + * *err (when non-NULL) populated. Mark-phase failure is fatal and + * aborts before any removal so a corrupt or torn manifest / config + * blob cannot cause prune to delete reachable entries. + */ +int oci_store_prune(oci_store_t *s, + oci_store_prune_options_t *opts, + const char **err); + +/* Plan 3 C3.2: per-layer unpack snapshot cache. + * + * Layer caches live under /layers/// in the same content- + * addressed shape as /blobs//. Each cache directory holds a + * snapshot of the unpack stage_dir state immediately after applying that + * layer's tar payload. clonefile(2) populates and consumes the snapshots so + * the cache and the live unpack stage must live on the same APFS volume; an + * EXDEV during snapshot is propagated as a hard error rather than silently + * falling back to a copy. + * + * Cache semantics are CUMULATIVE: the directory at layers/sha256// + * holds the stage_dir state assembled by the unpacker WHEN this layer was + * applied, which means it includes every prior layer's contribution along + * with the current layer. A second unpack of the same image short-circuits + * the extract loop entirely. Cross-image dedup (two images sharing a base + * layer prefix but diverging upstream) is NOT yet correct under this scheme; + * Plan 3 C3.3 introduces raw-tar staging + clonefile-stack assembly to fix + * the cross-image case. C3.2 lands the directory layout, the path helpers, + * and the per-image fast path the C3.3 rewrite will build on. + * + * C3.2 deliberately does NOT extend oci_store_collect_roots / oci_store_prune + * to walk layers/. The cache grows monotonically until C3.5's + * `oci image rebuild-cache` (or C3.3's prune work) consumes it. Skipping the + * keep-set walk now keeps this commit focused on the layout invariants. + * + * No refcount sidecar is written. Reachability is recomputed at GC time from + * each manifest's image-config rootfs.diff_ids list, mirroring how blobs/ + * reachability is recomputed by oci_store_collect_roots. + * + * Concurrency: cache_has is a single stat(2) and is inherently racy with + * concurrent writers, but the worst outcome is a redundant extract. + * oci_store_layer_commit publishes via rename(2) (atomic) and treats + * EEXIST / ENOTEMPTY at the destination as a benign loss to a racing + * winner: the loser's staging directory is removed and 0 is returned so the + * caller can proceed as though the entry was already on disk. No store-wide + * lock is required. + */ + +/* Probe whether /layers/// exists. diff_id is in canonical + * ":" form. Returns 1 (present, is a directory), 0 (absent), or + * -1 with errno preserved on any unexpected IO error. A malformed diff_id + * returns -1 with errno=EINVAL. + */ +int oci_store_layer_has(oci_store_t *s, const char *diff_id); + +/* Compose /layers/// for diff_id into out. Trailing slash + * included so a downstream strcat(child) composes cleanly. Pure path + * computation; does not stat or mkdir. Returns 0 on success, -1 with errno + * EINVAL on malformed diff_id, ENAMETOOLONG on buffer overflow. + */ +int oci_store_layer_resolve(oci_store_t *s, + const char *diff_id, + char *out, + size_t cap); + +/* Compose /layers/.staging/-- for diff_id into out. + * The path is unique per call; the directory is NOT created (clonefile(2) + * creates it as a side effect). Returns 0 on success, -1 with errno EINVAL + * on malformed diff_id, ENAMETOOLONG on overflow, or other errno values + * propagated from getentropy(2). + */ +int oci_store_layer_stage_path(oci_store_t *s, + const char *diff_id, + char *out, + size_t cap); + +/* Atomically publish a populated staging directory as the layer cache entry + * for diff_id via rename(stage_path, /layers///). If the + * destination already exists (EEXIST / ENOTEMPTY: a concurrent writer landed + * the same entry first) the staging directory is removed and 0 is returned. + * Any other failure returns -1 with errno preserved and *err (when non-NULL) + * populated; the staging directory is left in place so the caller can retry + * or inspect it. + */ +int oci_store_layer_commit(oci_store_t *s, + const char *stage_path, + const char *diff_id, + const char **err); + +/* Plan 3 C3.3c: ChainID-keyed assembled-stack cache. + * + * The stack cache lives under /layers/stacks/// in the same + * content-addressed shape as the per-layer raw cache. Each entry holds an + * assembled cumulative stage_dir state through some prefix of an image's + * layer list, keyed by the OCI ChainID for the terminating layer (see + * src/oci/digest.h::oci_chainid_compute). Cross-image dedup works because + * any two images that share the same ordered layer prefix produce the same + * ChainID for that prefix; the longest-prefix match short-circuits the + * per-layer assembly during oci_unpack. + * + * Staging shares the /layers/.staging/ directory with the per-layer + * raw cache. Stack stage paths are prefixed with "stack-" so a debug walk + * of .staging/ can tell the two artifact families apart at a glance. The + * commit destination is what disambiguates the two on disk. + * + * Concurrency mirrors the layer-cache APIs: stack_has is a single stat(2) + * (racy with concurrent writers; worst case is one redundant assembly), + * stack_commit publishes via rename(2) and treats EEXIST / ENOTEMPTY as a + * benign loss to the racing winner with the staging tree torn down. No + * store-wide lock is required. + */ + +/* Probe whether /layers/stacks/// exists. chain_id is in + * canonical ":" form. Returns 1 (present, is a directory), 0 + * (absent), or -1 with errno preserved on any unexpected IO error. A + * malformed chain_id returns -1 with errno=EINVAL. + */ +int oci_store_stack_has(oci_store_t *s, const char *chain_id); + +/* Compose /layers/stacks/// for chain_id into out. + * Trailing slash included so a downstream strcat(child) composes cleanly. + * Pure path computation; does not stat or mkdir. Returns 0 on success, -1 + * with errno EINVAL on malformed chain_id, ENAMETOOLONG on buffer overflow. + */ +int oci_store_stack_resolve(oci_store_t *s, + const char *chain_id, + char *out, + size_t cap); + +/* Compose /layers/.staging/stack--- for chain_id + * into out. The path is unique per call; the directory is NOT created + * (clonefile(2) creates it as a side effect). Returns 0 on success, -1 with + * errno EINVAL on malformed chain_id, ENAMETOOLONG on overflow, or other + * errno values propagated from getentropy(2). + */ +int oci_store_stack_stage_path(oci_store_t *s, + const char *chain_id, + char *out, + size_t cap); + +/* Atomically publish a populated staging directory as the stack cache entry + * for chain_id via rename(stage_path, /layers/stacks///). + * If the destination already exists (EEXIST / ENOTEMPTY: a concurrent writer + * landed the same entry first) the staging directory is removed and 0 is + * returned. Any other failure returns -1 with errno preserved and *err + * (when non-NULL) populated; the staging directory is left in place so the + * caller can retry or inspect it. + */ +int oci_store_stack_commit(oci_store_t *s, + const char *stage_path, + const char *chain_id, + const char **err); diff --git a/src/oci/tar.c b/src/oci/tar.c new file mode 100644 index 0000000..6dbf22b --- /dev/null +++ b/src/oci/tar.c @@ -0,0 +1,759 @@ +/* OCI tar reader implementation + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include + +#include "oci/tar.h" + +#define TAR_BLOCK_SIZE 512 +#define TAR_PATH_MAX 4096 +#define TAR_LINKNAME_MAX 4096 +#define TAR_MAX_LONG_RECORD (TAR_PATH_MAX * 4) + +/* POSIX 1003.1-1990 ustar header offsets and lengths. The reader does + * not synthesize the struct; it indexes the raw block directly to + * avoid any padding/alignment ambiguity. + */ +#define OFF_NAME 0 +#define OFF_MODE 100 +#define OFF_UID 108 +#define OFF_GID 116 +#define OFF_SIZE 124 +#define OFF_MTIME 136 +#define OFF_CHKSUM 148 +#define OFF_TYPEFLAG 156 +#define OFF_LINKNAME 157 +#define OFF_MAGIC 257 +#define OFF_VERSION 263 +#define OFF_PREFIX 345 + +#define LEN_NAME 100 +#define LEN_MODE 8 +#define LEN_UID 8 +#define LEN_GID 8 +#define LEN_SIZE 12 +#define LEN_MTIME 12 +#define LEN_CHKSUM 8 +#define LEN_LINKNAME 100 +#define LEN_MAGIC 6 +#define LEN_VERSION 2 +#define LEN_PREFIX 155 + +struct oci_tar_reader { + oci_tar_read_fn read_fn; + void *ctx; + + /* Current entry buffers. path and linkname are reused across + * entries; they grow on demand up to TAR_PATH_MAX / TAR_LINKNAME_MAX. + */ + char *path; + size_t path_cap; + char *linkname; + size_t linkname_cap; + + /* GNU long-name pending payload. When the previous block was an 'L' + * or 'K' typeflag, the next non-extension entry consumes this + * buffer instead of the in-header name/linkname. + */ + char *pending_long_name; + char *pending_long_link; + + /* Payload tracking for the current entry. bytes_remaining counts + * unread payload bytes; padding_remaining is the trailing zero + * padding the reader must consume before the next header. + */ + uint64_t bytes_remaining; + uint32_t padding_remaining; + + /* Two consecutive zero blocks terminate the archive. */ + bool saw_first_zero_block; +}; + +static int read_full(oci_tar_reader_t *r, void *buf, size_t want) +{ + uint8_t *p = buf; + size_t got = 0; + while (got < want) { + ssize_t n = r->read_fn(r->ctx, p + got, want - got); + if (n < 0) + return -1; + if (n == 0) + return 0; + got += (size_t) n; + } + return 1; +} + +static int fill_block(oci_tar_reader_t *r, uint8_t *block) +{ + return read_full(r, block, TAR_BLOCK_SIZE); +} + +static int discard_bytes(oci_tar_reader_t *r, uint64_t bytes) +{ + uint8_t scratch[TAR_BLOCK_SIZE]; + while (bytes > 0) { + size_t take = bytes > TAR_BLOCK_SIZE ? TAR_BLOCK_SIZE : (size_t) bytes; + int rc = read_full(r, scratch, take); + if (rc <= 0) + return -1; + bytes -= take; + } + return 0; +} + +static bool is_zero_block(const uint8_t *block) +{ + for (size_t i = 0; i < TAR_BLOCK_SIZE; i++) + if (block[i] != 0) + return false; + return true; +} + +/* Parse a NUL- or space-terminated octal field. Returns -1 on + * unparseable input. Empty (all-zero) fields read as 0, which is the + * tar convention for fields the writer left unset. + */ +static int parse_octal(const uint8_t *field, size_t len, uint64_t *out) +{ + /* GNU base-256 extension: high bit set in the first byte means the + * remaining bytes are a big-endian binary integer. tar uses this + * for sizes that overflow the 11-octal-digit ceiling (~8 GiB). + */ + if (field[0] & 0x80) { + uint64_t v = 0; + for (size_t i = 1; i < len; i++) { + if (v > (UINT64_MAX >> 8)) + return -1; + v = (v << 8) | field[i]; + } + /* The first byte's low 7 bits also belong to the integer. */ + v |= (uint64_t) (field[0] & 0x7f) << ((len - 1) * 8); + *out = v; + return 0; + } + + uint64_t v = 0; + size_t i = 0; + while (i < len && (field[i] == ' ' || field[i] == '\0')) + i++; + if (i == len) { + *out = 0; + return 0; + } + for (; i < len; i++) { + uint8_t c = field[i]; + if (c == ' ' || c == '\0') + break; + if (c < '0' || c > '7') + return -1; + if (v > (UINT64_MAX >> 3)) + return -1; + v = (v << 3) | (uint64_t) (c - '0'); + } + *out = v; + return 0; +} + +static bool verify_chksum(const uint8_t *block) +{ + /* The chksum field itself is included in the sum as 8 spaces. */ + uint64_t parsed = 0; + if (parse_octal(block + OFF_CHKSUM, LEN_CHKSUM, &parsed) < 0) + return false; + uint32_t expected = (uint32_t) parsed; + + uint32_t sum_unsigned = 0; + int32_t sum_signed = 0; + for (size_t i = 0; i < TAR_BLOCK_SIZE; i++) { + uint8_t b = block[i]; + if (i >= OFF_CHKSUM && i < OFF_CHKSUM + LEN_CHKSUM) + b = ' '; + sum_unsigned += b; + sum_signed += (int8_t) b; + } + /* Historical tar implementations used signed bytes when computing + * the checksum; accept either to interop with both. + */ + return sum_unsigned == expected || (uint32_t) sum_signed == expected; +} + +static bool is_ustar_magic(const uint8_t *block) +{ + /* POSIX "ustar\0" version "00", or GNU "ustar \0" (space-space-NUL + * starting at OFF_MAGIC). Both shapes are common in real images. + */ + if (memcmp(block + OFF_MAGIC, "ustar", 5) != 0) + return false; + return true; +} + +static int copy_field_string(const uint8_t *field, + size_t field_len, + char *out, + size_t out_cap) +{ + /* Tar fields are NUL-terminated only when shorter than the slot. + * Truncation is normal: copy up to the first NUL (or field_len), + * then NUL-terminate the destination. + */ + size_t n = 0; + while (n < field_len && field[n] != '\0') + n++; + if (n + 1 > out_cap) + return -1; + memcpy(out, field, n); + out[n] = '\0'; + return 0; +} + +static int build_path_from_header(const uint8_t *block, + char *out, + size_t out_cap) +{ + char name[LEN_NAME + 1]; + char prefix[LEN_PREFIX + 1]; + if (copy_field_string(block + OFF_NAME, LEN_NAME, name, sizeof(name)) < 0) + return -1; + if (copy_field_string(block + OFF_PREFIX, LEN_PREFIX, prefix, + sizeof(prefix)) < 0) + return -1; + if (prefix[0] == '\0') { + if (strlen(name) + 1 > out_cap) + return -1; + memcpy(out, name, strlen(name) + 1); + return 0; + } + /* ustar joins prefix + '/' + name. */ + size_t pn = strlen(prefix); + size_t nn = strlen(name); + if (pn + 1 + nn + 1 > out_cap) + return -1; + memcpy(out, prefix, pn); + out[pn] = '/'; + memcpy(out + pn + 1, name, nn); + out[pn + 1 + nn] = '\0'; + return 0; +} + +static int ensure_capacity(char **buf, size_t *cap, size_t want) +{ + if (*cap >= want) + return 0; + size_t new_cap = *cap == 0 ? 256 : *cap; + while (new_cap < want) + new_cap *= 2; + char *grown = realloc(*buf, new_cap); + if (!grown) + return -1; + *buf = grown; + *cap = new_cap; + return 0; +} + +static const char *strip_trailing_slash(char *s) +{ + size_t n = strlen(s); + while (n > 1 && s[n - 1] == '/') + s[--n] = '\0'; + return s; +} + +static void apply_whiteout_flags(oci_tar_entry_t *e) +{ + e->is_whiteout = false; + e->is_opaque_whiteout = false; + if (!e->path) + return; + const char *slash = strrchr(e->path, '/'); + const char *base = slash ? slash + 1 : e->path; + if (strcmp(base, ".wh..wh..opq") == 0) + e->is_opaque_whiteout = true; + else if (strncmp(base, ".wh.", 4) == 0) + e->is_whiteout = true; +} + +oci_tar_reader_t *oci_tar_reader_new(oci_tar_read_fn read_fn, void *ctx) +{ + if (!read_fn) + return NULL; + oci_tar_reader_t *r = calloc(1, sizeof(*r)); + if (!r) + return NULL; + r->read_fn = read_fn; + r->ctx = ctx; + return r; +} + +void oci_tar_reader_free(oci_tar_reader_t *r) +{ + if (!r) + return; + free(r->path); + free(r->linkname); + free(r->pending_long_name); + free(r->pending_long_link); + free(r); +} + +/* Consume a GNU 'L' or 'K' typeflag payload into a freshly allocated + * buffer that will be claimed by the next non-extension entry. + */ +static int consume_long_record(oci_tar_reader_t *r, + uint64_t size, + char **out, + const char **err) +{ + if (size == 0 || size > TAR_MAX_LONG_RECORD) { + *err = "tar GNU long-name record out of bounds"; + errno = ENAMETOOLONG; + return -1; + } + char *buf = malloc((size_t) size + 1); + if (!buf) { + *err = "tar long-name buffer allocation failed"; + errno = ENOMEM; + return -1; + } + int rc = read_full(r, buf, (size_t) size); + if (rc <= 0) { + free(buf); + *err = "tar GNU long-name record truncated"; + errno = EIO; + return -1; + } + /* Padding alignment is computed against the on-wire record length, + * NOT the C-string length after stripping trailing NULs. Doing the + * trim before discard_bytes would drift the source position by the + * trim count and misalign the next header read. + */ + uint64_t pad = (TAR_BLOCK_SIZE - (size % TAR_BLOCK_SIZE)) % TAR_BLOCK_SIZE; + if (pad > 0 && discard_bytes(r, pad) < 0) { + free(buf); + *err = "tar GNU long-name padding truncated"; + errno = EIO; + return -1; + } + /* GNU records are NUL-terminated on the wire, but defensively + * ensure the caller-visible string has a terminator regardless of + * the in-record byte layout. + */ + buf[size] = '\0'; + free(*out); + *out = buf; + return 0; +} + +/* Consume an 'x' (per-file) or 'g' (global) PAX extended-header + * payload. Per POSIX.1-2001 the payload is a stream of records of + * the form " =\n" where is the total byte + * count of the record including its own length digits and trailing + * newline. The unpack pipeline cares only about long names: `path` + * overrides the next entry's name and `linkpath` overrides its + * linkname. Both are promoted into the same pending buffers the GNU + * 'L'/'K' path uses so downstream code stays unaware of the format. + * + * Global ('g') records establish defaults for all subsequent + * entries; container builders almost never set path / linkpath + * defaults globally (the use case is local mtime / atime / uid + * defaults), so the implementation discards the payload bytes- + * correctly without parsing. Other PAX keys (size, mtime, atime, + * uid, gid, xattrs) are not tracked by the unpack pipeline and are + * silently ignored from per-file records as well. + */ +static int consume_pax_record(oci_tar_reader_t *r, + uint64_t size, + int is_global, + const char **err) +{ + if (size > TAR_MAX_LONG_RECORD) { + *err = "tar PAX record out of bounds"; + errno = ENAMETOOLONG; + return -1; + } + char *buf = NULL; + if (size > 0) { + buf = malloc((size_t) size + 1); + if (!buf) { + *err = "tar PAX buffer allocation failed"; + errno = ENOMEM; + return -1; + } + int rc = read_full(r, buf, (size_t) size); + if (rc <= 0) { + free(buf); + *err = "tar PAX record truncated"; + errno = EIO; + return -1; + } + buf[size] = '\0'; + } + /* Padding alignment is computed against the on-wire record length, + * matching consume_long_record so the next header read stays + * aligned even when size is not a multiple of TAR_BLOCK_SIZE. + */ + uint64_t pad = (TAR_BLOCK_SIZE - (size % TAR_BLOCK_SIZE)) % TAR_BLOCK_SIZE; + if (pad > 0 && discard_bytes(r, pad) < 0) { + free(buf); + *err = "tar PAX padding truncated"; + errno = EIO; + return -1; + } + + if (is_global || size == 0) { + free(buf); + return 0; + } + + char *p = buf; + char *end = buf + size; + while (p < end) { + char *space = memchr(p, ' ', (size_t) (end - p)); + if (!space) { + free(buf); + *err = "tar PAX record missing length terminator"; + errno = EINVAL; + return -1; + } + char *endp = NULL; + long len = strtol(p, &endp, 10); + if (endp != space || len <= 0) { + free(buf); + *err = "tar PAX record length unparseable"; + errno = EINVAL; + return -1; + } + char *record_end = p + len; + if (record_end > end || record_end[-1] != '\n') { + free(buf); + *err = "tar PAX record framing invalid"; + errno = EINVAL; + return -1; + } + char *kvp = space + 1; + char *eq = memchr(kvp, '=', (size_t) (record_end - 1 - kvp)); + if (eq) { + size_t key_len = (size_t) (eq - kvp); + const char *val = eq + 1; + size_t val_len = (size_t) (record_end - 1 - val); + char **slot = NULL; + if (key_len == 4 && memcmp(kvp, "path", 4) == 0) + slot = &r->pending_long_name; + else if (key_len == 8 && memcmp(kvp, "linkpath", 8) == 0) + slot = &r->pending_long_link; + if (slot) { + char *copy = malloc(val_len + 1); + if (!copy) { + free(buf); + *err = "tar PAX value allocation failed"; + errno = ENOMEM; + return -1; + } + memcpy(copy, val, val_len); + copy[val_len] = '\0'; + free(*slot); + *slot = copy; + } + } + p = record_end; + } + free(buf); + return 0; +} + +static int classify_typeflag(uint8_t flag, oci_tar_type_t *out) +{ + switch (flag) { + case '\0': + case '0': + case '7': /* contiguous file: treat as regular */ + *out = OCI_TAR_REG; + return 0; + case '1': + *out = OCI_TAR_HARDLINK; + return 0; + case '2': + *out = OCI_TAR_SYMLINK; + return 0; + case '3': + case '4': + case '6': + *out = OCI_TAR_UNSUPPORTED; /* char/block/fifo */ + return 0; + case '5': + *out = OCI_TAR_DIR; + return 0; + case 'L': + case 'K': + return 1; /* GNU long-name extension; caller handles payload */ + case 'x': + case 'g': + return 2; /* PAX extended; caller rejects */ + default: + return -1; /* unknown */ + } +} + +int oci_tar_next(oci_tar_reader_t *r, oci_tar_entry_t *out, const char **err) +{ + static const char *dummy_err; + if (!err) + err = &dummy_err; + *err = NULL; + + if (!r || !out) { + *err = "tar next called with NULL argument"; + errno = EINVAL; + return -1; + } + + /* Any unconsumed payload from a prior entry must be drained before + * the next header is read; the contract is that callers either + * read fully or skip explicitly, but defensively flush here too. + */ + if (r->bytes_remaining > 0 || r->padding_remaining > 0) { + if (oci_tar_skip_payload(r, err) < 0) + return -1; + } + + for (;;) { + uint8_t block[TAR_BLOCK_SIZE]; + int rc = fill_block(r, block); + if (rc < 0) { + *err = "tar header read failed"; + errno = EIO; + return -1; + } + if (rc == 0) { + /* Stream ended cleanly mid-archive. Treat as EOF; the + * archive may have omitted the final zero blocks, which + * happens with hand-rolled tarballs. + */ + return 0; + } + + if (is_zero_block(block)) { + if (r->saw_first_zero_block) + return 0; + r->saw_first_zero_block = true; + continue; + } + r->saw_first_zero_block = false; + + if (!verify_chksum(block)) { + *err = "tar header checksum mismatch"; + errno = EINVAL; + return -1; + } + if (!is_ustar_magic(block)) { + *err = "tar header missing ustar magic"; + errno = EINVAL; + return -1; + } + + uint8_t typeflag = block[OFF_TYPEFLAG]; + oci_tar_type_t type; + int klass = classify_typeflag(typeflag, &type); + if (klass < 0) { + *err = "tar header carries unknown typeflag"; + errno = EINVAL; + return -1; + } + + uint64_t size = 0; + if (parse_octal(block + OFF_SIZE, LEN_SIZE, &size) < 0) { + *err = "tar header size field unparseable"; + errno = EINVAL; + return -1; + } + + if (klass == 1) { + /* GNU long-name / long-link extension entry. The payload + * carries the path or linkname for the NEXT real entry. + */ + char **slot = + typeflag == 'L' ? &r->pending_long_name : &r->pending_long_link; + if (consume_long_record(r, size, slot, err) < 0) + return -1; + continue; + } + if (klass == 2) { + if (consume_pax_record(r, size, typeflag == 'g', err) < 0) + return -1; + continue; + } + + /* Real entry. Materialize path and linkname into reader-owned + * buffers so the caller can read entry.path stably until the + * next oci_tar_next call. + */ + if (r->pending_long_name) { + size_t want = strlen(r->pending_long_name) + 1; + if (ensure_capacity(&r->path, &r->path_cap, want) < 0) { + *err = "tar path buffer allocation failed"; + errno = ENOMEM; + return -1; + } + memcpy(r->path, r->pending_long_name, want); + free(r->pending_long_name); + r->pending_long_name = NULL; + } else { + if (ensure_capacity(&r->path, &r->path_cap, TAR_PATH_MAX) < 0) { + *err = "tar path buffer allocation failed"; + errno = ENOMEM; + return -1; + } + if (build_path_from_header(block, r->path, r->path_cap) < 0) { + *err = "tar header path overflow"; + errno = ENAMETOOLONG; + return -1; + } + } + if (r->pending_long_link) { + size_t want = strlen(r->pending_long_link) + 1; + if (ensure_capacity(&r->linkname, &r->linkname_cap, want) < 0) { + *err = "tar linkname buffer allocation failed"; + errno = ENOMEM; + return -1; + } + memcpy(r->linkname, r->pending_long_link, want); + free(r->pending_long_link); + r->pending_long_link = NULL; + } else { + if (ensure_capacity(&r->linkname, &r->linkname_cap, + LEN_LINKNAME + 1) < 0) { + *err = "tar linkname buffer allocation failed"; + errno = ENOMEM; + return -1; + } + if (copy_field_string(block + OFF_LINKNAME, LEN_LINKNAME, + r->linkname, r->linkname_cap) < 0) { + *err = "tar linkname overflow"; + errno = ENAMETOOLONG; + return -1; + } + } + + /* Dirs may carry a trailing slash in name; normalize it away so + * downstream matchers do not have to special-case both forms. + */ + if (type == OCI_TAR_DIR) + strip_trailing_slash(r->path); + + /* DIR entries should have size 0 in practice, but tolerate + * archives that record an unused size. The payload contract + * is "advertise size bytes; the reader consumes them". + */ + out->path = r->path; + out->linkname = (type == OCI_TAR_SYMLINK || type == OCI_TAR_HARDLINK) + ? r->linkname + : NULL; + out->type = type; + + uint64_t mode = 0; + uint64_t uid = 0; + uint64_t gid = 0; + uint64_t mtime = 0; + if (parse_octal(block + OFF_MODE, LEN_MODE, &mode) < 0 || + parse_octal(block + OFF_UID, LEN_UID, &uid) < 0 || + parse_octal(block + OFF_GID, LEN_GID, &gid) < 0 || + parse_octal(block + OFF_MTIME, LEN_MTIME, &mtime) < 0) { + *err = "tar header numeric field unparseable"; + errno = EINVAL; + return -1; + } + out->mode = (uint32_t) (mode & 07777); + out->uid = uid; + out->gid = gid; + out->mtime = mtime; + out->size = type == OCI_TAR_REG ? size : 0; + + apply_whiteout_flags(out); + + /* Stage payload tracking. Even non-regular entries may have a + * recorded size that the writer expects the reader to skip + * (rare, but real). The reader honors whatever size field the + * header advertises so the stream stays aligned. + */ + r->bytes_remaining = size; + r->padding_remaining = + size > 0 ? (uint32_t) ((TAR_BLOCK_SIZE - (size % TAR_BLOCK_SIZE)) % + TAR_BLOCK_SIZE) + : 0; + + return 1; + } +} + +int oci_tar_read_payload(oci_tar_reader_t *r, + void *buf, + size_t cap, + size_t *got, + const char **err) +{ + static const char *dummy_err; + if (!err) + err = &dummy_err; + *err = NULL; + if (got) + *got = 0; + if (!r || !buf || !got) { + *err = "tar read called with NULL argument"; + errno = EINVAL; + return -1; + } + + if (r->bytes_remaining == 0) { + /* Drain any tail padding so the next oci_tar_next aligns. */ + if (r->padding_remaining > 0) { + if (discard_bytes(r, r->padding_remaining) < 0) { + *err = "tar padding read failed"; + errno = EIO; + return -1; + } + r->padding_remaining = 0; + } + return 0; + } + + size_t want = cap > r->bytes_remaining ? (size_t) r->bytes_remaining : cap; + int rc = read_full(r, buf, want); + if (rc <= 0) { + *err = "tar payload truncated"; + errno = EIO; + return -1; + } + r->bytes_remaining -= want; + *got = want; + return 0; +} + +int oci_tar_skip_payload(oci_tar_reader_t *r, const char **err) +{ + static const char *dummy_err; + if (!err) + err = &dummy_err; + *err = NULL; + if (!r) { + *err = "tar skip called with NULL reader"; + errno = EINVAL; + return -1; + } + uint64_t drop = r->bytes_remaining + r->padding_remaining; + r->bytes_remaining = 0; + r->padding_remaining = 0; + if (drop == 0) + return 0; + if (discard_bytes(r, drop) < 0) { + *err = "tar skip read failed"; + errno = EIO; + return -1; + } + return 0; +} diff --git a/src/oci/tar.h b/src/oci/tar.h new file mode 100644 index 0000000..54e9615 --- /dev/null +++ b/src/oci/tar.h @@ -0,0 +1,88 @@ +/* OCI tar reader (POSIX ustar + GNU long-name; PAX rejected) + * + * Streams tar entries out of a generic byte source so the parser stays + * compression-agnostic; oci/decompress.c feeds either zlib, libzstd, or + * a passthrough stream into the read callback. The applier in + * oci/layer-apply.c then walks entries in order, dispatching by + * oci_tar_type_t. + * + * Block, char, fifo, and socket entries collapse to OCI_TAR_UNSUPPORTED + * so the applier can emit a precise refusal without re-decoding the + * typeflag. PAX extended headers are rejected outright per + * oci-roadmap.md Q3 asymmetric subset; if a real image is ever found + * to require PAX mtime or path records, expand the accept list with + * targeted parsing rather than enabling generic PAX. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include +#include + +typedef enum { + OCI_TAR_REG, + OCI_TAR_DIR, + OCI_TAR_SYMLINK, + OCI_TAR_HARDLINK, + OCI_TAR_UNSUPPORTED, +} oci_tar_type_t; + +typedef struct { + /* path and linkname are owned by the reader and remain valid until + * the next oci_tar_next call. They are const to the caller: the backing + * storage belongs to the reader, so callers must not mutate it, and any + * value kept past the next iteration must be duplicated. + */ + const char *path; + const char *linkname; + uint64_t size; + uint32_t mode; + uint64_t uid; + uint64_t gid; + uint64_t mtime; + oci_tar_type_t type; + bool is_whiteout; + bool is_opaque_whiteout; +} oci_tar_entry_t; + +/* Byte source callback. Returns >=0 bytes read into buf (0 means EOF) + * or -1 on I/O error with errno set. Short reads are tolerated; the + * reader retries until a full 512-byte block is available. + */ +typedef ssize_t (*oci_tar_read_fn)(void *ctx, void *buf, size_t cap); + +typedef struct oci_tar_reader oci_tar_reader_t; + +oci_tar_reader_t *oci_tar_reader_new(oci_tar_read_fn read_fn, void *ctx); +void oci_tar_reader_free(oci_tar_reader_t *r); + +/* Pull the next header. + * returns 1: entry populated; payload available via read or skip + * returns 0: clean EOF (two zero blocks or stream end) + * returns -1: protocol or I/O error; *err points to a static string + * + * Caller must call exactly one of oci_tar_read_payload or + * oci_tar_skip_payload (or read until *got == 0) before the next + * oci_tar_next so the reader can realign to the next 512-byte block. + */ +int oci_tar_next(oci_tar_reader_t *r, oci_tar_entry_t *out, const char **err); + +/* Copy up to cap bytes of the current entry's payload into buf. + * Returns 0 on success; *got carries the byte count (0 once the + * payload is exhausted). Returns -1 on read error with *err set. + */ +int oci_tar_read_payload(oci_tar_reader_t *r, + void *buf, + size_t cap, + size_t *got, + const char **err); + +/* Discard the rest of the current entry's payload plus its 512-byte + * block padding so the next oci_tar_next sees a fresh header. + */ +int oci_tar_skip_payload(oci_tar_reader_t *r, const char **err); diff --git a/src/oci/unpack.c b/src/oci/unpack.c new file mode 100644 index 0000000..fd99917 --- /dev/null +++ b/src/oci/unpack.c @@ -0,0 +1,1155 @@ +/* OCI layer unpack orchestrator implementation + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "oci/blob-store.h" +#include "oci/decompress.h" +#include "oci/digest.h" +#include "oci/layer-apply.h" +#include "oci/layer-meta.h" +#include "oci/manifest.h" +#include "oci/media-type.h" +#include "oci/origin-meta.h" +#include "oci/ref.h" +#include "oci/store.h" +#include "oci/tar.h" +#include "oci/unpack.h" +#include "oci/volume.h" + +#define UN_PATH_MAX 4096 +#define UN_BLOB_BUF 65536 + +typedef struct { + oci_stream_t *s; +} unpack_stream_ctx_t; + +static ssize_t unpack_stream_read_cb(void *ctx, void *buf, size_t cap) +{ + unpack_stream_ctx_t *c = ctx; + return oci_stream_read(c->s, buf, cap); +} + +static int set_err(const char **err, const char *msg, int err_no) +{ + if (err) + *err = msg; + errno = err_no; + return -1; +} + +static int mkdir_p(const char *path) +{ + char buf[UN_PATH_MAX]; + size_t n = strlen(path); + if (n >= sizeof(buf)) + return -1; + memcpy(buf, path, n + 1); + for (char *p = buf + 1; *p; p++) { + if (*p != '/') + continue; + *p = '\0'; + if (mkdir(buf, 0755) < 0 && errno != EEXIST) + return -1; + *p = '/'; + } + if (mkdir(buf, 0755) < 0 && errno != EEXIST) + return -1; + return 0; +} + +static int rand_hex(char *out, size_t n_hex) +{ + size_t need = n_hex / 2; + uint8_t buf[16]; + if (need > sizeof(buf)) + return -1; + if (getentropy(buf, need) < 0) + return -1; + static const char hex[] = "0123456789abcdef"; + for (size_t i = 0; i < need; i++) { + out[i * 2] = hex[buf[i] >> 4]; + out[i * 2 + 1] = hex[buf[i] & 0xf]; + } + out[n_hex] = '\0'; + return 0; +} + +static int read_blob(oci_blob_store_t *bs, + oci_digest_algo_t algo, + const char *hex, + uint8_t **out_buf, + size_t *out_len, + const char **err) +{ + char path[UN_PATH_MAX]; + if (oci_blob_store_path(bs, algo, hex, path, sizeof(path)) < 0) + return set_err(err, "unpack: blob path resolve failed", errno); + int fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + return set_err(err, "unpack: blob open failed", errno); + struct stat st; + if (fstat(fd, &st) < 0) { + close(fd); + return set_err(err, "unpack: blob fstat failed", errno); + } + if (st.st_size < 0 || st.st_size > (off_t) (256 * 1024 * 1024)) { + close(fd); + return set_err(err, "unpack: blob size out of bounds", EINVAL); + } + uint8_t *buf = malloc((size_t) st.st_size + 1); + if (!buf) { + close(fd); + return set_err(err, "unpack: blob buffer alloc failed", ENOMEM); + } + ssize_t got = read(fd, buf, (size_t) st.st_size); + close(fd); + if (got != st.st_size) { + free(buf); + return set_err(err, "unpack: blob short read", EIO); + } + buf[got] = '\0'; + *out_buf = buf; + *out_len = (size_t) got; + return 0; +} + +/* Open the on-disk blob path and confirm its sha256 hash matches the + * descriptor's expected digest. Phase 1 already verified at write time, + * but unpack re-verifies in case a host-side tool modified the blob. + */ +static int reverify_layer_digest(oci_blob_store_t *bs, + const oci_descriptor_t *desc, + const char **err) +{ + char path[UN_PATH_MAX]; + if (oci_blob_store_path(bs, desc->algo, desc->hex, path, sizeof(path)) < 0) + return set_err(err, "unpack: layer path resolve failed", errno); + int fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + return set_err(err, "unpack: layer open failed", errno); + + oci_digester_t *d = oci_digester_new(desc->algo); + if (!d) { + close(fd); + return set_err(err, "unpack: digester alloc failed", ENOMEM); + } + uint8_t buf[UN_BLOB_BUF]; + for (;;) { + ssize_t n = read(fd, buf, sizeof(buf)); + if (n < 0) { + if (errno == EINTR) + continue; + oci_digester_free(d); + close(fd); + return set_err(err, "unpack: layer read failed", errno); + } + if (n == 0) + break; + oci_digester_update(d, buf, (size_t) n); + } + close(fd); + char got_hex[OCI_DIGEST_HEX_MAX + 1]; + oci_digester_finish_hex(d, got_hex); + oci_digester_free(d); + if (strcmp(got_hex, desc->hex) != 0) + return set_err(err, "unpack: layer blob digest mismatch", EINVAL); + return 0; +} + +/* Recursively rm a path so clonefile(2) can recreate it. Matches the + * discipline used in src/oci/clone-rootfs.c: lstat + recurse, no shell-out. + * Returns 0 on success or when path was already absent; -1 with errno set + * on any unexpected IO error. The caller is responsible for ensuring path + * is safe to remove (e.g. a freshly mkdir'd stage_dir, not a user dir). + */ +static int rm_recursive(const char *path) +{ + struct stat st; + if (lstat(path, &st) < 0) + return errno == ENOENT ? 0 : -1; + if (!S_ISDIR(st.st_mode)) + return unlink(path); + DIR *d = opendir(path); + if (!d) + return -1; + struct dirent *de; + int rc = 0; + while ((de = readdir(d))) { + if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) + continue; + char child[UN_PATH_MAX]; + int n = snprintf(child, sizeof(child), "%s/%s", path, de->d_name); + if (n < 0 || (size_t) n >= sizeof(child)) { + errno = ENAMETOOLONG; + rc = -1; + break; + } + if (rm_recursive(child) < 0) { + rc = -1; + break; + } + } + closedir(d); + if (rc == 0 && rmdir(path) < 0) + rc = -1; + return rc; +} + +/* Shared reverify + decompress + apply pipeline for the two + * single-layer entry points. UNPACK_MODE_OVERLAY drives + * oci_layer_apply (whiteout / opaque interpreted against root_dir); + * UNPACK_MODE_RAW drives oci_layer_apply_raw_tar (whiteout markers + * preserved as zero-byte regular files for the C3.3c raw per-layer + * cache populate path). + */ +typedef enum { + UNPACK_MODE_OVERLAY, + UNPACK_MODE_RAW, +} unpack_mode_t; + +static int unpack_layer_impl(oci_blob_store_t *bs, + const oci_descriptor_t *desc, + const char *root_dir, + unpack_mode_t mode, + oci_layer_apply_stats_t *stats, + oci_meta_table_t *meta, + const char *log_label, + const char **err) +{ + if (!bs || !desc || !root_dir) + return set_err(err, "unpack_layer: NULL argument", EINVAL); + + if (oci_media_type_is_foreign(desc->media_type)) + return set_err(err, "unpack: layer is foreign / nondistributable", + ENOTSUP); + + if (reverify_layer_digest(bs, desc, err) < 0) + return -1; + char path[UN_PATH_MAX]; + if (oci_blob_store_path(bs, desc->algo, desc->hex, path, sizeof(path)) < 0) + return set_err(err, "unpack: layer path resolve failed", errno); + int fd = open(path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + return set_err(err, "unpack: layer open failed", errno); + + oci_compression_t alg = oci_media_type_compression(desc->media_type); + oci_stream_t *stream = oci_decompress_open(fd, alg, err); + if (!stream) { + close(fd); + return -1; + } + + unpack_stream_ctx_t tctx = {.s = stream}; + oci_tar_reader_t *r = oci_tar_reader_new(unpack_stream_read_cb, &tctx); + if (!r) { + oci_stream_close(stream); + close(fd); + return set_err(err, "unpack: tar reader alloc failed", ENOMEM); + } + + if (log_label) + fprintf(stderr, " %s: %s\n", log_label, desc->digest_str); + + oci_layer_apply_stats_t local_stats = {0}; + int rc; + if (mode == UNPACK_MODE_OVERLAY) + rc = oci_layer_apply(r, root_dir, &local_stats, meta, err); + else + rc = oci_layer_apply_raw_tar(r, root_dir, &local_stats, meta, err); + + oci_tar_reader_free(r); + oci_stream_close(stream); + close(fd); + + if (rc < 0) + return -1; + if (stats) { + stats->files += local_stats.files; + stats->dirs += local_stats.dirs; + stats->symlinks += local_stats.symlinks; + stats->hardlinks += local_stats.hardlinks; + stats->whiteouts += local_stats.whiteouts; + stats->opaques += local_stats.opaques; + } + if (log_label) + fprintf(stderr, + " +files=%zu dirs=%zu symlinks=%zu hardlinks=%zu " + "whiteouts=%zu opaques=%zu\n", + local_stats.files, local_stats.dirs, local_stats.symlinks, + local_stats.hardlinks, local_stats.whiteouts, + local_stats.opaques); + return 0; +} + +int oci_unpack_layer(oci_blob_store_t *bs, + const oci_descriptor_t *desc, + const char *stage_dir, + oci_layer_apply_stats_t *stats, + oci_meta_table_t *meta, + const char *log_label, + const char **err) +{ + static const char *dummy_err; + if (!err) + err = &dummy_err; + *err = NULL; + return unpack_layer_impl(bs, desc, stage_dir, UNPACK_MODE_OVERLAY, stats, + meta, log_label, err); +} + +int oci_unpack_layer_raw(oci_blob_store_t *bs, + const oci_descriptor_t *desc, + const char *raw_dir, + oci_layer_apply_stats_t *stats, + oci_meta_table_t *meta, + const char *log_label, + const char **err) +{ + static const char *dummy_err; + if (!err) + err = &dummy_err; + *err = NULL; + return unpack_layer_impl(bs, desc, raw_dir, UNPACK_MODE_RAW, stats, meta, + log_label, err); +} + +/* --- C3.3c-ii two-pass overlay assembler ------------------------------- */ + +#define UN_RAW_META_SIDECAR ".elfuse-meta.layer.json" + +static bool is_whiteout_name(const char *name) +{ + return strncmp(name, ".wh.", 4) == 0; +} + +/* Remove every direct child of path, leaving path itself in place. Used + * to honour the OCI ".wh..wh..opq" opaque marker: the parent directory + * stays so this layer's siblings can land on top. + */ +static int clear_dir_contents(const char *path, const char **err) +{ + DIR *d = opendir(path); + if (!d) { + if (errno == ENOENT) + return 0; + return set_err(err, "assemble: clear opendir failed", errno); + } + struct dirent *de; + int rc = 0; + while ((de = readdir(d))) { + if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) + continue; + char child[UN_PATH_MAX]; + int n = snprintf(child, sizeof(child), "%s/%s", path, de->d_name); + if (n < 0 || (size_t) n >= sizeof(child)) { + rc = set_err(err, "assemble: clear path overflow", ENAMETOOLONG); + break; + } + if (rm_recursive(child) < 0) { + rc = set_err(err, "assemble: clear rm child failed", errno); + break; + } + } + closedir(d); + return rc; +} + +static int assembly_walk_whiteouts(const char *raw_dir, + const char *stage_dir, + const char **err) +{ + DIR *d = opendir(raw_dir); + if (!d) + return set_err(err, "assemble: whiteout opendir failed", errno); + struct dirent *de; + int rc = 0; + while ((de = readdir(d))) { + if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) + continue; + if (strcmp(de->d_name, UN_RAW_META_SIDECAR) == 0) + continue; + char raw_child[UN_PATH_MAX]; + char stage_child[UN_PATH_MAX]; + int n1 = snprintf(raw_child, sizeof(raw_child), "%s/%s", raw_dir, + de->d_name); + int n2 = snprintf(stage_child, sizeof(stage_child), "%s/%s", stage_dir, + de->d_name); + if (n1 < 0 || (size_t) n1 >= sizeof(raw_child) || n2 < 0 || + (size_t) n2 >= sizeof(stage_child)) { + rc = set_err(err, "assemble: whiteout path overflow", ENAMETOOLONG); + break; + } + struct stat st; + if (lstat(raw_child, &st) < 0) { + rc = set_err(err, "assemble: whiteout lstat failed", errno); + break; + } + if (S_ISDIR(st.st_mode)) { + /* Recurse: subdirectories may carry their own markers. The + * stage_child counterpart may not exist yet (pass 2 creates + * the missing directories), which is fine: descending into + * the raw side still finds the markers, and the rm-r / + * clear-contents calls below tolerate a missing target. + */ + if (assembly_walk_whiteouts(raw_child, stage_child, err) < 0) { + rc = -1; + break; + } + continue; + } + if (!S_ISREG(st.st_mode)) + continue; + if (strcmp(de->d_name, ".wh..wh..opq") == 0) { + if (clear_dir_contents(stage_dir, err) < 0) { + rc = -1; + break; + } + continue; + } + if (is_whiteout_name(de->d_name)) { + char target[UN_PATH_MAX]; + int nt = snprintf(target, sizeof(target), "%s/%s", stage_dir, + de->d_name + 4); + if (nt < 0 || (size_t) nt >= sizeof(target)) { + rc = set_err(err, "assemble: whiteout target overflow", + ENAMETOOLONG); + break; + } + if (rm_recursive(target) < 0) { + rc = set_err(err, "assemble: whiteout rm failed", errno); + break; + } + } + } + closedir(d); + return rc; +} + +static int assembly_walk_content(const char *raw_dir, + const char *stage_dir, + const char **err) +{ + DIR *d = opendir(raw_dir); + if (!d) + return set_err(err, "assemble: content opendir failed", errno); + struct dirent *de; + int rc = 0; + while ((de = readdir(d))) { + if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0) + continue; + if (strcmp(de->d_name, UN_RAW_META_SIDECAR) == 0) + continue; + if (is_whiteout_name(de->d_name)) + continue; + char raw_child[UN_PATH_MAX]; + char stage_child[UN_PATH_MAX]; + int n1 = snprintf(raw_child, sizeof(raw_child), "%s/%s", raw_dir, + de->d_name); + int n2 = snprintf(stage_child, sizeof(stage_child), "%s/%s", stage_dir, + de->d_name); + if (n1 < 0 || (size_t) n1 >= sizeof(raw_child) || n2 < 0 || + (size_t) n2 >= sizeof(stage_child)) { + rc = set_err(err, "assemble: content path overflow", ENAMETOOLONG); + break; + } + struct stat raw_st; + if (lstat(raw_child, &raw_st) < 0) { + rc = set_err(err, "assemble: content lstat failed", errno); + break; + } + if (S_ISDIR(raw_st.st_mode)) { + struct stat dst; + if (lstat(stage_child, &dst) < 0) { + if (errno != ENOENT) { + rc = set_err(err, "assemble: stage dir stat failed", errno); + break; + } + if (mkdir(stage_child, 0755) < 0) { + rc = set_err(err, "assemble: stage mkdir failed", errno); + break; + } + } else if (!S_ISDIR(dst.st_mode)) { + /* Lower-layer non-dir collides with this layer's dir. + * Overlay semantics: this layer's dir wins. unlink the + * lower entry and create the dir. + */ + if (unlink(stage_child) < 0) { + rc = set_err(err, "assemble: stage unlink-for-dir failed", + errno); + break; + } + if (mkdir(stage_child, 0755) < 0) { + rc = set_err(err, "assemble: stage mkdir-replace failed", + errno); + break; + } + } + if (assembly_walk_content(raw_child, stage_child, err) < 0) { + rc = -1; + break; + } + continue; + } + /* Regular file or symlink (or any other non-directory): unlink + * any existing destination then copyfile with COPYFILE_CLONE + * so APFS COW keeps the byte cost flat when raw cache and + * stage share a volume, and falls back to a byte copy when + * they do not (default elfuse layout puts the store on the + * root volume and the stage on a sparsebundle, so the EXDEV + * fallback is the steady-state path for fresh unpacks until + * the layouts are unified). Per D8, hardlink relationships + * from the tar are not reconstructed (each copyfile produces + * an independent inode). + */ + struct stat dst; + if (lstat(stage_child, &dst) == 0) { + if (rm_recursive(stage_child) < 0) { + rc = set_err(err, "assemble: unlink dst failed", errno); + break; + } + } else if (errno != ENOENT) { + rc = set_err(err, "assemble: dst lstat failed", errno); + break; + } + if (copyfile(raw_child, stage_child, NULL, + COPYFILE_CLONE | COPYFILE_ALL) < 0) { + rc = set_err(err, "assemble: copyfile failed", errno); + break; + } + } + closedir(d); + return rc; +} + +int oci_unpack_assemble_layer(const char *raw_dir, + const char *stage_dir, + const char **err) +{ + static const char *dummy_err; + if (!err) + err = &dummy_err; + *err = NULL; + if (!raw_dir || !stage_dir) + return set_err(err, "assemble: NULL argument", EINVAL); + struct stat st; + if (lstat(raw_dir, &st) < 0 || !S_ISDIR(st.st_mode)) + return set_err(err, "assemble: raw_dir is not a directory", ENOTDIR); + if (lstat(stage_dir, &st) < 0 || !S_ISDIR(st.st_mode)) + return set_err(err, "assemble: stage_dir is not a directory", ENOTDIR); + if (assembly_walk_whiteouts(raw_dir, stage_dir, err) < 0) + return -1; + if (assembly_walk_content(raw_dir, stage_dir, err) < 0) + return -1; + return 0; +} + +/* Resolve the manifest digest for ref: prefer ref->digest_str when + * present, else read the pin file via oci_store_get_ref. + */ +static int resolve_manifest_digest(oci_store_t *store, + const oci_ref_t *ref, + char *out_str, + size_t out_cap, + const char **err) +{ + if (ref->digest && ref->digest[0]) { + if (strlen(ref->digest) + 1 > out_cap) + return set_err(err, "unpack: digest string overflow", ENAMETOOLONG); + memcpy(out_str, ref->digest, strlen(ref->digest) + 1); + return 0; + } + char *pin = NULL; + const char *perr = NULL; + if (oci_store_get_ref(store, ref, &pin, &perr) < 0) { + if (errno == ENOENT) + return set_err( + err, "unpack: tag pin missing; run 'elfuse oci pull' first", + ENOENT); + return set_err(err, perr ? perr : "unpack: tag pin read failed", + errno ? errno : EIO); + } + if (strlen(pin) + 1 > out_cap) { + free(pin); + return set_err(err, "unpack: pin string overflow", ENAMETOOLONG); + } + memcpy(out_str, pin, strlen(pin) + 1); + free(pin); + return 0; +} + +int oci_unpack(oci_store_t *store, + const oci_ref_t *ref, + const oci_unpack_options_t *opts, + char **out_image_dir, + const char **err) +{ + static const char *dummy_err; + if (!err) + err = &dummy_err; + *err = NULL; + if (!store || !ref || !out_image_dir) + return set_err(err, "unpack: NULL argument", EINVAL); + *out_image_dir = NULL; + + bool quiet = opts && opts->quiet; + bool force = opts && opts->force_relayer; + + /* Resolve / provision the sysroot volume. */ + char *volume_root = NULL; + if (oci_volume_ensure(opts ? opts->volume_root : NULL, &volume_root, err) < + 0) + return -1; + + /* Ensure images/ and images/.staging/ exist. */ + char *images_dir = NULL; + char *staging_dir = NULL; + if (oci_volume_subdir(volume_root, "images", &images_dir, err) < 0) + goto fail_volume; + if (oci_volume_subdir(volume_root, "images/.staging", &staging_dir, err) < + 0) + goto fail_images; + + /* Resolve the manifest digest. */ + char manifest_digest[OCI_DIGEST_HEX_MAX + 16]; + if (resolve_manifest_digest(store, ref, manifest_digest, + sizeof(manifest_digest), err) < 0) + goto fail_staging; + + oci_digest_algo_t algo; + char manifest_hex[OCI_DIGEST_HEX_MAX + 1]; + if (!oci_digest_parse(manifest_digest, &algo, manifest_hex)) + return set_err(err, "unpack: manifest digest parse failed", EINVAL); + + /* Read the manifest blob. If it is an image-index, pick linux/arm64 + * and re-read the sub-manifest. + */ + oci_blob_store_t *bs = oci_store_blobs(store); + if (!bs) { + set_err(err, "unpack: blob store unavailable", EIO); + goto fail_staging; + } + uint8_t *body = NULL; + size_t body_len = 0; + if (read_blob(bs, algo, manifest_hex, &body, &body_len, err) < 0) + goto fail_staging; + + oci_manifest_t manifest = {0}; + oci_index_t index = {0}; + const char *perr = NULL; + char *image_hex = NULL; + /* Try manifest first; if it fails, try index. */ + if (oci_manifest_parse((const char *) body, body_len, &manifest, &perr) < + 0) { + memset(&manifest, 0, sizeof(manifest)); + if (oci_index_parse((const char *) body, body_len, &index, &perr) < 0) { + set_err(err, perr ? perr : "unpack: manifest parse failed", EINVAL); + free(body); + goto fail_staging; + } + free(body); + const oci_index_entry_t *pick = oci_index_pick_linux_arm64(&index); + if (!pick) { + oci_index_free(&index); + set_err(err, "unpack: no linux/arm64 entry in image index", ENOENT); + goto fail_staging; + } + char sub_digest[OCI_DIGEST_HEX_MAX + 16]; + if (strlen(pick->desc.digest_str) >= sizeof(sub_digest)) { + oci_index_free(&index); + set_err(err, "unpack: sub-manifest digest overflow", ENAMETOOLONG); + goto fail_staging; + } + memcpy(sub_digest, pick->desc.digest_str, + strlen(pick->desc.digest_str) + 1); + oci_digest_algo_t sub_algo; + char sub_hex[OCI_DIGEST_HEX_MAX + 1]; + if (!oci_digest_parse(sub_digest, &sub_algo, sub_hex)) { + oci_index_free(&index); + set_err(err, "unpack: sub-manifest digest parse failed", EINVAL); + goto fail_staging; + } + oci_index_free(&index); + if (read_blob(bs, sub_algo, sub_hex, &body, &body_len, err) < 0) + goto fail_staging; + if (oci_manifest_parse((const char *) body, body_len, &manifest, + &perr) < 0) { + set_err(err, perr ? perr : "unpack: sub-manifest parse failed", + EINVAL); + free(body); + goto fail_staging; + } + image_hex = strdup(sub_hex); + } else { + image_hex = strdup(manifest_hex); + } + free(body); + + if (!image_hex) { + oci_manifest_free(&manifest); + set_err(err, "unpack: image hex strdup failed", ENOMEM); + goto fail_staging; + } + + /* Final target: /images/sha256-/. The directory has + * '-' instead of ':' to keep the path filesystem-friendly. + */ + char final_dir[UN_PATH_MAX]; + if ((size_t) snprintf(final_dir, sizeof(final_dir), "%s/sha256-%s", + images_dir, image_hex) >= sizeof(final_dir)) { + free(image_hex); + oci_manifest_free(&manifest); + set_err(err, "unpack: final dir overflow", ENAMETOOLONG); + goto fail_staging; + } + + struct stat st; + if (lstat(final_dir, &st) == 0 && !force) { + /* Idempotent rerun: image sysroot already exists. */ + free(image_hex); + oci_manifest_free(&manifest); + size_t want = strlen(final_dir) + 2; + char *dup = malloc(want); + if (!dup) { + set_err(err, "unpack: strdup final path failed", ENOMEM); + goto fail_staging; + } + snprintf(dup, want, "%s/", final_dir); + *out_image_dir = dup; + free(staging_dir); + free(images_dir); + free(volume_root); + return 0; + } + if (force) { + /* Remove any prior commit so the staging rename does not race. */ + char rm[UN_PATH_MAX]; + snprintf(rm, sizeof(rm), "rm -rf '%s'", final_dir); + (void) system(rm); + } + + /* Stage under /images/.staging// */ + char stage_id[13]; + if (rand_hex(stage_id, 12) < 0) { + free(image_hex); + oci_manifest_free(&manifest); + set_err(err, "unpack: getentropy failed", errno); + goto fail_staging; + } + char stage_dir[UN_PATH_MAX]; + if ((size_t) snprintf(stage_dir, sizeof(stage_dir), "%s/%s", staging_dir, + stage_id) >= sizeof(stage_dir)) { + free(image_hex); + oci_manifest_free(&manifest); + set_err(err, "unpack: stage dir overflow", ENAMETOOLONG); + goto fail_staging; + } + if (mkdir_p(stage_dir) < 0) { + free(image_hex); + oci_manifest_free(&manifest); + set_err(err, "unpack: mkdir stage failed", errno); + goto fail_staging; + } + + if (!quiet) + fprintf(stderr, "elfuse oci unpack: applying %zu layer(s)\n", + manifest.nlayers); + + /* Read + parse the image-config blob up-front so per-layer diff_ids + * are available to the cache hook in oci_unpack_layer. The Plan 1 + * origin sidecar still consumes the same struct later in this + * function, so the read happens exactly once. + */ + oci_image_config_t cfg = {0}; + { + uint8_t *cfg_body = NULL; + size_t cfg_len = 0; + if (read_blob(bs, manifest.config.algo, manifest.config.hex, &cfg_body, + &cfg_len, err) < 0) { + free(image_hex); + oci_manifest_free(&manifest); + goto fail_stage_dir; + } + const char *cparse_err = NULL; + if (oci_image_config_parse((const char *) cfg_body, cfg_len, &cfg, + &cparse_err) < 0) { + set_err( + err, + cparse_err ? cparse_err : "unpack: image config parse failed", + EINVAL); + free(cfg_body); + free(image_hex); + oci_manifest_free(&manifest); + goto fail_stage_dir; + } + free(cfg_body); + } + + /* Validate that diff_ids[] length matches manifest.layers[] length. A + * mismatch is a malformed image (the OCI image-spec mandates one + * diff_id per layer in order); fail-fast so the cache never associates + * a diff_id with the wrong layer payload. + */ + size_t diff_ids_count = 0; + if (cfg.rootfs_diff_ids) + while (cfg.rootfs_diff_ids[diff_ids_count]) + diff_ids_count++; + if (diff_ids_count != manifest.nlayers) { + set_err(err, "unpack: image config rootfs.diff_ids count mismatch", + EINVAL); + oci_image_config_free(&cfg); + free(image_hex); + oci_manifest_free(&manifest); + goto fail_stage_dir; + } + + /* C3.3c-ii orchestrator state. cum_meta accumulates the running + * cumulative meta table (uid/gid/mode per guest path); layer_meta + * is reset to a fresh table at the start of every loop iteration; + * chains holds the precomputed OCI ChainID strings for every + * layer so the stack-cache prefix search is one stat(2) per layer. + */ + oci_meta_table_t *cum_meta = NULL; + oci_meta_table_t *layer_meta = NULL; + char (*chains)[OCI_DIGEST_HEX_MAX + 16] = NULL; + + cum_meta = oci_meta_table_new(); + if (!cum_meta) { + set_err(err, "unpack: meta table alloc failed", ENOMEM); + goto fail_orch; + } + + if (manifest.nlayers > 0) { + chains = malloc(manifest.nlayers * sizeof(*chains)); + if (!chains) { + set_err(err, "unpack: chain array alloc failed", ENOMEM); + goto fail_orch; + } + const char *prev = NULL; + for (size_t i = 0; i < manifest.nlayers; i++) { + if (oci_chainid_compute(prev, cfg.rootfs_diff_ids[i], chains[i], + sizeof(chains[i])) < 0) { + set_err(err, "unpack: chain compute failed", + errno ? errno : EINVAL); + goto fail_orch; + } + prev = chains[i]; + } + } + + /* Search the stack cache backwards for the longest matching prefix + * snapshot. On hit, clonefile-restore the assembled stage_dir + * straight from cache and continue with the trailing layers only. + * No hit -> stage_dir stays at the empty mkdir_p state and the + * orchestrator iterates over every layer. + */ + size_t start_i = 0; + for (size_t k = manifest.nlayers; k-- > 0;) { + int hit = oci_store_stack_has(store, chains[k]); + if (hit < 0) { + set_err(err, "unpack: stack lookup failed", errno); + goto fail_orch; + } + if (hit != 1) + continue; + char stack_dir[UN_PATH_MAX]; + if (oci_store_stack_resolve(store, chains[k], stack_dir, + sizeof(stack_dir)) < 0) { + set_err(err, "unpack: stack resolve failed", errno); + goto fail_orch; + } + size_t sl = strlen(stack_dir); + if (sl > 0 && stack_dir[sl - 1] == '/') + stack_dir[sl - 1] = '\0'; + /* copyfile with COPYFILE_CLONE prefers an APFS clone (cheap + * COW) and falls back to a recursive byte copy on EXDEV, so + * stack restore works whether the store and stage share a + * volume or not. COPYFILE_CLONE implies an exclusive + * destination; the rm_recursive above prepares an absent + * target for both code paths. + */ + if (rm_recursive(stage_dir) < 0) { + set_err(err, "unpack: stage rm-for-stack failed", errno); + goto fail_orch; + } + if (copyfile(stack_dir, stage_dir, NULL, + COPYFILE_CLONE | COPYFILE_RECURSIVE | COPYFILE_NOFOLLOW | + COPYFILE_ALL) < 0) { + int saved = errno; + set_err(err, "unpack: stack restore copyfile failed", saved); + goto fail_orch; + } + /* Re-load the cumulative meta sidecar the stack snapshot + * persisted so trailing layers accumulate on top. A missing + * sidecar (older snapshot) is benign: cum_meta stays empty. + */ + oci_meta_table_t *restored = NULL; + const char *merr = NULL; + if (oci_meta_read(stage_dir, &restored, &merr) < 0) { + if (errno != ENOENT) { + set_err(err, merr ? merr : "unpack: stack meta read failed", + errno); + goto fail_orch; + } + errno = 0; + } else { + int mrc = oci_meta_merge(cum_meta, restored); + int saved = errno; + oci_meta_table_free(restored); + if (mrc < 0) { + set_err(err, "unpack: stack meta merge failed", saved); + goto fail_orch; + } + } + start_i = k + 1; + if (!quiet) + fprintf(stderr, "elfuse oci unpack: stack hit at chain %zu/%zu\n", + start_i, manifest.nlayers); + break; + } + + if (!quiet && manifest.nlayers > 0) + fprintf(stderr, + "elfuse oci unpack: applying %zu layer(s) (cache start %zu)\n", + manifest.nlayers - start_i, start_i); + + for (size_t i = start_i; i < manifest.nlayers; i++) { + char label[32]; + const char *log_label = NULL; + if (!quiet) { + snprintf(label, sizeof(label), "layer %zu", i + 1); + log_label = label; + } + + layer_meta = oci_meta_table_new(); + if (!layer_meta) { + set_err(err, "unpack: layer meta alloc failed", ENOMEM); + goto fail_orch; + } + + const char *diff_id = cfg.rootfs_diff_ids[i]; + char raw_cache_dir[UN_PATH_MAX]; + int raw_hit = oci_store_layer_has(store, diff_id); + if (raw_hit < 0) { + set_err(err, "unpack: raw cache lookup failed", errno); + goto fail_orch; + } + if (raw_hit == 1) { + if (oci_store_layer_resolve(store, diff_id, raw_cache_dir, + sizeof(raw_cache_dir)) < 0) { + set_err(err, "unpack: raw cache resolve failed", errno); + goto fail_orch; + } + size_t rl = strlen(raw_cache_dir); + if (rl > 0 && raw_cache_dir[rl - 1] == '/') + raw_cache_dir[rl - 1] = '\0'; + /* Load the per-layer sidecar so cum_meta picks up the + * uid/gid/mode entries the cache writer recorded at + * populate time. Missing sidecar is benign (older or + * hand-seeded entry). + */ + oci_meta_table_t *loaded = NULL; + const char *merr = NULL; + if (oci_meta_read_named(raw_cache_dir, UN_RAW_META_SIDECAR, &loaded, + &merr) < 0) { + if (errno != ENOENT) { + set_err(err, merr ? merr : "unpack: raw meta read failed", + errno); + goto fail_orch; + } + errno = 0; + } else { + oci_meta_table_free(layer_meta); + layer_meta = loaded; + } + if (log_label) + fprintf(stderr, " %s: %s (raw cached)\n", log_label, + manifest.layers[i].digest_str); + } else { + char raw_stage[UN_PATH_MAX]; + if (oci_store_layer_stage_path(store, diff_id, raw_stage, + sizeof(raw_stage)) < 0) { + set_err(err, "unpack: raw stage_path resolve failed", errno); + goto fail_orch; + } + if (mkdir(raw_stage, 0755) < 0) { + set_err(err, "unpack: raw stage mkdir failed", errno); + goto fail_orch; + } + if (oci_unpack_layer_raw(bs, &manifest.layers[i], raw_stage, NULL, + layer_meta, log_label, err) < 0) { + (void) rm_recursive(raw_stage); + goto fail_orch; + } + const char *mwerr = NULL; + if (oci_meta_write_named(layer_meta, raw_stage, UN_RAW_META_SIDECAR, + &mwerr) < 0) { + set_err(err, mwerr ? mwerr : "unpack: raw meta write failed", + errno); + (void) rm_recursive(raw_stage); + goto fail_orch; + } + const char *cerr = NULL; + if (oci_store_layer_commit(store, raw_stage, diff_id, &cerr) < 0) { + int saved = errno; + set_err(err, cerr ? cerr : "unpack: raw cache commit failed", + saved); + (void) rm_recursive(raw_stage); + goto fail_orch; + } + if (oci_store_layer_resolve(store, diff_id, raw_cache_dir, + sizeof(raw_cache_dir)) < 0) { + set_err(err, "unpack: raw cache resolve failed", errno); + goto fail_orch; + } + size_t rl = strlen(raw_cache_dir); + if (rl > 0 && raw_cache_dir[rl - 1] == '/') + raw_cache_dir[rl - 1] = '\0'; + } + + if (oci_unpack_assemble_layer(raw_cache_dir, stage_dir, err) < 0) + goto fail_orch; + + int mrc = oci_meta_merge(cum_meta, layer_meta); + int saved_errno = errno; + oci_meta_table_free(layer_meta); + layer_meta = NULL; + if (mrc < 0) { + set_err(err, "unpack: cum meta merge failed", saved_errno); + goto fail_orch; + } + + /* Snapshot stage_dir into the per-prefix stack cache so future + * unpacks sharing this chain prefix short-circuit. Failure here + * is fatal: silently degrading the cache would defeat the + * dedup path. + */ + if (oci_meta_write(cum_meta, stage_dir, err) < 0) + goto fail_orch; + char stack_stage[UN_PATH_MAX]; + if (oci_store_stack_stage_path(store, chains[i], stack_stage, + sizeof(stack_stage)) < 0) { + set_err(err, "unpack: stack stage_path resolve failed", errno); + goto fail_orch; + } + /* Same copyfile + COPYFILE_CLONE rationale as the stack + * restore: prefer APFS clone, fall back to recursive byte + * copy on EXDEV so the cache populates regardless of which + * volume holds the stage. + */ + if (copyfile(stage_dir, stack_stage, NULL, + COPYFILE_CLONE | COPYFILE_RECURSIVE | COPYFILE_NOFOLLOW | + COPYFILE_ALL) < 0) { + int saved = errno; + set_err(err, "unpack: stack snapshot copyfile failed", saved); + goto fail_orch; + } + const char *scerr = NULL; + if (oci_store_stack_commit(store, stack_stage, chains[i], &scerr) < 0) { + int saved = errno; + (void) rm_recursive(stack_stage); + set_err(err, scerr ? scerr : "unpack: stack commit failed", saved); + goto fail_orch; + } + } + + /* The per-iteration writes already produced an up-to-date sidecar + * on disk. On the full-stack-hit path (no iterations ran) the + * clonefile-restored stage_dir also already carries the snapshot's + * sidecar, so a final write would only re-emit identical bytes. + */ + oci_meta_table_free(cum_meta); + cum_meta = NULL; + free(chains); + chains = NULL; + + /* Origin sidecar: records manifest_digest + config_digest + diff_ids + * the Plan 1 keep-set walker reads. A failure here aborts the commit + * because a missing origin file would let prune silently delete layer + * blobs still backing this unpacked tree. + */ + { + char manifest_full[OCI_DIGEST_HEX_MAX + 16]; + if ((size_t) snprintf(manifest_full, sizeof(manifest_full), "sha256:%s", + image_hex) >= sizeof(manifest_full)) { + oci_image_config_free(&cfg); + set_err(err, "unpack: manifest digest overflow", ENAMETOOLONG); + free(image_hex); + oci_manifest_free(&manifest); + goto fail_stage_dir; + } + + const char *origin_err = NULL; + if (oci_origin_write(stage_dir, manifest_full, + manifest.config.digest_str, cfg.rootfs_diff_ids, + &origin_err) < 0) { + set_err(err, + origin_err ? origin_err : "unpack: origin write failed", + errno ? errno : EIO); + oci_image_config_free(&cfg); + free(image_hex); + oci_manifest_free(&manifest); + goto fail_stage_dir; + } + } + oci_image_config_free(&cfg); + + oci_manifest_free(&manifest); + + /* Atomic commit. */ + if (rename(stage_dir, final_dir) < 0) { + set_err(err, "unpack: stage rename failed", errno); + free(image_hex); + goto fail_stage_dir; + } + free(image_hex); + + size_t want = strlen(final_dir) + 2; + char *dup = malloc(want); + if (!dup) { + set_err(err, "unpack: strdup final path failed", ENOMEM); + goto fail_staging; + } + snprintf(dup, want, "%s/", final_dir); + *out_image_dir = dup; + + free(staging_dir); + free(images_dir); + free(volume_root); + return 0; + +fail_orch: + oci_meta_table_free(layer_meta); + oci_meta_table_free(cum_meta); + free(chains); + oci_image_config_free(&cfg); + free(image_hex); + oci_manifest_free(&manifest); +fail_stage_dir: { + char rm[UN_PATH_MAX]; + snprintf(rm, sizeof(rm), "rm -rf '%s'", stage_dir); + (void) system(rm); +} +fail_staging: + free(staging_dir); +fail_images: + free(images_dir); +fail_volume: + free(volume_root); + return -1; +} diff --git a/src/oci/unpack.h b/src/oci/unpack.h new file mode 100644 index 0000000..084c350 --- /dev/null +++ b/src/oci/unpack.h @@ -0,0 +1,162 @@ +/* OCI layer unpack orchestrator + * + * Drives the full Phase 2 pipeline: resolve a ref to a manifest digest + * through oci_store, read the manifest from the blob store, walk its + * layers, re-verify each layer blob's digest, decompress, and apply + * via oci_layer_apply into a staging directory under the sysroot + * volume's images/.staging/ subtree. Successful unpack commits via + * atomic rename into images/sha256-/. + * + * Phase 3 will consume the resulting directory via `elfuse run IMAGE`. + * Phase 2 stops at producing the directory; the user wires it manually + * through `elfuse --sysroot `. + * + * Copyright 2026 elfuse contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +#include "oci/blob-store.h" +#include "oci/layer-apply.h" +#include "oci/layer-meta.h" +#include "oci/manifest.h" +#include "oci/ref.h" +#include "oci/store.h" + +typedef struct { + const char *volume_root; /* NULL -> default sparse APFS volume */ + bool quiet; + bool force_relayer; +} oci_unpack_options_t; + +/* Apply one OCI layer's tar payload into stage_dir as a pure overlay + * extract. + * + * Re-verifies the compressed blob digest against desc, opens the blob + * via the running blob store, decompresses per desc->media_type, then + * drives oci_layer_apply (overlay mode) against stage_dir. stage_dir + * must already exist and be writable; the helper does not mkdir it. + * + * Whiteout and opaque tar entries are processed by oci_layer_apply + * with overlay semantics: ".wh." deletes upper-layer state in + * stage_dir; ".wh..wh..opq" clears the containing directory. This + * helper has no concern with caches; Plan 3 C3.3c moved all cache + * orchestration up into oci_unpack itself, which drives raw-tar + * populate via oci_unpack_layer_raw and assembles via + * oci_unpack_assemble_layer. + * + * Parameters: + * bs - blob store backing the layer payload. + * desc - layer descriptor; algo / hex / media_type / size used. + * stage_dir - destination directory, absolute path, no trailing '/'. + * stats - optional; per-layer counters are summed when non-NULL. + * meta - optional; tar uid/gid/mode entries recorded when non-NULL. + * log_label - optional; when non-NULL the helper prints + * "