diff --git a/.clang-tidy b/.clang-tidy new file mode 100644 index 0000000..389b644 --- /dev/null +++ b/.clang-tidy @@ -0,0 +1,19 @@ +--- +Checks: >- + -*, + clang-analyzer-*, + bugprone-*, + performance-*, + readability-*, + portability-*, + misc-*, + -readability-magic-numbers, + -readability-identifier-length, + -bugprone-narrowing-conversions, + -bugprone-easily-swappable-parameters +WarningsAsErrors: '' +HeaderFilterRegex: '^(include|src|port)/' +AnalyzeTemporaryDtors: false +FormatStyle: none +... + diff --git a/CODE_OF_CONDUCT.md b/.github/CODE_OF_CONDUCT.md similarity index 100% rename from CODE_OF_CONDUCT.md rename to .github/CODE_OF_CONDUCT.md diff --git a/CONTRIBUTING.md b/.github/CONTRIBUTING.md similarity index 100% rename from CONTRIBUTING.md rename to .github/CONTRIBUTING.md diff --git a/SECURITY.md b/.github/SECURITY.md similarity index 100% rename from SECURITY.md rename to .github/SECURITY.md diff --git a/SUPPORT.md b/.github/SUPPORT.md similarity index 100% rename from SUPPORT.md rename to .github/SUPPORT.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a8b827a..5676ae3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -96,3 +96,48 @@ jobs: with: name: static-analysis-style-report path: docs/results/cppcheck_style_report.txt + + clang-tidy: + runs-on: ubuntu-latest + continue-on-error: true + steps: + - uses: actions/checkout@v4 + - name: Install clang-tidy + run: sudo apt-get update && sudo apt-get install -y clang-tidy + - name: Configure (compile_commands.json) + run: cmake -S . -B build/ci-clang-tidy-linux -DCMAKE_BUILD_TYPE=Debug -DCMAKE_EXPORT_COMPILE_COMMANDS=ON + - name: Run clang-tidy (non-blocking) + run: | + clang-tidy -p build/ci-clang-tidy-linux \ + src/*.c port/posix/*.c port/ram/*.c \ + --quiet \ + | tee clang-tidy-report.txt + - name: Upload clang-tidy report + if: always() + uses: actions/upload-artifact@v4 + with: + name: clang-tidy-report + path: clang-tidy-report.txt + + coverage-linux: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install lcov + run: sudo apt-get update && sudo apt-get install -y lcov + - name: Configure (coverage) + run: cmake --preset ci-coverage-linux + - name: Build (coverage) + run: cmake --build --preset ci-coverage-linux + - name: Test (coverage) + run: ctest --preset ci-coverage-linux + - name: Capture coverage (lcov) + run: | + lcov --capture --directory build/ci-coverage-linux --output-file coverage.info + lcov --remove coverage.info '*/build/*' '*/tests/*' --ignore-errors unused --output-file coverage.info + lcov --list coverage.info + - name: Upload coverage artifact + uses: actions/upload-artifact@v4 + with: + name: coverage-lcov + path: coverage.info diff --git a/.github/workflows/nightly-fuzz.yml b/.github/workflows/nightly-fuzz.yml new file mode 100644 index 0000000..5ee284a --- /dev/null +++ b/.github/workflows/nightly-fuzz.yml @@ -0,0 +1,30 @@ +name: Nightly Fuzz + +on: + schedule: + - cron: "0 2 * * *" + workflow_dispatch: + +jobs: + fuzz-linux: + runs-on: ubuntu-latest + continue-on-error: true + steps: + - uses: actions/checkout@v4 + - name: Install clang + run: sudo apt-get update && sudo apt-get install -y clang + - name: Build fuzz harnesses + run: bash tests/fuzz/build.sh + - name: Run fuzz_wal_parser (~10 min) + run: bash tests/fuzz/run_one.sh fuzz_wal_parser 600 + - name: Upload fuzz artifacts (if any) + if: always() + uses: actions/upload-artifact@v4 + with: + name: fuzz-artifacts + path: | + crash-* + leak-* + timeout-* + oom-* + diff --git a/CMakeLists.txt b/CMakeLists.txt index 138649c..912a711 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,6 +2,7 @@ cmake_minimum_required(VERSION 3.16) project(loxdb VERSION 1.0.0 LANGUAGES C CXX) include(GNUInstallDirs) +include(CMakePackageConfigHelpers) set(CMAKE_C_STANDARD 99) set(CMAKE_C_STANDARD_REQUIRED ON) @@ -623,6 +624,25 @@ install( DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/loxdb ) +configure_package_config_file( + ${CMAKE_CURRENT_SOURCE_DIR}/cmake/loxdbConfig.cmake.in + ${CMAKE_CURRENT_BINARY_DIR}/loxdbConfig.cmake + INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/loxdb +) + +write_basic_package_version_file( + ${CMAKE_CURRENT_BINARY_DIR}/loxdbConfigVersion.cmake + VERSION ${PROJECT_VERSION} + COMPATIBILITY SameMajorVersion +) + +install( + FILES + ${CMAKE_CURRENT_BINARY_DIR}/loxdbConfig.cmake + ${CMAKE_CURRENT_BINARY_DIR}/loxdbConfigVersion.cmake + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/loxdb +) + if(LOX_BUILD_TOOLS) if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tools/lox_verify.c") add_executable(lox_verify diff --git a/CMakePresets.json b/CMakePresets.json index 5f6acbd..005f766 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -18,6 +18,22 @@ "LOX_FS_MATRIX_LONG_MAX_MS": "15000" } }, + { + "name": "ci-coverage-linux", + "displayName": "CI Coverage Linux", + "binaryDir": "${sourceDir}/build/ci-coverage-linux", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug", + "CMAKE_C_FLAGS_DEBUG": "-O0 -g --coverage", + "CMAKE_CXX_FLAGS_DEBUG": "-O0 -g --coverage", + "CMAKE_EXE_LINKER_FLAGS_DEBUG": "--coverage", + "CMAKE_SHARED_LINKER_FLAGS_DEBUG": "--coverage", + "LOX_MANAGED_STRESS_SMOKE_MAX_MS": "3000", + "LOX_MANAGED_STRESS_LONG_MAX_MS": "12000", + "LOX_FS_MATRIX_SMOKE_MAX_MS": "3500", + "LOX_FS_MATRIX_LONG_MAX_MS": "15000" + } + }, { "name": "ci-debug-windows", "displayName": "CI Debug Windows", @@ -106,6 +122,11 @@ "configurePreset": "ci-debug-linux", "configuration": "Debug" }, + { + "name": "ci-coverage-linux", + "configurePreset": "ci-coverage-linux", + "configuration": "Debug" + }, { "name": "ci-debug-windows", "configurePreset": "ci-debug-windows", @@ -146,6 +167,14 @@ "outputOnFailure": true } }, + { + "name": "ci-coverage-linux", + "configurePreset": "ci-coverage-linux", + "configuration": "Debug", + "output": { + "outputOnFailure": true + } + }, { "name": "ci-debug-windows", "configurePreset": "ci-debug-windows", diff --git a/README.md b/README.md index e6e0e4a..48c05fe 100644 --- a/README.md +++ b/README.md @@ -1,383 +1,109 @@ -![loxdb](docs/banner.svg) +![loxdb](docs/banner.svg) # loxdb -> Embedded database for microcontrollers. -> Three engines. One malloc. Zero dependencies. -> Deterministic durable storage core for MCU/embedded systems. +> Predictable-memory database for microcontrollers. KV + time-series + relational, one malloc, WAL recovery. [![CI](https://github.com/Vanderhell/loxdb/actions/workflows/ci.yml/badge.svg)](https://github.com/Vanderhell/loxdb/actions/workflows/ci.yml) [![Language: C99](https://img.shields.io/badge/language-C99-blue)](https://en.wikipedia.org/wiki/C99) [![License: MIT](https://img.shields.io/badge/license-MIT-green)](LICENSE) [![Platform: MCU | Linux | Windows | macOS](https://img.shields.io/badge/platform-MCU%20%7C%20Linux%20%7C%20Windows%20%7C%20macOS-informational)](https://github.com/Vanderhell/loxdb) -[![Tests](https://img.shields.io/badge/tests-ctest-brightgreen)](https://github.com/Vanderhell/loxdb/actions/workflows/ci.yml) +[![Tests](https://img.shields.io/badge/tests-504%20microtests-brightgreen)](docs/TEST_SUITE_SIZE.md) [![Release](https://img.shields.io/github/v/release/Vanderhell/loxdb)](https://github.com/Vanderhell/loxdb/releases) -[![Wiki](https://img.shields.io/badge/docs-wiki-blue)](https://github.com/Vanderhell/loxdb/wiki) -[![Contributing](https://img.shields.io/badge/contributions-welcome-success)](CONTRIBUTING.md) -[![Security](https://img.shields.io/badge/security-policy-important)](SECURITY.md) ## What is loxdb? loxdb is a compact embedded database written in C99 for firmware and small edge runtimes. -It combines three storage models behind one API surface: - -- KV for configuration, caches, and TTL-backed state -- Time-series for sensor samples and rolling telemetry -- Relational for small indexed tables - -The library allocates exactly once in `lox_init()`, runs without external dependencies, -and can operate either in RAM-only mode or with a storage HAL for persistence and WAL recovery. - -## Recent additions (Unreleased) - -- Runtime integrity API: `lox_selfcheck(...)` -- WCET package: - - compile-time bounds: `include/lox_wcet.h` - - analysis guide: `docs/WCET_ANALYSIS.md` -- TS logarithmic retention: - - policy: `LOX_TS_POLICY_LOG_RETAIN` - - extended registration: `lox_ts_register_ex(...)` - -## Product Contract - -- Positioning: see [PRODUCT_POSITIONING.md](docs/PRODUCT_POSITIONING.md) -- Product brief (1 page): see [PRODUCT_BRIEF.md](docs/PRODUCT_BRIEF.md) -- Profile guarantees and limits: see [PROFILE_GUARANTEES.md](docs/PROFILE_GUARANTEES.md) -- Fail-code contract: see [FAIL_CODE_CONTRACT.md](docs/FAIL_CODE_CONTRACT.md) -- Runtime error text helper: `lox_err_to_string(lox_err_t)` -- Offline verifier contract: see [OFFLINE_VERIFIER.md](docs/OFFLINE_VERIFIER.md) -- WCET analysis: see [WCET_ANALYSIS.md](docs/WCET_ANALYSIS.md) -- Safety readiness package: see [SAFETY_READINESS.md](docs/SAFETY_READINESS.md) -- Professional readiness checklist: see [PROFESSIONAL_READINESS.md](docs/PROFESSIONAL_READINESS.md) -- Footprint-min contract: see [FOOTPRINT_MIN_CONTRACT.md](docs/FOOTPRINT_MIN_CONTRACT.md) -- Latest hard verdict (currently 2026-04-19): see [hard_verdict_20260419.md](docs/results/hard_verdict_20260419.md) -- Full validation artifacts and trend dashboard: see [docs/results/](docs/results/) and [trend_dashboard.md](docs/results/trend_dashboard.md) -- Getting started (5 min): see [GETTING_STARTED_5_MIN.md](docs/GETTING_STARTED_5_MIN.md) -- Developer quickstart (10 min): see [GETTING_STARTED_DEV_10_MIN.md](docs/GETTING_STARTED_DEV_10_MIN.md) -- Limits and failures contract: see [LIMITS_AND_FAILURES.md](docs/LIMITS_AND_FAILURES.md) -- Startup decision flow: see [STARTUP_DECISION_FLOW.md](docs/STARTUP_DECISION_FLOW.md) -- Troubleshooting guide: see [TROUBLESHOOTING.md](docs/TROUBLESHOOTING.md) -- Golden hardware profiles: see [GOLDEN_PROFILES.md](docs/GOLDEN_PROFILES.md) -- Programmer manual: see [PROGRAMMER_MANUAL.md](docs/PROGRAMMER_MANUAL.md) -- Backend integration guide: see [BACKEND_INTEGRATION_GUIDE.md](docs/BACKEND_INTEGRATION_GUIDE.md) -- Port authoring guide (ESP32 reference): see [PORT_AUTHORING_GUIDE.md](docs/PORT_AUTHORING_GUIDE.md) -- Schema migration guide: see [SCHEMA_MIGRATION_GUIDE.md](docs/SCHEMA_MIGRATION_GUIDE.md) -- Full docs map: see [DOCS_MAP.md](docs/DOCS_MAP.md) -- Core/PRO docs sync plan: see [DOCS_SYNC_PLAN.md](docs/DOCS_SYNC_PLAN.md) -- Change cycle checklist: see [CHANGE_CYCLE_CHECKLIST.md](docs/CHANGE_CYCLE_CHECKLIST.md) -- Release checklist: see [RELEASE_CHECKLIST.md](docs/RELEASE_CHECKLIST.md) -- Release tag template: see [RELEASE_TAG_TEMPLATE.md](docs/RELEASE_TAG_TEMPLATE.md) - -## Project Governance - -- Contribution guide: [CONTRIBUTING.md](CONTRIBUTING.md) -- Changelog: [CHANGELOG.md](CHANGELOG.md) -- Release log: [RELEASE_LOG.md](RELEASE_LOG.md) -- Security policy: [SECURITY.md](SECURITY.md) -- Code of conduct: [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md) -- Support policy: [SUPPORT.md](SUPPORT.md) - -## Why not SQLite? - -SQLite is excellent, but it targets a different operating point. -loxdb is intentionally narrower: - -- one malloc at init, no allocator churn during normal operation -- fixed RAM budgeting across engines -- tiny integration surface for MCUs and RTOS firmware -- simpler persistence model for flash partitions and file-backed simulation - -If you need SQL, dynamic schemas, concurrent access, or large secondary indexes, use SQLite. -If you need predictable memory and embedded-first behavior, loxdb is the better fit. - -## Quick start - -**1. Add to your project:** -```cmake -add_subdirectory(loxdb) -target_link_libraries(your_app PRIVATE loxdb) -``` - -**2. Configure and initialize:** -```c -#define LOX_RAM_KB 32 -#include "lox.h" - -static lox_t db; - -lox_cfg_t cfg = { - .storage = NULL, // RAM-only; provide HAL for persistence - .now = NULL, // provide timestamp fn for TTL support -}; -lox_init(&db, &cfg); -``` - -## C++ wrapper (incremental) +It provides one unified API over three engines (KV, time-series, relational) and is designed around predictable memory behavior. +The library allocates once at `lox_init()` and runs without allocator churn during normal operation. +Persistence is optional via a small storage HAL (read/write/erase/sync), with WAL + recovery when enabled. -Header: -- `include/lox_cpp.hpp` +Test suite size: **504 microtest cases across 48 test files (+1 C++ wrapper test), organized into ~78 CTest entries including RAM-budget sweep matrices.** -Current wrapper surface: -- lifecycle: `init/deinit/flush` -- startup gating: `preflight` -- diagnostics: `stats`, `db_stats`, `kv_stats`, `ts_stats`, `rel_stats`, `effective_capacity`, `pressure` -- KV: `kv_set/kv_put/kv_get/kv_del/kv_exists/kv_iter/kv_clear/kv_purge_expired`, `admit_kv_set` -- TS: `ts_register/ts_insert/ts_last/ts_query/ts_query_buf/ts_count/ts_clear`, `admit_ts_insert` -- REL: schema/table helpers + `rel_insert/find/find_by/delete/iter/count/clear`, `admit_rel_insert` -- txn: `txn_begin/txn_commit/txn_rollback` +## Why loxdb? (When to use / when not to) -Minimal example: -```cpp -#include "lox_cpp.hpp" +| Use loxdb when you need... | Avoid loxdb when you need... | +|---|---| +| bounded RAM and predictable allocation behavior | unbounded queries / SQL flexibility | +| durability with WAL recovery on flash-like media | a full SQL database with complex query planning | +| KV + telemetry streams + small indexed tables in one library | multi-process concurrency / server database features | +| a small storage HAL integration | transparent large-object storage and advanced indexing | -loxdb::cpp::Database db; -lox_cfg_t cfg{}; -cfg.ram_kb = 32u; -if (db.init(cfg) != LOX_OK) { /* handle error */ } +## Quick start (RAM-backed) -uint8_t v = 7u, out = 0u; -db.kv_put("k", &v, 1u); -db.kv_get("k", &out, 1u); +```c +#include "lox.h" +#include "lox_port_ram.h" -db.txn_begin(); -db.kv_put("k2", &v, 1u); -db.txn_commit(); +int main(void) { + lox_t db; + lox_storage_t storage; + lox_cfg_t cfg = {0}; -db.deinit(); -``` + lox_port_ram_init(&storage, 64u * 1024u); + cfg.storage = &storage; + cfg.ram_kb = 32u; + if (lox_init(&db, &cfg) != LOX_OK) return 1; -Preflight before init: -```cpp -#include "lox_cpp.hpp" + uint8_t v = 7u, out = 0u; + lox_kv_put(&db, "k", &v, 1u); + lox_kv_get(&db, "k", &out, 1u); -lox_cfg_t cfg{}; -cfg.ram_kb = 64u; -lox_preflight_report_t rep{}; -if (loxdb::cpp::preflight(cfg, &rep) != LOX_OK) { - // use rep.status + sizing fields to pick fallback profile + lox_deinit(&db); + lox_port_ram_deinit(&storage); + return 0; } ``` -## Optional wrappers and adapter modules - -Core `loxdb` is intentionally lean. Extra wrappers/adapters are separate modules and can be toggled in CMake. +## Build & test (desktop) -Build toggles: -- `LOX_BUILD_JSON_WRAPPER` (default `ON`) -- `LOX_BUILD_IMPORT_EXPORT` (default `ON`) -- `LOX_BUILD_OPTIONAL_BACKENDS` (default `ON`) -- `LOX_BUILD_BACKEND_ALIGNED_STUB` / `LOX_BUILD_BACKEND_NAND_STUB` / `LOX_BUILD_BACKEND_EMMC_STUB` / `LOX_BUILD_BACKEND_SD_STUB` / `LOX_BUILD_BACKEND_FS_STUB` / `LOX_BUILD_BACKEND_BLOCK_STUB` - -Wrapper targets: -- `lox_json_wrapper` -- `lox_import_export` (links to `lox_json_wrapper` when available) -- `lox_backend_registry` -- `lox_backend_compat` -- `lox_backend_decision` -- `lox_backend_aligned_adapter` -- `lox_backend_managed_adapter` -- `lox_backend_fs_adapter` -- `lox_backend_open` - -Core contract: -- optional modules are not linked into `loxdb` core by default. -- `loxdb` must remain independent from optional wrapper/backend targets. - -**3. Use all three engines:** -```c -// Key-value -float temp = 23.5f; -lox_kv_put(&db, "temperature", &temp, sizeof(temp)); - -// Time-series -lox_ts_register(&db, "sensor", LOX_TS_F32, 0); -lox_ts_insert(&db, "sensor", time_now(), &temp); - -// Relational -lox_schema_t schema; -lox_schema_init(&schema, "devices", 32); -lox_schema_add(&schema, "id", LOX_COL_U16, 2, true); -lox_schema_add(&schema, "name", LOX_COL_STR, 16, false); -lox_schema_seal(&schema); -lox_table_create(&db, &schema); +```bash +cmake --preset ci-debug-linux +cmake --build --preset ci-debug-linux +ctest --preset ci-debug-linux ``` -## Configuration - -Configuration is compile-time first, with a small runtime override surface in `lox_cfg_t`. - -- `LOX_RAM_KB` sets the total heap budget -- `LOX_RAM_KV_PCT`, `LOX_RAM_TS_PCT`, `LOX_RAM_REL_PCT` set default engine slices -- `cfg.ram_kb` overrides the total budget per instance -- `cfg.kv_pct`, `cfg.ts_pct`, `cfg.rel_pct` override the slice split per instance -- `LOX_ENABLE_WAL` toggles WAL persistence when a storage HAL is present -- `cfg.wal_sync_mode` selects WAL durability/latency mode: - - `LOX_WAL_SYNC_ALWAYS` (default): sync on each append, strongest per-op durability - - `LOX_WAL_SYNC_FLUSH_ONLY`: sync on explicit `lox_flush()`, lower write latency - - see measured ESP32 tradeoffs in `bench/loxdb_esp32_s3_bench_head/README.md` ("WAL Sync Mode Decision Table") -- `LOX_LOG(level, fmt, ...)` enables internal diagnostic logging -- smallest-size variant is available as CMake target `lox_tiny` (KV-only, TS/REL/WAL disabled, weaker power-fail durability) -- strict smallest **durable** profile is available as `LOX_PROFILE_FOOTPRINT_MIN` (KV + WAL + reopen/recovery contract) - -Storage budget (separate from RAM budget): -- storage capacity comes from `lox_storage_t.capacity` (bytes) -- geometry comes from `lox_storage_t.erase_size` and `lox_storage_t.write_size` -- current fail-fast storage contract requires `erase_size > 0` and `write_size == 1` -- use `tools/lox_capacity_estimator.html` for storage/layout planning (`2/4/8/16/32 MiB` profiles) - -## KV engine - -The KV engine stores short keys with binary values and optional TTL. - -- fixed-size hash table with overwrite or reject overflow policy -- LRU eviction for `LOX_KV_POLICY_OVERWRITE` -- TTL expiration checked on access -- WAL-backed persistence for set, delete, and clear - -## Time-series engine - -The time-series engine stores named streams of `F32`, `I32`, `U32`, or raw samples. +## Three engines in 30 seconds -- one ring buffer per registered stream -- range queries by timestamp -- overflow policies: drop oldest, reject, downsample, or logarithmic retain -- per-stream extended registration via `lox_ts_register_ex(...)` -- WAL-backed persistence for inserts and stream metadata +- **KV (key-value):** config/state, binary-safe values, optional TTL, bounded by compile-time limits. +- **TS (time-series):** typed telemetry streams (`F32/I32/U32/RAW`) with timestamp range queries and retention policies. +- **REL (relational):** small fixed-schema tables with one indexed column, designed for predictable memory use. -## Relational engine +## Verified hardware -The relational engine stores small fixed schemas with packed rows. - -- one indexed column per table -- binary-search index on the indexed field -- linear scans for non-index lookups -- insertion-order iteration -- WAL-backed persistence for inserts, deletes, and table metadata - -## Storage HAL - -loxdb supports three storage modes: - -- RAM-only: `cfg.storage = NULL` -- POSIX file HAL for tests and simulation -- ESP32 partition HAL via `esp_partition_*` -- RTOS skeleton templates: `examples/freertos_port/`, `examples/zephyr_port/` -- SD + FatFS glue skeleton: `examples/sd_fatfs_port/` - -## Supported Platforms - -Verified hardware: -- ESP32-S3 N16R8 (`run_real` PASS, benchmarked) - -Commonly compatible targets: -- direct byte-write flash ports (ESP32 family, STM32H7/F4, RP2040, nRF52, SAMD51) -- aligned-write media via `lox_backend_aligned_adapter` (`write_size > 1`) - -Current hard limits: -- core durable path expects byte-write storage (`write_size == 1`) -- AVR/MSP430-class tiny targets are out of scope for current memory/storage contract +| Platform | Status | Benchmarks | +|---|---|---| +| ESP32-S3 N16R8 (16MB NOR flash, 8MB PSRAM) | Verified (KV/TS/REL + WAL recovery + power-loss scenarios) | `docs/BENCHMARKS.md` | Notes: -- latency numbers are board/flash dependent; treat all values as directional and measure on target hardware -- for full platform matrix, adapter contracts, and managed media notes, see [BACKEND_INTEGRATION_GUIDE.md](docs/BACKEND_INTEGRATION_GUIDE.md) and [PORT_AUTHORING_GUIDE.md](docs/PORT_AUTHORING_GUIDE.md) - -## Read-only diagnostics API - -System stats are exposed through read-only APIs (not user KV keys): - -- `lox_get_db_stats(...)` -- `lox_get_kv_stats(...)` -- `lox_get_ts_stats(...)` -- `lox_get_rel_stats(...)` -- `lox_get_effective_capacity(...)` -- `lox_get_pressure(...)` -- `lox_selfcheck(...)` -- `lox_admit_kv_set(...)` -- `lox_admit_ts_insert(...)` -- `lox_admit_rel_insert(...)` - -Semantics: -- `lox_admission_t.status` carries operation-level decision -- `lox_get_pressure(...)` exposes `kv/ts/rel/wal` pressure and near-full risk -- see [PROGRAMMER_MANUAL.md](docs/PROGRAMMER_MANUAL.md) for detailed field-level behavior +- Verified using the existing ESP32-S3 bench runners under `bench/`. +- Published benchmark results live in `docs/BENCHMARKS.md` (template-only until filled with real measurements). -## Migrations vs Snapshots +## Project status & roadmap -Three separate concepts: -1. schema migration API (`schema_version` + `cfg.on_migrate`) for REL tables -2. internal durable snapshot banks for WAL/compact/recovery (not public user snapshots) -3. query-time consistency checks (returns `LOX_ERR_MODIFIED` on concurrent mutation) +- Current release line: `v1.4.0` (see `CHANGELOG.md`). +- Young project focused on embedded correctness and predictable memory behavior; production use-cases and feedback are welcome. -Detailed behavior: [SCHEMA_MIGRATION_GUIDE.md](docs/SCHEMA_MIGRATION_GUIDE.md) and [PROGRAMMER_MANUAL.md](docs/PROGRAMMER_MANUAL.md) +## loxdb vs loxdb_pro -## RAM budget guide +This repository is the MIT-licensed OSS edition. A planned commercial edition (`loxdb_pro`) will live in a separate repository as additive modules on top of `loxdb` (it will not replace or relicense the MIT core). See `docs/EDITIONS.md`. -| LOX_RAM_KB | KV entries (est.) | TS samples/stream (est.) | REL rows (est.) | Typical use | -|---------------|-------------------|--------------------------|-----------------|-------------| -| 8 KB | ~3 | ~32 | ~4 | Ultra-tiny KV-focused profile | -| 16 KB | ~40 | ~136 | ~8 | Small MCU baseline | -| 32 KB | ~64 | ~1 500 | ~30 | General embedded node | -| 64 KB | ~150 | ~3 000 | ~80 | Rich sensing / control node | -| 128 KB | ~300 | ~6 000 | ~160 | MCU + external RAM | -| 256 KB | ~600 | ~12 000 | ~320 | High-retention edge node | -| 512 KB | ~1 200 | ~24 000 | ~640 | Linux embedded | -| 1024 KB | ~2 500 | ~48 000 | ~1 300 | Resource-rich MCU | -| txn staging overhead | `LOX_TXN_STAGE_KEYS * sizeof(lox_txn_stage_entry_t)` bytes | same | same | Reserved from KV slice | +## Documentation -Estimates assume default 40/40/20 RAM split and default column sizes. -Override with `LOX_RAM_KV_PCT`, `LOX_RAM_TS_PCT`, `LOX_RAM_REL_PCT`. +- Getting started: `docs/GETTING_STARTED_5_MIN.md` +- Programmer manual: `docs/PROGRAMMER_MANUAL.md` +- Backend integration: `docs/BACKEND_INTEGRATION_GUIDE.md` +- Port authoring (ESP32 reference): `docs/PORT_AUTHORING_GUIDE.md` +- Schema migration: `docs/SCHEMA_MIGRATION_GUIDE.md` +- Docs index: `docs/README.md` -Capacity planning helper: -- open `tools/lox_capacity_estimator.html` for profile-based storage/layout estimation (`2/4/8/16/32 MiB`) and rough record-fit planning. +## Contributing & support -## Design decisions and known limitations - -- single `malloc` in `lox_init()` (predictable memory, no allocator churn) -- fixed RAM slices per engine (no runtime redistribution) -- one index per REL table (secondary indexes not planned for v1.x) -- KV overwrite mode uses O(n) LRU scan -- thread safety is hook-based (`LOX_THREAD_SAFE=1` + lock callbacks) -- no built-in compression/encryption (application-layer responsibility) - -Detailed rationale: [PRODUCT_POSITIONING.md](docs/PRODUCT_POSITIONING.md) and [PROGRAMMER_MANUAL.md](docs/PROGRAMMER_MANUAL.md) - -## Test coverage - -Coverage includes KV/TS/REL behavior, WAL recovery/corruption paths, RAM-profile variants, and footprint/profile contract gates. -Current CTest inventory (as configured in CI debug presets): **76 registered tests per platform**. -CI execution volume per `ci.yml` run is higher because the same suite runs on multiple lanes: -- `build` matrix (`linux`, `windows`, `macOS`): `3 x 76 = 228` test executions -- `sanitize-linux` lane: additional `76` test executions -- total in `ci.yml`: **304 test executions** (same test set across environments/instrumentation) -Nightly soak (`nightly-soak.yml`) is benchmark-oriented (not CTest-count-oriented): -- lanes: `linux-debug`, `windows-debug` -- per lane: `3` worstcase-matrix profile runs + `3` long soak profile runs -- total per nightly run: **12 benchmark runs** (`2 x (3 + 3)`) - -See CI and deep docs for current matrix: -- [ci.yml](.github/workflows/ci.yml) -- [PROGRAMMER_MANUAL.md](docs/PROGRAMMER_MANUAL.md) -- [docs/results/](docs/results/) - -## Integration note - -loxdb is storage-focused. Transport, serialization, and cryptography are handled by surrounding application components. - -## Wiki - -GitHub Wiki source pages are stored in [`wiki/`](wiki). -That keeps documentation versioned in the main repository and ready to publish into the GitHub wiki repo. +- Contributing guide: `.github/CONTRIBUTING.md` +- Support policy: `.github/SUPPORT.md` +- Security policy: `.github/SECURITY.md` ## License -MIT. - -License details and file-level SPDX policy: - -- [LICENSE](LICENSE) -- [docs/FREE_EDITION_LICENSING.md](docs/FREE_EDITION_LICENSING.md) -- SPDX tooling: - - `tools/apply_spdx_headers.ps1` - - `tools/check_spdx_headers.ps1` - +MIT (see `LICENSE`). diff --git a/bench/loxdb_esp32_s3_bench_base/README.md b/bench/loxdb_esp32_s3_bench_base/README.md index 6d3d9d0..1554b23 100644 --- a/bench/loxdb_esp32_s3_bench_base/README.md +++ b/bench/loxdb_esp32_s3_bench_base/README.md @@ -2,7 +2,9 @@ This folder contains a terminal-driven benchmark runner for `ESP32-S3 N16R8`. -- file: `lox_esp32_s3_bench.ino` +Published results (when available): `docs/BENCHMARKS.md` + +- file: `loxdb_esp32_s3_bench_base.ino` - goal: validate core API behavior and measure latency/throughput metrics - mode: **terminal/manual trigger**; tests do not auto-run on boot diff --git a/bench/loxdb_esp32_s3_bench_base/lox.h b/bench/loxdb_esp32_s3_bench_base/lox.h new file mode 100644 index 0000000..e3f6822 --- /dev/null +++ b/bench/loxdb_esp32_s3_bench_base/lox.h @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: MIT +// Arduino sketch compatibility shim. +// The base bench folder historically carried the public header as `loxdb.h`. +// The sketch includes `lox.h`, so include the real header here. +#pragma once + +#include "loxdb.h" + diff --git a/bench/loxdb_esp32_s3_bench_base/loxdb_esp32_s3_bench_base.ino b/bench/loxdb_esp32_s3_bench_base/loxdb_esp32_s3_bench_base.ino index 750da8f..190a69f 100644 --- a/bench/loxdb_esp32_s3_bench_base/loxdb_esp32_s3_bench_base.ino +++ b/bench/loxdb_esp32_s3_bench_base/loxdb_esp32_s3_bench_base.ino @@ -10,7 +10,11 @@ extern "C" { } #if defined(ARDUINO_ARCH_ESP32) +#if defined(ARDUINO_USB_CDC_ON_BOOT) && (ARDUINO_USB_CDC_ON_BOOT) +#define MDB_CONSOLE Serial +#else #define MDB_CONSOLE Serial0 +#endif #else #define MDB_CONSOLE Serial #endif diff --git a/bench/loxdb_esp32_s3_bench_base/src/lox_arena.h b/bench/loxdb_esp32_s3_bench_base/src/lox_arena.h new file mode 100644 index 0000000..9d2ce25 --- /dev/null +++ b/bench/loxdb_esp32_s3_bench_base/src/lox_arena.h @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: MIT +#pragma once + +#include "microdb_arena.h" + diff --git a/bench/loxdb_esp32_s3_bench_base/src/lox_crc.h b/bench/loxdb_esp32_s3_bench_base/src/lox_crc.h new file mode 100644 index 0000000..369d19f --- /dev/null +++ b/bench/loxdb_esp32_s3_bench_base/src/lox_crc.h @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: MIT +#pragma once + +#include "microdb_crc.h" + diff --git a/bench/loxdb_esp32_s3_bench_base/src/lox_internal.h b/bench/loxdb_esp32_s3_bench_base/src/lox_internal.h new file mode 100644 index 0000000..3f5ddb1 --- /dev/null +++ b/bench/loxdb_esp32_s3_bench_base/src/lox_internal.h @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: MIT +// Compatibility shim: base bench sources are stored as `microdb_*` but include `lox_*`. +#pragma once + +#include "microdb_internal.h" + diff --git a/bench/loxdb_esp32_s3_bench_base/src/lox_lock.h b/bench/loxdb_esp32_s3_bench_base/src/lox_lock.h new file mode 100644 index 0000000..6b8d669 --- /dev/null +++ b/bench/loxdb_esp32_s3_bench_base/src/lox_lock.h @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: MIT +#pragma once + +#include "microdb_lock.h" + diff --git a/bench/loxdb_esp32_s3_bench_head/README.md b/bench/loxdb_esp32_s3_bench_head/README.md index acd36b0..30fa15c 100644 --- a/bench/loxdb_esp32_s3_bench_head/README.md +++ b/bench/loxdb_esp32_s3_bench_head/README.md @@ -2,6 +2,8 @@ This folder contains a terminal-driven benchmark runner for `ESP32-S3 N16R8`. +Published results (when available): `docs/BENCHMARKS.md` + - file: `lox_esp32_s3_bench.ino` - goal: validate core API behavior and measure latency/throughput metrics - mode: **terminal/manual trigger**; tests do not auto-run on boot diff --git a/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench.ino b/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench.ino index 192b417..6f9a5f1 100644 --- a/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench.ino +++ b/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench.ino @@ -13,7 +13,11 @@ extern "C" { } #if defined(ARDUINO_ARCH_ESP32) +#if defined(ARDUINO_USB_CDC_ON_BOOT) && (ARDUINO_USB_CDC_ON_BOOT) +#define MDB_CONSOLE Serial +#else #define MDB_CONSOLE Serial0 +#endif #else #define MDB_CONSOLE Serial #endif diff --git a/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/lox.h b/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/lox.h new file mode 100644 index 0000000..e51c09e --- /dev/null +++ b/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/lox.h @@ -0,0 +1,527 @@ +// SPDX-License-Identifier: MIT +#ifndef LOX_H +#define LOX_H + +#include +#include +#include + +#ifndef LOX_PROFILE_CORE_HIMEM +#define LOX_PROFILE_CORE_HIMEM 1 +#endif + +#ifndef LOX_PROFILE_CORE_MIN +#define LOX_PROFILE_CORE_MIN 0 +#endif +#ifndef LOX_PROFILE_CORE_WAL +#define LOX_PROFILE_CORE_WAL 0 +#endif +#ifndef LOX_PROFILE_CORE_PERF +#define LOX_PROFILE_CORE_PERF 0 +#endif +#ifndef LOX_PROFILE_CORE_HIMEM +#define LOX_PROFILE_CORE_HIMEM 0 +#endif +#ifndef LOX_PROFILE_FOOTPRINT_MIN +#define LOX_PROFILE_FOOTPRINT_MIN 0 +#endif + +#if (LOX_PROFILE_CORE_MIN + LOX_PROFILE_CORE_WAL + LOX_PROFILE_CORE_PERF + LOX_PROFILE_CORE_HIMEM + LOX_PROFILE_FOOTPRINT_MIN) > 1 +#error "Only one LOX_PROFILE_* profile may be enabled" +#endif +#if (LOX_PROFILE_CORE_MIN + LOX_PROFILE_CORE_WAL + LOX_PROFILE_CORE_PERF + LOX_PROFILE_CORE_HIMEM + LOX_PROFILE_FOOTPRINT_MIN) == 0 +#undef LOX_PROFILE_CORE_WAL +#define LOX_PROFILE_CORE_WAL 1 +#endif + +#if LOX_PROFILE_FOOTPRINT_MIN +#ifndef LOX_RAM_KB +#define LOX_RAM_KB 8u +#endif +#ifndef LOX_ENABLE_KV +#define LOX_ENABLE_KV 1 +#endif +#ifndef LOX_ENABLE_TS +#define LOX_ENABLE_TS 0 +#endif +#ifndef LOX_ENABLE_REL +#define LOX_ENABLE_REL 0 +#endif +#ifndef LOX_ENABLE_WAL +#define LOX_ENABLE_WAL 1 +#endif +#ifndef LOX_KV_MAX_KEYS +#define LOX_KV_MAX_KEYS 16u +#endif +#ifndef LOX_TXN_STAGE_KEYS +#define LOX_TXN_STAGE_KEYS 2u +#endif +#ifndef LOX_KV_KEY_MAX_LEN +#define LOX_KV_KEY_MAX_LEN 16u +#endif +#ifndef LOX_KV_VAL_MAX_LEN +#define LOX_KV_VAL_MAX_LEN 64u +#endif +#ifndef LOX_TS_MAX_STREAMS +#define LOX_TS_MAX_STREAMS 1u +#endif +#ifndef LOX_REL_MAX_TABLES +#define LOX_REL_MAX_TABLES 1u +#endif +#ifndef LOX_REL_MAX_COLS +#define LOX_REL_MAX_COLS 1u +#endif +#endif + +#if LOX_PROFILE_CORE_MIN +#ifndef LOX_RAM_KB +#define LOX_RAM_KB 32u +#endif +#ifndef LOX_KV_MAX_KEYS +#define LOX_KV_MAX_KEYS 48u +#endif +#ifndef LOX_TS_MAX_STREAMS +#define LOX_TS_MAX_STREAMS 4u +#endif +#ifndef LOX_REL_MAX_TABLES +#define LOX_REL_MAX_TABLES 2u +#endif +#ifndef LOX_REL_MAX_COLS +#define LOX_REL_MAX_COLS 8u +#endif +#endif + +#if LOX_PROFILE_CORE_WAL +#ifndef LOX_RAM_KB +#define LOX_RAM_KB 32u +#endif +#ifndef LOX_KV_MAX_KEYS +#define LOX_KV_MAX_KEYS 64u +#endif +#endif + +#if LOX_PROFILE_CORE_PERF +#ifndef LOX_RAM_KB +#define LOX_RAM_KB 64u +#endif +#ifndef LOX_KV_MAX_KEYS +#define LOX_KV_MAX_KEYS 128u +#endif +#endif + +#if LOX_PROFILE_CORE_HIMEM +#ifndef LOX_RAM_KB +#define LOX_RAM_KB 128u +#endif +#ifndef LOX_KV_MAX_KEYS +#define LOX_KV_MAX_KEYS 256u +#endif +#endif + +#ifndef LOX_RAM_KB +#define LOX_RAM_KB 32u +#endif +#ifndef LOX_ENABLE_KV +#define LOX_ENABLE_KV 1 +#endif +#ifndef LOX_ENABLE_TS +#define LOX_ENABLE_TS 1 +#endif +#ifndef LOX_ENABLE_REL +#define LOX_ENABLE_REL 1 +#endif +#ifndef LOX_RAM_KV_PCT +#define LOX_RAM_KV_PCT 40u +#endif +#ifndef LOX_RAM_TS_PCT +#define LOX_RAM_TS_PCT 40u +#endif +#ifndef LOX_RAM_REL_PCT +#define LOX_RAM_REL_PCT 20u +#endif +#ifndef LOX_KV_MAX_KEYS +#define LOX_KV_MAX_KEYS 64u +#endif +#ifndef LOX_KV_KEY_MAX_LEN +#define LOX_KV_KEY_MAX_LEN 32u +#endif +#ifndef LOX_KV_VAL_MAX_LEN +#define LOX_KV_VAL_MAX_LEN 128u +#endif +/* Transaction staging reserves this many KV slots from kv_arena at init time. */ +#ifndef LOX_TXN_STAGE_KEYS +#define LOX_TXN_STAGE_KEYS 8u +#endif +#ifndef LOX_KV_ENABLE_TTL +#define LOX_KV_ENABLE_TTL 1 +#endif +#define LOX_KV_POLICY_OVERWRITE 0u +#define LOX_KV_POLICY_REJECT 1u +#ifndef LOX_KV_OVERFLOW_POLICY +#define LOX_KV_OVERFLOW_POLICY LOX_KV_POLICY_OVERWRITE +#endif +#ifndef LOX_TS_MAX_STREAMS +#define LOX_TS_MAX_STREAMS 8u +#endif +#ifndef LOX_TS_STREAM_NAME_LEN +#define LOX_TS_STREAM_NAME_LEN 16u +#endif +#ifndef LOX_TS_RAW_MAX +#define LOX_TS_RAW_MAX 16u +#endif +#define LOX_TS_POLICY_DROP_OLDEST 0u +#define LOX_TS_POLICY_REJECT 1u +#define LOX_TS_POLICY_DOWNSAMPLE 2u +#ifndef LOX_TS_OVERFLOW_POLICY +#define LOX_TS_OVERFLOW_POLICY LOX_TS_POLICY_DROP_OLDEST +#endif +#ifndef LOX_REL_MAX_TABLES +#define LOX_REL_MAX_TABLES 4u +#endif +#ifndef LOX_REL_MAX_COLS +#define LOX_REL_MAX_COLS 16u +#endif +#ifndef LOX_REL_COL_NAME_LEN +#define LOX_REL_COL_NAME_LEN 16u +#endif +#ifndef LOX_REL_TABLE_NAME_LEN +#define LOX_REL_TABLE_NAME_LEN 16u +#endif +#ifndef LOX_ENABLE_WAL +#define LOX_ENABLE_WAL 1 +#endif +#ifndef LOX_TIMESTAMP_TYPE +#define LOX_TIMESTAMP_TYPE uint32_t +#endif +#ifndef LOX_THREAD_SAFE +#define LOX_THREAD_SAFE 0 +#endif + +/* Debug logging + * Define LOX_LOG before #include "lox.h" to enable internal logging. + * Default is a no-op with zero production overhead. + * + * Example (printf): + * #define LOX_LOG(level, fmt, ...) \ + * printf("[loxdb][%s] " fmt "\n", level, ##__VA_ARGS__) + * + * Example (ESP-IDF): + * #define LOX_LOG(level, fmt, ...) \ + * ESP_LOGI("loxdb", "[%s] " fmt, level, ##__VA_ARGS__) + */ +#ifndef LOX_LOG +#define LOX_LOG(level, fmt, ...) ((void)0) +#endif + +/* Optional platform I/O hooks for aligned/DMA-friendly integrations. + * Hooks must not change persistence semantics; defaults are strict no-op. + */ +#ifndef LOX_IO_BEFORE_READ +#define LOX_IO_BEFORE_READ(offset, len) ((void)(offset), (void)(len)) +#endif +#ifndef LOX_IO_AFTER_READ +#define LOX_IO_AFTER_READ(offset, len, rc) ((void)(offset), (void)(len), (void)(rc)) +#endif +#ifndef LOX_IO_BEFORE_WRITE +#define LOX_IO_BEFORE_WRITE(offset, len) ((void)(offset), (void)(len)) +#endif +#ifndef LOX_IO_AFTER_WRITE +#define LOX_IO_AFTER_WRITE(offset, len, rc) ((void)(offset), (void)(len), (void)(rc)) +#endif +#ifndef LOX_IO_BEFORE_ERASE +#define LOX_IO_BEFORE_ERASE(offset, len) ((void)(offset), (void)(len)) +#endif +#ifndef LOX_IO_AFTER_ERASE +#define LOX_IO_AFTER_ERASE(offset, len, rc) ((void)(offset), (void)(len), (void)(rc)) +#endif +#ifndef LOX_IO_BEFORE_SYNC +#define LOX_IO_BEFORE_SYNC() ((void)0) +#endif +#ifndef LOX_IO_AFTER_SYNC +#define LOX_IO_AFTER_SYNC(rc) ((void)(rc)) +#endif + +#define LOX_STATIC_ASSERT(name, expr) typedef char lox_static_assert_##name[(expr) ? 1 : -1] + +LOX_STATIC_ASSERT(ram_pct_sum, (LOX_RAM_KV_PCT + LOX_RAM_TS_PCT + LOX_RAM_REL_PCT) == 100u); +LOX_STATIC_ASSERT(ram_kb_min, LOX_RAM_KB >= 8u); +LOX_STATIC_ASSERT(ram_kb_max, LOX_RAM_KB <= 4096u); +LOX_STATIC_ASSERT(txn_stage_lt_kv_keys, LOX_TXN_STAGE_KEYS < LOX_KV_MAX_KEYS); + +typedef LOX_TIMESTAMP_TYPE lox_timestamp_t; + +#ifndef LOX_HANDLE_SIZE +#define LOX_HANDLE_SIZE 8192u +#endif +#ifndef LOX_SCHEMA_SIZE +#define LOX_SCHEMA_SIZE 880u +#endif +#ifndef LOX_REL_INDEX_KEY_MAX +#define LOX_REL_INDEX_KEY_MAX 16u +#endif + +typedef struct { + uint8_t _opaque[LOX_HANDLE_SIZE]; +} lox_t; + +typedef struct { + uint16_t schema_version; + uintptr_t _align; + uint8_t _opaque[LOX_SCHEMA_SIZE]; +} lox_schema_t; + +typedef struct lox_table_s lox_table_t; + +typedef enum { + LOX_OK = 0, + LOX_ERR_INVALID = -1, + LOX_ERR_NO_MEM = -2, + LOX_ERR_FULL = -3, + LOX_ERR_NOT_FOUND = -4, + LOX_ERR_EXPIRED = -5, + LOX_ERR_STORAGE = -6, + LOX_ERR_CORRUPT = -7, + LOX_ERR_SEALED = -8, + LOX_ERR_EXISTS = -9, + LOX_ERR_DISABLED = -10, + LOX_ERR_OVERFLOW = -11, + LOX_ERR_SCHEMA = -12, + LOX_ERR_TXN_ACTIVE = -13, + LOX_ERR_MODIFIED = -14 +} lox_err_t; + +/* Returns a stable symbolic name for a loxdb error code. + * Unknown values return "LOX_ERR_UNKNOWN". + */ +const char *lox_err_to_string(lox_err_t err); + +typedef struct { + /* Legacy aggregate stats (kept for backward compatibility). */ + uint32_t kv_entries_used; + uint32_t kv_entries_max; + uint8_t kv_fill_pct; + uint32_t kv_collision_count; + uint32_t kv_eviction_count; + uint32_t ts_streams_registered; + uint32_t ts_samples_total; + uint8_t ts_fill_pct; + uint32_t wal_bytes_used; + uint32_t wal_bytes_total; + uint8_t wal_fill_pct; + uint32_t rel_tables_count; + uint32_t rel_rows_total; +} lox_stats_t; + +typedef struct { + uint32_t effective_capacity_bytes; + uint32_t wal_bytes_total; + uint32_t wal_bytes_used; + uint8_t wal_fill_pct; + /* Runtime-only counters; reset on each successful lox_init. */ + uint32_t compact_count; + uint32_t reopen_count; + uint32_t recovery_count; + /* Sticky last non-OK runtime operation status since init. */ + lox_err_t last_runtime_error; + /* Last status produced by open/recovery path in current process lifetime. */ + lox_err_t last_recovery_status; + uint32_t active_generation; + uint32_t active_bank; +} lox_db_stats_t; + +typedef struct { + uint32_t live_keys; + uint32_t collisions; + uint32_t evictions; + uint32_t tombstones; + uint32_t value_bytes_used; + uint8_t fill_pct; +} lox_kv_stats_t; + +typedef struct { + uint32_t stream_count; + uint32_t retained_samples; + uint32_t dropped_samples; + uint8_t fill_pct; +} lox_ts_stats_t; + +typedef struct { + uint32_t table_count; + uint32_t rows_live; + uint32_t rows_free; + uint32_t indexed_tables; + uint32_t index_entries; +} lox_rel_stats_t; + +typedef struct { + uint32_t kv_entries_usable; + uint32_t kv_entries_free; + uint32_t kv_value_bytes_usable; + uint32_t kv_value_bytes_free_now; + uint32_t ts_samples_usable; + uint32_t ts_samples_retained; + uint32_t ts_samples_free; + uint32_t wal_budget_total; + uint32_t wal_budget_used; + uint32_t wal_budget_free; + uint32_t wal_safety_reserved; + uint32_t compact_threshold_pct; + uint32_t limiting_flags; +} lox_effective_capacity_t; + +typedef struct { + uint8_t kv_fill_pct; + uint8_t ts_fill_pct; + uint8_t rel_fill_pct; + uint8_t wal_fill_pct; + uint8_t compact_pressure_pct; + uint8_t near_full_risk_pct; + uint32_t risk_flags; +} lox_pressure_t; + +#define LOX_CAP_LIMIT_NONE 0u +#define LOX_CAP_LIMIT_KV_ENTRIES (1u << 0) +#define LOX_CAP_LIMIT_KV_VALUE_BYTES (1u << 1) +#define LOX_CAP_LIMIT_TS_SAMPLES (1u << 2) +#define LOX_CAP_LIMIT_WAL_BUDGET (1u << 3) +#define LOX_CAP_LIMIT_STORAGE_DISABLED (1u << 4) + +typedef struct { + lox_err_t status; + uint8_t would_compact; + uint8_t would_degrade; + uint8_t deterministic_budget_ok; + uint32_t required_bytes; + uint32_t available_bytes; + uint32_t required_wal_bytes; + uint32_t wal_bytes_free; +} lox_admission_t; + +typedef enum { + LOX_TS_F32 = 0, + LOX_TS_I32 = 1, + LOX_TS_U32 = 2, + LOX_TS_RAW = 3 +} lox_ts_type_t; + +typedef enum { + LOX_COL_U8 = 0, + LOX_COL_U16 = 1, + LOX_COL_U32 = 2, + LOX_COL_U64 = 3, + LOX_COL_I8 = 4, + LOX_COL_I16 = 5, + LOX_COL_I32 = 6, + LOX_COL_I64 = 7, + LOX_COL_F32 = 8, + LOX_COL_F64 = 9, + LOX_COL_BOOL = 10, + LOX_COL_STR = 11, + LOX_COL_BLOB = 12 +} lox_col_type_t; + +typedef struct { + lox_err_t (*read)(void *ctx, uint32_t offset, void *buf, size_t len); + lox_err_t (*write)(void *ctx, uint32_t offset, const void *buf, size_t len); + lox_err_t (*erase)(void *ctx, uint32_t offset); + lox_err_t (*sync)(void *ctx); + uint32_t capacity; + /* Storage contract (validated at lox_init): + * - erase_size must be > 0 + * - write_size must be exactly 1 in current releases + * (write_size > 1 is not yet supported and fails fast with LOX_ERR_INVALID) + */ + uint32_t erase_size; + uint32_t write_size; + void *ctx; +} lox_storage_t; + +#define LOX_WAL_SYNC_ALWAYS 0u +#define LOX_WAL_SYNC_FLUSH_ONLY 1u + +typedef struct { + lox_storage_t *storage; + uint32_t ram_kb; + lox_timestamp_t (*now)(void); + uint8_t kv_pct; + uint8_t ts_pct; + uint8_t rel_pct; + void *(*lock_create)(void); + void (*lock)(void *hdl); + void (*unlock)(void *hdl); + void (*lock_destroy)(void *hdl); + uint8_t wal_compact_auto; + uint8_t wal_compact_threshold_pct; + uint8_t wal_sync_mode; + lox_err_t (*on_migrate)(lox_t *db, const char *table_name, uint16_t old_version, uint16_t new_version); +} lox_cfg_t; + +typedef struct { + lox_timestamp_t ts; + union { + float f32; + int32_t i32; + uint32_t u32; + uint8_t raw[LOX_TS_RAW_MAX]; + } v; +} lox_ts_sample_t; + +lox_err_t lox_init(lox_t *db, const lox_cfg_t *cfg); +lox_err_t lox_deinit(lox_t *db); +lox_err_t lox_flush(lox_t *db); +lox_err_t lox_stats(const lox_t *db, lox_stats_t *out); +lox_err_t lox_inspect(lox_t *db, lox_stats_t *out); +lox_err_t lox_get_db_stats(lox_t *db, lox_db_stats_t *out); +lox_err_t lox_get_kv_stats(lox_t *db, lox_kv_stats_t *out); +lox_err_t lox_get_ts_stats(lox_t *db, lox_ts_stats_t *out); +lox_err_t lox_get_rel_stats(lox_t *db, lox_rel_stats_t *out); +lox_err_t lox_get_effective_capacity(lox_t *db, lox_effective_capacity_t *out); +lox_err_t lox_get_pressure(lox_t *db, lox_pressure_t *out); +lox_err_t lox_admit_kv_set(lox_t *db, const char *key, size_t val_len, lox_admission_t *out); +lox_err_t lox_admit_ts_insert(lox_t *db, const char *stream_name, size_t sample_len, lox_admission_t *out); +lox_err_t lox_admit_rel_insert(lox_t *db, const char *table_name, size_t row_len, lox_admission_t *out); +lox_err_t lox_compact(lox_t *db); + +typedef bool (*lox_kv_iter_cb_t)(const char *key, const void *val, size_t val_len, uint32_t ttl_remaining, void *ctx); +lox_err_t lox_kv_set(lox_t *db, const char *key, const void *val, size_t len, uint32_t ttl); +lox_err_t lox_kv_get(lox_t *db, const char *key, void *buf, size_t buf_len, size_t *out_len); +lox_err_t lox_kv_del(lox_t *db, const char *key); +lox_err_t lox_kv_exists(lox_t *db, const char *key); +lox_err_t lox_kv_iter(lox_t *db, lox_kv_iter_cb_t cb, void *ctx); +lox_err_t lox_kv_purge_expired(lox_t *db); +lox_err_t lox_kv_clear(lox_t *db); +#define lox_kv_put(db, key, val, len) lox_kv_set((db), (key), (val), (len), 0u) +lox_err_t lox_txn_begin(lox_t *db); +lox_err_t lox_txn_commit(lox_t *db); +lox_err_t lox_txn_rollback(lox_t *db); + +typedef bool (*lox_ts_query_cb_t)(const lox_ts_sample_t *sample, void *ctx); +lox_err_t lox_ts_register(lox_t *db, const char *name, lox_ts_type_t type, size_t raw_size); +lox_err_t lox_ts_insert(lox_t *db, const char *name, lox_timestamp_t ts, const void *val); +lox_err_t lox_ts_last(lox_t *db, const char *name, lox_ts_sample_t *out); +lox_err_t lox_ts_query(lox_t *db, const char *name, lox_timestamp_t from, lox_timestamp_t to, lox_ts_query_cb_t cb, void *ctx); +lox_err_t lox_ts_query_buf(lox_t *db, const char *name, lox_timestamp_t from, lox_timestamp_t to, lox_ts_sample_t *buf, size_t max_count, size_t *out_count); +lox_err_t lox_ts_count(lox_t *db, const char *name, lox_timestamp_t from, lox_timestamp_t to, size_t *out_count); +lox_err_t lox_ts_clear(lox_t *db, const char *name); + +typedef bool (*lox_rel_iter_cb_t)(const void *row_buf, void *ctx); +lox_err_t lox_schema_init(lox_schema_t *schema, const char *name, uint32_t max_rows); +lox_err_t lox_schema_add(lox_schema_t *schema, const char *col_name, lox_col_type_t type, size_t size, bool is_index); +lox_err_t lox_schema_seal(lox_schema_t *schema); +lox_err_t lox_table_create(lox_t *db, lox_schema_t *schema); +lox_err_t lox_table_get(lox_t *db, const char *name, lox_table_t **out_table); +/* Pure metadata helper; no db handle, no internal DB lock. */ +size_t lox_table_row_size(const lox_table_t *table); +/* Row buffer formatter/parser helpers; no db handle, no internal DB lock. */ +lox_err_t lox_row_set(const lox_table_t *table, void *row_buf, const char *col_name, const void *val); +lox_err_t lox_row_get(const lox_table_t *table, const void *row_buf, const char *col_name, void *out, size_t *out_len); +lox_err_t lox_rel_insert(lox_t *db, lox_table_t *table, const void *row_buf); +lox_err_t lox_rel_find(lox_t *db, lox_table_t *table, const void *search_val, lox_rel_iter_cb_t cb, void *ctx); +lox_err_t lox_rel_find_by(lox_t *db, lox_table_t *table, const char *col_name, const void *search_val, void *out_buf); +lox_err_t lox_rel_delete(lox_t *db, lox_table_t *table, const void *search_val, uint32_t *out_deleted); +lox_err_t lox_rel_iter(lox_t *db, lox_table_t *table, lox_rel_iter_cb_t cb, void *ctx); +/* Table metadata query helper; no db handle, no internal DB lock. */ +lox_err_t lox_rel_count(const lox_table_t *table, uint32_t *out_count); +lox_err_t lox_rel_clear(lox_t *db, lox_table_t *table); + +#endif diff --git a/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/lox_esp32_s3_bench.ino b/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/lox_esp32_s3_bench.ino new file mode 100644 index 0000000..6f9a5f1 --- /dev/null +++ b/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/lox_esp32_s3_bench.ino @@ -0,0 +1,1234 @@ +#include +#include +#include +#if defined(ARDUINO_ARCH_ESP32) +#include "esp_heap_caps.h" +#endif + +#define LOX_PROFILE_CORE_HIMEM 1 +extern "C" { +#include "lox.h" +#include "lox_json_wrapper.h" +#include "lox_import_export.h" +} + +#if defined(ARDUINO_ARCH_ESP32) +#if defined(ARDUINO_USB_CDC_ON_BOOT) && (ARDUINO_USB_CDC_ON_BOOT) +#define MDB_CONSOLE Serial +#else +#define MDB_CONSOLE Serial0 +#endif +#else +#define MDB_CONSOLE Serial +#endif + +#define BENCH_STORAGE_BYTES (512u * 1024u) +#define BENCH_STORAGE_ERASE 4096u +#define BENCH_MAX_LAT_SAMPLES 2048u +#define BENCH_MAX_LAST_METRICS 24u +#define BENCH_WAL_COLD_OPS 64u + +typedef struct { + const char *name; + uint32_t ram_kb; + uint8_t kv_pct; + uint8_t ts_pct; + uint8_t rel_pct; + uint8_t wal_threshold_pct; + uint32_t kv_ops; + uint32_t ts_ops; + uint32_t rel_rows; + uint32_t wal_ops; + uint32_t wal_key_span; + uint32_t wal_val_bytes; + uint32_t pace_every_ops; + uint32_t pace_us; + uint32_t flush_every_ops; +} bench_profile_t; + +typedef struct { + const char *name; + uint32_t ops; + uint64_t total_us; + uint64_t bytes; + float avg_us; + float ops_per_s; + float mb_per_s; + uint32_t min_us; + uint32_t p50_us; + uint32_t p95_us; + uint32_t max_us; + uint32_t max_op_approx; + uint32_t samples; + uint32_t spike_gt_1ms; + uint32_t spike_gt_5ms; + uint32_t first_spike_1ms_op; + uint32_t first_spike_5ms_op; + float max_over_p50; + int32_t heap_delta; +} bench_metric_t; + +typedef struct { + uint8_t *bytes; + size_t size; +} bench_storage_ctx_t; + +static const bench_profile_t g_profiles[] = { + {"quick", 128u, 40u, 40u, 20u, 75u, 96u, 256u, 160u, 400u, 40u, 16u, 0u, 0u, 0u}, + {"deterministic", 224u, 45u, 35u, 20u, 70u, 192u, 384u, 240u, 700u, 140u, 24u, 1u, 12u, 0u}, + {"balanced", 256u, 40u, 40u, 20u, 75u, 320u, 640u, 500u, 1200u, 200u, 32u, 0u, 0u, 0u}, + {"stress", 320u, 45u, 35u, 20u, 80u, 900u, 2400u, 1200u, 3200u, 320u, 64u, 0u, 0u, 0u}, +}; + +static bench_storage_ctx_t g_store_ctx; +static lox_storage_t g_storage; +static lox_t g_db; +static size_t g_profile_idx = 2u; + +static volatile uint32_t g_migrate_calls = 0u; +static volatile uint16_t g_migrate_old = 0u; +static volatile uint16_t g_migrate_new = 0u; + +static uint32_t g_lat[BENCH_MAX_LAT_SAMPLES]; +static bench_metric_t g_last[BENCH_MAX_LAST_METRICS]; +static size_t g_last_count = 0u; +static void reset_db_and_open(bool wipe); +static bool g_paced_mode = false; + +static const bench_profile_t *P(void) { return &g_profiles[g_profile_idx]; } + +static uint32_t heap_free_8bit(void) { +#if defined(ARDUINO_ARCH_ESP32) + return (uint32_t)heap_caps_get_free_size(MALLOC_CAP_8BIT); +#else + return 0u; +#endif +} + +static lox_timestamp_t bench_now(void) { return (lox_timestamp_t)millis(); } + +static lox_err_t st_read(void *ctx, uint32_t off, void *buf, size_t len) { + bench_storage_ctx_t *s = (bench_storage_ctx_t *)ctx; + if (s == NULL || s->bytes == NULL || buf == NULL || (off + len) > s->size) return LOX_ERR_INVALID; + memcpy(buf, &s->bytes[off], len); + return LOX_OK; +} + +static lox_err_t st_write(void *ctx, uint32_t off, const void *buf, size_t len) { + bench_storage_ctx_t *s = (bench_storage_ctx_t *)ctx; + if (s == NULL || s->bytes == NULL || buf == NULL || (off + len) > s->size) return LOX_ERR_INVALID; + memcpy(&s->bytes[off], buf, len); + return LOX_OK; +} + +static lox_err_t st_erase(void *ctx, uint32_t off) { + bench_storage_ctx_t *s = (bench_storage_ctx_t *)ctx; + if (s == NULL || s->bytes == NULL || off >= s->size || (off + BENCH_STORAGE_ERASE) > s->size) return LOX_ERR_INVALID; + memset(&s->bytes[off], 0xFF, BENCH_STORAGE_ERASE); + return LOX_OK; +} + +static lox_err_t st_sync(void *ctx) { + (void)ctx; + return LOX_OK; +} + +static bool storage_alloc(void) { + if (g_store_ctx.bytes != NULL) return true; +#if defined(ARDUINO_ARCH_ESP32) + g_store_ctx.bytes = (uint8_t *)heap_caps_malloc(BENCH_STORAGE_BYTES, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); + if (g_store_ctx.bytes == NULL) g_store_ctx.bytes = (uint8_t *)heap_caps_malloc(BENCH_STORAGE_BYTES, MALLOC_CAP_8BIT); +#else + g_store_ctx.bytes = (uint8_t *)malloc(BENCH_STORAGE_BYTES); +#endif + if (g_store_ctx.bytes == NULL) return false; + g_store_ctx.size = BENCH_STORAGE_BYTES; + memset(g_store_ctx.bytes, 0xFF, g_store_ctx.size); + return true; +} + +static void storage_reset(void) { + if (g_store_ctx.bytes != NULL) memset(g_store_ctx.bytes, 0xFF, g_store_ctx.size); + memset(&g_storage, 0, sizeof(g_storage)); + g_storage.read = st_read; + g_storage.write = st_write; + g_storage.erase = st_erase; + g_storage.sync = st_sync; + g_storage.capacity = (uint32_t)g_store_ctx.size; + g_storage.erase_size = BENCH_STORAGE_ERASE; + g_storage.write_size = 1u; + g_storage.ctx = &g_store_ctx; +} + +static lox_err_t on_migrate(lox_t *db, const char *name, uint16_t old_v, uint16_t new_v) { + (void)db; + (void)name; + g_migrate_calls++; + g_migrate_old = old_v; + g_migrate_new = new_v; + return LOX_OK; +} + +static lox_err_t db_open(bool wipe, bool with_mig) { + lox_cfg_t cfg; + if (wipe) storage_reset(); + memset(&cfg, 0, sizeof(cfg)); + cfg.storage = &g_storage; + cfg.ram_kb = P()->ram_kb; + cfg.now = bench_now; + cfg.kv_pct = P()->kv_pct; + cfg.ts_pct = P()->ts_pct; + cfg.rel_pct = P()->rel_pct; + cfg.wal_compact_auto = 0u; + cfg.wal_compact_threshold_pct = P()->wal_threshold_pct; + cfg.wal_sync_mode = LOX_WAL_SYNC_FLUSH_ONLY; + cfg.on_migrate = with_mig ? on_migrate : NULL; + return lox_init(&g_db, &cfg); +} + +static int cmp_u32(const void *a, const void *b) { + uint32_t x = *(const uint32_t *)a; + uint32_t y = *(const uint32_t *)b; + return (x > y) - (x < y); +} + +static uint32_t pct(const uint32_t *arr, uint32_t n, uint32_t p) { + if (n == 0u) return 0u; + return arr[((uint64_t)(n - 1u) * p) / 100u]; +} + +static uint32_t sample_stride(uint32_t ops) { + if (ops == 0u || ops <= BENCH_MAX_LAT_SAMPLES) return 1u; + return (ops + BENCH_MAX_LAT_SAMPLES - 1u) / BENCH_MAX_LAT_SAMPLES; +} + +static void clear_metrics(void) { + g_last_count = 0u; + memset(g_last, 0, sizeof(g_last)); +} + +static void print_effective_capacity(void) { + lox_stats_t st; + memset(&st, 0, sizeof(st)); + if (lox_inspect(&g_db, &st) != LOX_OK) return; + MDB_CONSOLE.printf("[EFFECTIVE] kv_capacity=%lu (target=%lu) wal_total=%luB\n", (unsigned long)st.kv_entries_max, + (unsigned long)P()->kv_ops, (unsigned long)st.wal_bytes_total); +} + +static void print_phase_split(const char *name, uint32_t cold_ops, uint64_t cold_total, uint32_t steady_ops, uint64_t steady_total) { + float cold_avg = (cold_ops > 0u) ? ((float)cold_total / (float)cold_ops) : 0.0f; + float steady_avg = (steady_ops > 0u) ? ((float)steady_total / (float)steady_ops) : 0.0f; + MDB_CONSOLE.printf("[PHASE] %-16s cold_ops=%lu cold_avg=%.3f us steady_ops=%lu steady_avg=%.3f us\n", name, + (unsigned long)cold_ops, (double)cold_avg, (unsigned long)steady_ops, (double)steady_avg); +} + +static uint32_t wal_min_steady_ops(void) { + if (strcmp(P()->name, "deterministic") == 0) return 256u; + if (strcmp(P()->name, "quick") == 0) return 64u; + if (strcmp(P()->name, "balanced") == 0) return 128u; + return 256u; +} + +static bool is_deterministic_profile(void) { + return strcmp(P()->name, "deterministic") == 0; +} + +static void maybe_apply_write_control(uint32_t op_index) { + const bench_profile_t *p = P(); + if (!g_paced_mode) return; + if (p->pace_every_ops > 0u && p->pace_us > 0u && ((op_index + 1u) % p->pace_every_ops) == 0u) { + delayMicroseconds((unsigned int)p->pace_us); + } +} + +static void set_paced_mode(bool enabled) { + g_paced_mode = enabled; + MDB_CONSOLE.printf("[PACED] mode=%s\n", g_paced_mode ? "ON" : "OFF"); +} + +static void report_slo(const bench_metric_t *m) { + uint32_t max_us_limit; + uint32_t spike_5ms_limit; + bool ok; + + if (m->ops < 64u) return; + + if (strcmp(P()->name, "deterministic") == 0) { + max_us_limit = 5000u; + spike_5ms_limit = 0u; + } else if (strcmp(P()->name, "quick") == 0) { + max_us_limit = 12000u; + spike_5ms_limit = 2u; + } else if (strcmp(P()->name, "balanced") == 0) { + max_us_limit = 15000u; + spike_5ms_limit = 12u; + } else { + max_us_limit = 25000u; + spike_5ms_limit = 30u; + } + + ok = (m->max_us <= max_us_limit) && (m->spike_gt_5ms <= spike_5ms_limit); + if (ok) { + MDB_CONSOLE.printf("[SLO] %-16s OK (max=%lu<=%lu, spk>5ms=%lu<=%lu)\n", m->name, (unsigned long)m->max_us, + (unsigned long)max_us_limit, (unsigned long)m->spike_gt_5ms, (unsigned long)spike_5ms_limit); + } else { + MDB_CONSOLE.printf("[SLO] %-16s WARN (max=%lu%s%lu, spk>5ms=%lu%s%lu)\n", m->name, (unsigned long)m->max_us, + (m->max_us <= max_us_limit) ? "<=" : ">", (unsigned long)max_us_limit, (unsigned long)m->spike_gt_5ms, + (m->spike_gt_5ms <= spike_5ms_limit) ? "<=" : ">", (unsigned long)spike_5ms_limit); + } +} + +static void emit_metric(const char *name, uint32_t ops, uint64_t total_us, uint64_t bytes, uint32_t *lat, uint32_t n, + uint32_t sample_stride_ops, uint32_t heap0, uint32_t heap1) { + bench_metric_t m; + float sec = (float)total_us / 1000000.0f; + uint32_t i; + uint32_t max_sample_idx = 0u; + bool has_spike_1ms = false; + bool has_spike_5ms = false; + memset(&m, 0, sizeof(m)); + m.name = name; + m.ops = ops; + m.total_us = total_us; + m.bytes = bytes; + m.avg_us = (ops == 0u) ? 0.0f : ((float)total_us / (float)ops); + m.ops_per_s = (sec > 0.0f) ? ((float)ops / sec) : 0.0f; + m.mb_per_s = (sec > 0.0f) ? (((float)bytes / (1024.0f * 1024.0f)) / sec) : 0.0f; + m.samples = n; + m.heap_delta = (int32_t)heap1 - (int32_t)heap0; + m.first_spike_1ms_op = 0xFFFFFFFFu; + m.first_spike_5ms_op = 0xFFFFFFFFu; + for (i = 0u; i < n; ++i) { + if (lat[i] >= m.max_us) { + m.max_us = lat[i]; + max_sample_idx = i; + } + if (lat[i] > 1000u) { + m.spike_gt_1ms++; + if (!has_spike_1ms) { + m.first_spike_1ms_op = i * sample_stride_ops; + has_spike_1ms = true; + } + } + if (lat[i] > 5000u) { + m.spike_gt_5ms++; + if (!has_spike_5ms) { + m.first_spike_5ms_op = i * sample_stride_ops; + has_spike_5ms = true; + } + } + } + m.max_op_approx = max_sample_idx * sample_stride_ops; + if (n > 0u) { + qsort(lat, n, sizeof(uint32_t), cmp_u32); + m.min_us = lat[0u]; + m.p50_us = pct(lat, n, 50u); + m.p95_us = pct(lat, n, 95u); + if (m.p50_us > 0u) m.max_over_p50 = (float)m.max_us / (float)m.p50_us; + } + MDB_CONSOLE.printf("[BENCH] %-16s total=%.3f ms avg=%.3f us p50=%lu p95=%lu min=%lu max=%lu max_op~%lu xmax/p50=%.1f spk>1ms=%lu@%lu spk>5ms=%lu@%lu ops/s=%.1f MB/s=%.3f ops=%lu samp=%lu heap_d=%ld\n", + m.name, (double)((float)m.total_us / 1000.0f), (double)m.avg_us, (unsigned long)m.p50_us, + (unsigned long)m.p95_us, (unsigned long)m.min_us, (unsigned long)m.max_us, (unsigned long)m.max_op_approx, + (double)m.max_over_p50, (unsigned long)m.spike_gt_1ms, + (unsigned long)(has_spike_1ms ? m.first_spike_1ms_op : 0u), (unsigned long)m.spike_gt_5ms, + (unsigned long)(has_spike_5ms ? m.first_spike_5ms_op : 0u), (double)m.ops_per_s, (double)m.mb_per_s, + (unsigned long)m.ops, (unsigned long)m.samples, (long)m.heap_delta); + report_slo(&m); + if (g_last_count < BENCH_MAX_LAST_METRICS) g_last[g_last_count++] = m; +} + +static bool run_kv_bench(void) { + lox_stats_t st; + uint32_t i; + uint32_t ops; + uint32_t stride; + uint32_t n; + uint64_t total; + uint64_t bytes; + uint64_t cold_total; + uint64_t steady_total; + uint32_t cold_ops; + uint32_t steady_ops; + uint32_t heap0, heap1; + char key[16]; + uint32_t v; + uint32_t out; + size_t out_len; + + memset(&st, 0, sizeof(st)); + if (lox_inspect(&g_db, &st) != LOX_OK || st.kv_entries_max == 0u) return false; + + ops = P()->kv_ops; + if (ops > st.kv_entries_max) { + ops = st.kv_entries_max; + MDB_CONSOLE.printf("[KV] capped ops to capacity: %lu\n", (unsigned long)ops); + } + stride = sample_stride(ops); + + heap0 = heap_free_8bit(); + n = 0u; + total = 0u; + bytes = 0u; + cold_total = 0u; + steady_total = 0u; + cold_ops = 0u; + steady_ops = 0u; + for (i = 0u; i < ops; ++i) { + uint32_t t0 = micros(); + snprintf(key, sizeof(key), "k%05lu", (unsigned long)i); + v = i + 1u; + if (lox_kv_put(&g_db, key, &v, sizeof(v)) != LOX_OK) return false; + maybe_apply_write_control(i); + { + uint32_t dt = micros() - t0; + total += dt; + bytes += sizeof(v); + if (i < 64u) { + cold_total += dt; + cold_ops++; + } else { + steady_total += dt; + steady_ops++; + } + if ((i % stride) == 0u && n < BENCH_MAX_LAT_SAMPLES) g_lat[n++] = dt; + } + } + heap1 = heap_free_8bit(); + emit_metric("kv_put", ops, total, bytes, g_lat, n, stride, heap0, heap1); + print_phase_split("kv_put", cold_ops, cold_total, steady_ops, steady_total); + + heap0 = heap_free_8bit(); + n = 0u; + total = 0u; + bytes = 0u; + cold_total = 0u; + steady_total = 0u; + cold_ops = 0u; + steady_ops = 0u; + for (i = 0u; i < ops; ++i) { + uint32_t t0 = micros(); + snprintf(key, sizeof(key), "k%05lu", (unsigned long)i); + out = 0u; + out_len = 0u; + if (lox_kv_get(&g_db, key, &out, sizeof(out), &out_len) != LOX_OK) return false; + if (out != (i + 1u) || out_len != sizeof(out)) return false; + { + uint32_t dt = micros() - t0; + total += dt; + bytes += sizeof(out); + if (i < 64u) { + cold_total += dt; + cold_ops++; + } else { + steady_total += dt; + steady_ops++; + } + if ((i % stride) == 0u && n < BENCH_MAX_LAT_SAMPLES) g_lat[n++] = dt; + } + } + heap1 = heap_free_8bit(); + emit_metric("kv_get", ops, total, bytes, g_lat, n, stride, heap0, heap1); + print_phase_split("kv_get", cold_ops, cold_total, steady_ops, steady_total); + + heap0 = heap_free_8bit(); + n = 0u; + total = 0u; + cold_total = 0u; + steady_total = 0u; + cold_ops = 0u; + steady_ops = 0u; + for (i = 0u; i < ops; ++i) { + uint32_t t0 = micros(); + snprintf(key, sizeof(key), "k%05lu", (unsigned long)i); + if (lox_kv_del(&g_db, key) != LOX_OK) return false; + maybe_apply_write_control(i); + { + uint32_t dt = micros() - t0; + total += dt; + if (i < 64u) { + cold_total += dt; + cold_ops++; + } else { + steady_total += dt; + steady_ops++; + } + if ((i % stride) == 0u && n < BENCH_MAX_LAT_SAMPLES) g_lat[n++] = dt; + } + } + heap1 = heap_free_8bit(); + emit_metric("kv_del", ops, total, 0u, g_lat, n, stride, heap0, heap1); + print_phase_split("kv_del", cold_ops, cold_total, steady_ops, steady_total); + return true; +} + +static bool run_ts_bench(void) { + uint32_t i; + uint32_t ops = P()->ts_ops; + uint32_t stride = sample_stride(ops); + uint32_t n = 0u; + uint64_t total = 0u; + uint64_t bytes = 0u; + uint64_t cold_total = 0u; + uint64_t steady_total = 0u; + uint32_t cold_ops = 0u; + uint32_t steady_ops = 0u; + uint32_t heap0, heap1; + uint32_t value; + lox_ts_sample_t out_buf[64]; + size_t retained = 0u; + size_t out_count = 0u; + lox_err_t e; + + e = lox_ts_register(&g_db, "temp", LOX_TS_U32, 0u); + if (e != LOX_OK && e != LOX_ERR_EXISTS) return false; + (void)lox_ts_clear(&g_db, "temp"); + + heap0 = heap_free_8bit(); + for (i = 0u; i < ops; ++i) { + uint32_t t0 = micros(); + value = i; + if (lox_ts_insert(&g_db, "temp", i, &value) != LOX_OK) return false; + maybe_apply_write_control(i); + { + uint32_t dt = micros() - t0; + total += dt; + bytes += sizeof(value); + if (i < 64u) { + cold_total += dt; + cold_ops++; + } else { + steady_total += dt; + steady_ops++; + } + if ((i % stride) == 0u && n < BENCH_MAX_LAT_SAMPLES) g_lat[n++] = dt; + } + } + heap1 = heap_free_8bit(); + emit_metric("ts_insert", ops, total, bytes, g_lat, n, stride, heap0, heap1); + print_phase_split("ts_insert", cold_ops, cold_total, steady_ops, steady_total); + if (lox_ts_count(&g_db, "temp", 0u, (lox_timestamp_t)ops, &retained) == LOX_OK) { + MDB_CONSOLE.printf("[TS] target=%lu retained=%lu dropped=%lu\n", (unsigned long)ops, (unsigned long)retained, + (unsigned long)((ops > retained) ? (ops - retained) : 0u)); + } + + { + uint32_t t0 = micros(); + if (lox_ts_query_buf(&g_db, "temp", 0u, (lox_timestamp_t)ops, out_buf, 64u, &out_count) != LOX_OK && out_count == 0u) + return false; + g_lat[0] = micros() - t0; + } + emit_metric("ts_query_buf", 1u, g_lat[0], out_count * sizeof(lox_ts_sample_t), g_lat, 1u, 1u, heap_free_8bit(), heap_free_8bit()); + return true; +} + +static bool g_rel_found = false; +static bool rel_find_cb(const void *row_buf, void *ctx) { + const uint8_t *row = (const uint8_t *)row_buf; + uint32_t *want_id = (uint32_t *)ctx; + uint32_t got_id = 0u; + memcpy(&got_id, row, sizeof(got_id)); + if (got_id == *want_id) g_rel_found = true; + return false; +} + +static bool run_rel_bench(void) { + lox_schema_t schema; + lox_table_t *table = NULL; + uint8_t row[64]; + uint32_t rows = P()->rel_rows; + uint32_t i; + uint16_t temp_c; + uint32_t stride = sample_stride(rows); + uint32_t n = 0u; + uint64_t total = 0u; + uint64_t bytes = 0u; + uint64_t cold_total = 0u; + uint64_t steady_total = 0u; + uint32_t cold_ops = 0u; + uint32_t steady_ops = 0u; + uint32_t heap0, heap1; + uint32_t find_id = rows / 2u; + uint32_t count_rows = 0u; + lox_err_t e; + + memset(&schema, 0, sizeof(schema)); + if (lox_schema_init(&schema, "bench_rel", rows + 16u) != LOX_OK) return false; + schema.schema_version = 1u; + if (lox_schema_add(&schema, "id", LOX_COL_U32, sizeof(uint32_t), true) != LOX_OK) return false; + if (lox_schema_add(&schema, "temp", LOX_COL_U16, sizeof(uint16_t), false) != LOX_OK) return false; + if (lox_schema_seal(&schema) != LOX_OK) return false; + + e = lox_table_create(&g_db, &schema); + if (e != LOX_OK && e != LOX_ERR_EXISTS) return false; + if (lox_table_get(&g_db, "bench_rel", &table) != LOX_OK) return false; + if (lox_rel_clear(&g_db, table) != LOX_OK) return false; + + /* Isolate REL timing from prior stage WAL pressure in deterministic mode. */ + if (is_deterministic_profile() && lox_flush(&g_db) != LOX_OK) return false; + + heap0 = heap_free_8bit(); + for (i = 0u; i < rows; ++i) { + uint32_t t0 = micros(); + memset(row, 0, sizeof(row)); + temp_c = (uint16_t)(200u + (i % 50u)); + if (lox_row_set(table, row, "id", &i) != LOX_OK) return false; + if (lox_row_set(table, row, "temp", &temp_c) != LOX_OK) return false; + if (lox_rel_insert(&g_db, table, row) != LOX_OK) return false; + maybe_apply_write_control(i); + { + uint32_t dt = micros() - t0; + total += dt; + bytes += lox_table_row_size(table); + if (i < 64u) { + cold_total += dt; + cold_ops++; + } else { + steady_total += dt; + steady_ops++; + } + if ((i % stride) == 0u && n < BENCH_MAX_LAT_SAMPLES) g_lat[n++] = dt; + } + } + heap1 = heap_free_8bit(); + emit_metric("rel_insert", rows, total, bytes, g_lat, n, stride, heap0, heap1); + print_phase_split("rel_insert", cold_ops, cold_total, steady_ops, steady_total); + + g_rel_found = false; + { + uint32_t t0 = micros(); + if (lox_rel_find(&g_db, table, &find_id, rel_find_cb, &find_id) != LOX_OK) return false; + g_lat[0] = micros() - t0; + } + emit_metric("rel_find(index)", 1u, g_lat[0], lox_table_row_size(table), g_lat, 1u, 1u, heap_free_8bit(), heap_free_8bit()); + + if (!g_rel_found) return false; + if (lox_rel_count(table, &count_rows) != LOX_OK) return false; + MDB_CONSOLE.printf("[REL] rows_expected=%lu rows_actual=%lu\n", (unsigned long)rows, (unsigned long)count_rows); + return (count_rows == rows); +} + +static bool run_wal_compact_bench(void) { + static const char *kProbeKey = "wal_probe"; + lox_stats_t before; + lox_stats_t after; + lox_stats_t start; + uint32_t i = 0u; + uint32_t ops_target = P()->wal_ops; + uint32_t ops_done = 0u; + uint32_t key_span = P()->wal_key_span; + uint32_t val_bytes = P()->wal_val_bytes; + uint32_t min_steady = wal_min_steady_ops(); + uint32_t stride; + uint32_t n = 0u; + uint64_t total = 0u; + uint64_t bytes = 0u; + uint64_t cold_total = 0u; + uint64_t steady_total = 0u; + uint32_t cold_ops = 0u; + uint32_t steady_ops = 0u; + uint32_t heap0, heap1; + char key[20]; + uint8_t val[96]; + uint8_t probe_before[12]; + uint8_t probe_after[12]; + size_t probe_len = 0u; + uint8_t target_fill = P()->wal_threshold_pct; + uint8_t peak_fill = 0u; + uint32_t max_ops; + + if (key_span == 0u) key_span = 1u; + if (val_bytes > sizeof(val)) val_bytes = sizeof(val); + if (target_fill == 0u) target_fill = 75u; + max_ops = (ops_target == 0u) ? 1024u : (ops_target * 8u); + if (max_ops < (min_steady + BENCH_WAL_COLD_OPS)) max_ops = (min_steady + BENCH_WAL_COLD_OPS); + if (max_ops < 512u) max_ops = 512u; + stride = sample_stride(max_ops); + memset(&before, 0, sizeof(before)); + memset(&after, 0, sizeof(after)); + memset(&start, 0, sizeof(start)); + memset(val, 0xA5, sizeof(val)); + memset(probe_before, 0x3C, sizeof(probe_before)); + memset(probe_after, 0, sizeof(probe_after)); + + if (lox_compact(&g_db) != LOX_OK) return false; + if (lox_inspect(&g_db, &start) != LOX_OK) return false; + MDB_CONSOLE.printf("[WAL] baseline before warmup: used=%lu total=%lu fill=%u%%\n", (unsigned long)start.wal_bytes_used, + (unsigned long)start.wal_bytes_total, (unsigned)start.wal_fill_pct); + if (start.kv_entries_max > 2u && key_span >= (start.kv_entries_max - 1u)) { + key_span = start.kv_entries_max - 2u; + MDB_CONSOLE.printf("[WAL] key_span adjusted to %lu to keep probe key resident.\n", (unsigned long)key_span); + } + if (start.wal_fill_pct > 5u) { + MDB_CONSOLE.println("[WAL][WARN] baseline fill is not near-empty before warmup."); + } + + if (lox_kv_put(&g_db, kProbeKey, probe_before, sizeof(probe_before)) != LOX_OK) return false; + + heap0 = heap_free_8bit(); + for (i = 0u; i < max_ops; ++i) { + uint32_t t0 = micros(); + uint32_t dt; + uint32_t seq = i + 1u; + uint32_t salt = (i * 2654435761u) ^ 0xA5A5A5A5u; + + /* Force real WAL growth: every write changes payload contents. */ + memset(val, (uint8_t)(0xA5u ^ (uint8_t)(i & 0xFFu)), val_bytes); + if (val_bytes >= sizeof(seq)) memcpy(val, &seq, sizeof(seq)); + if (val_bytes >= (2u * sizeof(uint32_t))) memcpy(val + sizeof(uint32_t), &salt, sizeof(salt)); + + snprintf(key, sizeof(key), "w%05lu", (unsigned long)(i % key_span)); + if (lox_kv_put(&g_db, key, val, val_bytes) != LOX_OK) return false; + maybe_apply_write_control(i); + dt = micros() - t0; + total += dt; + bytes += val_bytes; + ops_done++; + if (ops_done <= BENCH_WAL_COLD_OPS) { + cold_total += dt; + cold_ops++; + } else { + steady_total += dt; + steady_ops++; + } + if ((i % stride) == 0u && n < BENCH_MAX_LAT_SAMPLES) g_lat[n++] = dt; + + if (((ops_done % 64u) == 0u) || (ops_done == max_ops)) { + if (lox_inspect(&g_db, &before) != LOX_OK) return false; + if (before.wal_fill_pct > peak_fill) peak_fill = before.wal_fill_pct; + if (before.wal_fill_pct >= target_fill && steady_ops >= min_steady) break; + } + } + heap1 = heap_free_8bit(); + emit_metric("wal_kv_put", ops_done, total, bytes, g_lat, n, stride, heap0, heap1); + + if (lox_inspect(&g_db, &before) != LOX_OK) return false; + MDB_CONSOLE.printf("[WAL] warmup target_fill=%u%% reached=%u%% peak=%u%% ops_done=%lu/%lu steady_ops=%lu (min=%lu)\n", + (unsigned)target_fill, (unsigned)before.wal_fill_pct, (unsigned)peak_fill, (unsigned long)ops_done, + (unsigned long)max_ops, (unsigned long)steady_ops, (unsigned long)min_steady); + if (before.wal_fill_pct < target_fill) { + MDB_CONSOLE.println("[WAL][WARN] target fill not reached before compact; compact metric is lighter-case."); + } + if (peak_fill >= target_fill && before.wal_fill_pct < target_fill) { + MDB_CONSOLE.println("[WAL][WARN] fill crossed target earlier but dropped before compact (WAL churn)."); + } + print_phase_split("wal_kv_put", cold_ops, cold_total, steady_ops, steady_total); + + MDB_CONSOLE.printf("[WAL] before compact: used=%lu total=%lu fill=%u%%\n", (unsigned long)before.wal_bytes_used, + (unsigned long)before.wal_bytes_total, (unsigned)before.wal_fill_pct); + + { + uint32_t t0 = micros(); + if (lox_compact(&g_db) != LOX_OK) return false; + g_lat[0] = micros() - t0; + } + emit_metric("compact", 1u, g_lat[0], 0u, g_lat, 1u, 1u, heap_free_8bit(), heap_free_8bit()); + + if (lox_inspect(&g_db, &after) != LOX_OK) return false; + MDB_CONSOLE.printf("[WAL] after compact: used=%lu total=%lu fill=%u%%\n", (unsigned long)after.wal_bytes_used, + (unsigned long)after.wal_bytes_total, (unsigned)after.wal_fill_pct); + if (lox_kv_get(&g_db, kProbeKey, probe_after, sizeof(probe_after), &probe_len) != LOX_OK) return false; + if (probe_len != sizeof(probe_before) || memcmp(probe_before, probe_after, sizeof(probe_before)) != 0) { + MDB_CONSOLE.println("[WAL][ERR] probe key mismatch after compact."); + return false; + } + return after.wal_bytes_used <= before.wal_bytes_used; +} + +static bool run_reopen_check(void) { + lox_stats_t st; + if (lox_deinit(&g_db) != LOX_OK) return false; + { + uint32_t t0 = micros(); + if (db_open(false, false) != LOX_OK) return false; + g_lat[0] = micros() - t0; + } + if (lox_inspect(&g_db, &st) != LOX_OK) return false; + emit_metric("reopen", 1u, g_lat[0], 0u, g_lat, 1u, 1u, heap_free_8bit(), heap_free_8bit()); + return true; +} + +static bool run_migration_check(void) { + lox_schema_t schema; + lox_table_t *table = NULL; + + g_migrate_calls = 0u; + g_migrate_old = 0u; + g_migrate_new = 0u; + + if (lox_deinit(&g_db) != LOX_OK) return false; + if (db_open(false, false) != LOX_OK) return false; + + memset(&schema, 0, sizeof(schema)); + if (lox_schema_init(&schema, "migr_tbl", 16u) != LOX_OK) return false; + schema.schema_version = 1u; + if (lox_schema_add(&schema, "id", LOX_COL_U32, sizeof(uint32_t), true) != LOX_OK) return false; + if (lox_schema_seal(&schema) != LOX_OK) return false; + if (lox_table_create(&g_db, &schema) != LOX_OK) return false; + if (lox_table_get(&g_db, "migr_tbl", &table) != LOX_OK) return false; + (void)table; + + if (lox_deinit(&g_db) != LOX_OK) return false; + if (db_open(false, true) != LOX_OK) return false; + + memset(&schema, 0, sizeof(schema)); + if (lox_schema_init(&schema, "migr_tbl", 16u) != LOX_OK) return false; + schema.schema_version = 2u; + if (lox_schema_add(&schema, "id", LOX_COL_U32, sizeof(uint32_t), true) != LOX_OK) return false; + if (lox_schema_seal(&schema) != LOX_OK) return false; + if (lox_table_create(&g_db, &schema) != LOX_OK) return false; + + MDB_CONSOLE.printf("[MIGRATE] calls=%lu old=%u new=%u\n", (unsigned long)g_migrate_calls, (unsigned)g_migrate_old, + (unsigned)g_migrate_new); + return (g_migrate_calls == 1u && g_migrate_old == 1u && g_migrate_new == 2u); +} + +static bool run_txn_check(void) { + uint32_t v1 = 111u, v2 = 222u, out = 0u; + if (lox_txn_begin(&g_db) != LOX_OK) return false; + if (lox_kv_put(&g_db, "txn_a", &v1, sizeof(v1)) != LOX_OK) return false; + if (lox_txn_commit(&g_db) != LOX_OK) return false; + if (lox_kv_get(&g_db, "txn_a", &out, sizeof(out), NULL) != LOX_OK || out != v1) return false; + + if (lox_txn_begin(&g_db) != LOX_OK) return false; + if (lox_kv_put(&g_db, "txn_a", &v2, sizeof(v2)) != LOX_OK) return false; + if (lox_txn_rollback(&g_db) != LOX_OK) return false; + out = 0u; + if (lox_kv_get(&g_db, "txn_a", &out, sizeof(out), NULL) != LOX_OK || out != v1) return false; + return true; +} + +static void print_stats_snapshot(void) { + lox_stats_t st; + if (lox_inspect(&g_db, &st) != LOX_OK) { + MDB_CONSOLE.println("[STATS] inspect failed"); + return; + } + MDB_CONSOLE.printf("[STATS] kv=%lu/%lu (%u%%) coll=%lu evict=%lu\n", (unsigned long)st.kv_entries_used, + (unsigned long)st.kv_entries_max, (unsigned)st.kv_fill_pct, (unsigned long)st.kv_collision_count, + (unsigned long)st.kv_eviction_count); + MDB_CONSOLE.printf("[STATS] ts_streams=%lu ts_samples=%lu ts_fill=%u%%\n", (unsigned long)st.ts_streams_registered, + (unsigned long)st.ts_samples_total, (unsigned)st.ts_fill_pct); + MDB_CONSOLE.printf("[STATS] wal=%lu/%lu (%u%%) rel_tables=%lu rel_rows=%lu\n", (unsigned long)st.wal_bytes_used, + (unsigned long)st.wal_bytes_total, (unsigned)st.wal_fill_pct, (unsigned long)st.rel_tables_count, + (unsigned long)st.rel_rows_total); +} + +static void print_config(void) { + MDB_CONSOLE.printf("[CONFIG] profile=%s storage=%luKB ram=%luKB split=%u/%u/%u wal_thr=%u%%\n", P()->name, + (unsigned long)(BENCH_STORAGE_BYTES / 1024u), (unsigned long)P()->ram_kb, (unsigned)P()->kv_pct, + (unsigned)P()->ts_pct, (unsigned)P()->rel_pct, (unsigned)P()->wal_threshold_pct); + MDB_CONSOLE.printf("[CONFIG] target_kv=%lu target_ts=%lu target_rel=%lu wal_ops=%lu wal_key=%lu wal_val=%lu\n", + (unsigned long)P()->kv_ops, (unsigned long)P()->ts_ops, (unsigned long)P()->rel_rows, + (unsigned long)P()->wal_ops, (unsigned long)P()->wal_key_span, (unsigned long)P()->wal_val_bytes); + MDB_CONSOLE.printf("[CONFIG] paced=%s pace_every=%lu pace_us=%lu flush_every=%lu\n", g_paced_mode ? "ON" : "OFF", + (unsigned long)P()->pace_every_ops, (unsigned long)P()->pace_us, (unsigned long)P()->flush_every_ops); + print_effective_capacity(); +} + +static void print_profiles(void) { + size_t i; + MDB_CONSOLE.println("Profiles:"); + for (i = 0u; i < (sizeof(g_profiles) / sizeof(g_profiles[0])); ++i) { + MDB_CONSOLE.printf(" %-13s ram=%lu kv=%lu ts=%lu rel=%lu wal=%lu%s\n", g_profiles[i].name, + (unsigned long)g_profiles[i].ram_kb, (unsigned long)g_profiles[i].kv_ops, + (unsigned long)g_profiles[i].ts_ops, (unsigned long)g_profiles[i].rel_rows, + (unsigned long)g_profiles[i].wal_ops, (i == g_profile_idx) ? " " : ""); + } +} + +static bool set_profile(const char *name) { + size_t i; + for (i = 0u; i < (sizeof(g_profiles) / sizeof(g_profiles[0])); ++i) { + if (strcmp(name, g_profiles[i].name) == 0) { + g_profile_idx = i; + g_paced_mode = (strcmp(name, "deterministic") == 0); + return true; + } + } + return false; +} + +static bool try_profile_shortcut(const char *cmd) { + if (set_profile(cmd)) { + MDB_CONSOLE.printf("[PROFILE] switched to %s paced=%s\n", P()->name, g_paced_mode ? "ON" : "OFF"); + reset_db_and_open(true); + return true; + } + return false; +} + +static void print_last_metrics(void) { + size_t i; + if (g_last_count == 0u) { + MDB_CONSOLE.println("[METRICS] no metrics captured yet"); + return; + } + MDB_CONSOLE.printf("[METRICS] count=%lu\n", (unsigned long)g_last_count); + for (i = 0u; i < g_last_count; ++i) { + MDB_CONSOLE.printf("[METRIC] %s total=%.3fms avg=%.3fus p50=%lu p95=%lu max=%lu@%lu xmax/p50=%.1f spk>1ms=%lu@%lu spk>5ms=%lu@%lu ops/s=%.1f MB/s=%.3f heap_d=%ld\n", g_last[i].name, + (double)((float)g_last[i].total_us / 1000.0f), (double)g_last[i].avg_us, + (unsigned long)g_last[i].p50_us, (unsigned long)g_last[i].p95_us, (unsigned long)g_last[i].max_us, + (unsigned long)g_last[i].max_op_approx, (double)g_last[i].max_over_p50, (unsigned long)g_last[i].spike_gt_1ms, + (unsigned long)((g_last[i].first_spike_1ms_op == 0xFFFFFFFFu) ? 0u : g_last[i].first_spike_1ms_op), + (unsigned long)g_last[i].spike_gt_5ms, + (unsigned long)((g_last[i].first_spike_5ms_op == 0xFFFFFFFFu) ? 0u : g_last[i].first_spike_5ms_op), (double)g_last[i].ops_per_s, + (double)g_last[i].mb_per_s, (long)g_last[i].heap_delta); + } +} + +static bool run_real_data_suite(void) { + const char *first_fail = NULL; + lox_table_t *table = NULL; + lox_schema_t schema; + uint32_t u32 = 0u; + uint32_t v_5000 = 5000u; + uint32_t v_1 = 1u; + uint32_t v_100 = 100u; + uint32_t v_999 = 999u; + uint8_t sev_3 = 3u; + float tf1 = 18.5f; + float tf2 = 19.2f; + size_t out_len = 0u; + lox_ts_sample_t ts_last; + size_t ts_count = 0u; + uint32_t rel_count = 0u; + uint32_t deleted = 0u; + uint8_t row[64]; + uint8_t out_row[64]; + char ie_json[1536]; + size_t ie_used = 0u; + uint32_t ie_exported = 0u; + uint32_t ie_imported = 0u; + uint32_t ie_skipped = 0u; + lox_ie_options_t ie_opts = lox_ie_default_options(); + const char *ie_keys[3] = {"wifi.ssid", "sensor.interval_ms", "json.counter"}; + lox_db_stats_t dbs; + lox_kv_stats_t kvs; + lox_ts_stats_t tss; + lox_rel_stats_t rs; + lox_effective_capacity_t ec; + lox_pressure_t p; + lox_admission_t adm; + +#define RD_CHECK_REAL(label, expr) \ + do { \ + uint32_t _t0 = micros(); \ + lox_err_t _rc = (expr); \ + uint32_t _dt = micros() - _t0; \ + MDB_CONSOLE.printf("[RD][%-30s] rc=%s (%d) %lu us\n", (label), lox_err_to_string(_rc), (int)_rc, (unsigned long)_dt); \ + if (_rc != LOX_OK) { \ + first_fail = (label); \ + goto rd_fail; \ + } \ + } while (0) + +#define RD_EXPECT_REAL(label, cond) \ + do { \ + bool _ok = (cond); \ + MDB_CONSOLE.printf("[RD][%-30s] expect=%s\n", (label), _ok ? "OK" : "FAIL"); \ + if (!_ok) { \ + first_fail = (label); \ + goto rd_fail; \ + } \ + } while (0) + + RD_CHECK_REAL("kv_put/wifi.ssid", lox_kv_put(&g_db, "wifi.ssid", "HomeNetwork_5G", 14u)); + RD_CHECK_REAL("kv_set/interval", lox_kv_set(&g_db, "sensor.interval_ms", &v_5000, sizeof(uint32_t), 0u)); + RD_CHECK_REAL("kv_set/boot.count", lox_kv_set(&g_db, "boot.count", &v_1, sizeof(uint32_t), 2u)); + RD_CHECK_REAL("kv_get/interval", lox_kv_get(&g_db, "sensor.interval_ms", &u32, sizeof(u32), &out_len)); + RD_EXPECT_REAL("assert/interval", u32 == 5000u && out_len == sizeof(uint32_t)); + RD_CHECK_REAL("kv_del/boot.count", lox_kv_del(&g_db, "boot.count")); + RD_CHECK_REAL("admit_kv_set", lox_admit_kv_set(&g_db, "wifi.ssid", 16u, &adm)); + RD_CHECK_REAL("json/set_u32", lox_json_kv_set_u32(&g_db, "json.counter", 9876u, 0u)); + RD_CHECK_REAL("json/get_u32", lox_json_kv_get_u32(&g_db, "json.counter", &u32)); + RD_EXPECT_REAL("assert/json.counter", u32 == 9876u); + RD_CHECK_REAL("ie/export_kv", lox_ie_export_kv_json(&g_db, ie_keys, 3u, ie_json, sizeof(ie_json), &ie_used, &ie_exported)); + RD_EXPECT_REAL("assert/ie.exported", ie_exported == 3u); + RD_CHECK_REAL("kv_del/wifi.ssid", lox_kv_del(&g_db, "wifi.ssid")); + RD_CHECK_REAL("kv_del/sensor.interval", lox_kv_del(&g_db, "sensor.interval_ms")); + RD_CHECK_REAL("kv_del/json.counter", lox_kv_del(&g_db, "json.counter")); + RD_CHECK_REAL("ie/import_kv", lox_ie_import_kv_json(&g_db, ie_json, &ie_opts, &ie_imported, &ie_skipped)); + RD_EXPECT_REAL("assert/ie.imported", ie_imported == 3u && ie_skipped == 0u); + RD_CHECK_REAL("json/get_u32/reimport", lox_json_kv_get_u32(&g_db, "json.counter", &u32)); + RD_EXPECT_REAL("assert/json.reimport", u32 == 9876u); + + RD_CHECK_REAL("ts_register/temp", lox_ts_register(&g_db, "temperature", LOX_TS_F32, 0u)); + RD_CHECK_REAL("ts_insert/t1", lox_ts_insert(&g_db, "temperature", 1700000000u, &tf1)); + RD_CHECK_REAL("ts_insert/t2", lox_ts_insert(&g_db, "temperature", 1700000120u, &tf2)); + RD_CHECK_REAL("ts_last/temp", lox_ts_last(&g_db, "temperature", &ts_last)); + RD_EXPECT_REAL("assert/ts_last", ts_last.ts == 1700000120u); + RD_CHECK_REAL("ts_count/temp", lox_ts_count(&g_db, "temperature", 0u, (lox_timestamp_t)0xFFFFFFFFu, &ts_count)); + RD_EXPECT_REAL("assert/ts_count", ts_count >= 2u); + + memset(&schema, 0, sizeof(schema)); + RD_CHECK_REAL("rel_schema_init", lox_schema_init(&schema, "event_log", 16u)); + RD_CHECK_REAL("rel_schema_add/id", lox_schema_add(&schema, "id", LOX_COL_U32, sizeof(uint32_t), true)); + RD_CHECK_REAL("rel_schema_add/sev", lox_schema_add(&schema, "severity", LOX_COL_U8, sizeof(uint8_t), false)); + RD_CHECK_REAL("rel_schema_seal", lox_schema_seal(&schema)); + { + lox_err_t rc = lox_table_create(&g_db, &schema); + MDB_CONSOLE.printf("[RD][%-30s] rc=%s (%d)\n", "rel_table_create", lox_err_to_string(rc), (int)rc); + if (rc != LOX_OK && rc != LOX_ERR_EXISTS) { + first_fail = "rel_table_create"; + goto rd_fail; + } + } + RD_CHECK_REAL("rel_table_get", lox_table_get(&g_db, "event_log", &table)); + RD_CHECK_REAL("rel_clear", lox_rel_clear(&g_db, table)); + memset(row, 0, sizeof(row)); + RD_CHECK_REAL("rel_row_set/id", lox_row_set(table, row, "id", &v_1)); + RD_CHECK_REAL("rel_row_set/sev", lox_row_set(table, row, "severity", &sev_3)); + RD_CHECK_REAL("rel_insert", lox_rel_insert(&g_db, table, row)); + RD_CHECK_REAL("rel_find_by/id", lox_rel_find_by(&g_db, table, "id", &v_1, out_row)); + RD_CHECK_REAL("rel_count", lox_rel_count(table, &rel_count)); + RD_EXPECT_REAL("assert/rel_count", rel_count == 1u); + RD_CHECK_REAL("rel_delete/id", lox_rel_delete(&g_db, table, &v_1, &deleted)); + RD_EXPECT_REAL("assert/rel_delete", deleted == 1u); + RD_CHECK_REAL("admit_rel_insert", lox_admit_rel_insert(&g_db, "event_log", lox_table_row_size(table), &adm)); + + RD_CHECK_REAL("txn_begin", lox_txn_begin(&g_db)); + RD_CHECK_REAL("txn_set/a", lox_kv_set(&g_db, "txn.a", &v_100, sizeof(uint32_t), 0u)); + RD_CHECK_REAL("txn_commit", lox_txn_commit(&g_db)); + RD_CHECK_REAL("txn_begin2", lox_txn_begin(&g_db)); + RD_CHECK_REAL("txn_set/undo", lox_kv_set(&g_db, "txn.undo", &v_999, sizeof(uint32_t), 0u)); + RD_CHECK_REAL("txn_rollback", lox_txn_rollback(&g_db)); + + RD_CHECK_REAL("flush", lox_flush(&g_db)); + RD_CHECK_REAL("deinit", lox_deinit(&g_db)); + RD_CHECK_REAL("reinit", db_open(false, false)); + RD_CHECK_REAL("recover/kv_get", lox_kv_get(&g_db, "wifi.ssid", row, sizeof(row), &out_len)); + RD_CHECK_REAL("recover/ts_count", lox_ts_count(&g_db, "temperature", 0u, (lox_timestamp_t)0xFFFFFFFFu, &ts_count)); + RD_CHECK_REAL("db_stats", lox_get_db_stats(&g_db, &dbs)); + RD_CHECK_REAL("kv_stats", lox_get_kv_stats(&g_db, &kvs)); + RD_CHECK_REAL("ts_stats", lox_get_ts_stats(&g_db, &tss)); + RD_CHECK_REAL("rel_stats", lox_get_rel_stats(&g_db, &rs)); + RD_CHECK_REAL("eff_cap", lox_get_effective_capacity(&g_db, &ec)); + RD_CHECK_REAL("pressure", lox_get_pressure(&g_db, &p)); + + MDB_CONSOLE.println("[REAL_DATA] PASS"); + return true; + +rd_fail: + MDB_CONSOLE.printf("[REAL_DATA] FAIL: %s\n", first_fail != NULL ? first_fail : "unknown"); + return false; + +#undef RD_CHECK_REAL +#undef RD_EXPECT_REAL +} + +static void run_full_bench_once(void) { + bool ok; + bool stage_flush = g_paced_mode || is_deterministic_profile(); + lox_err_t deinit_rc; + lox_err_t open_rc; + clear_metrics(); + MDB_CONSOLE.println(); + MDB_CONSOLE.printf("=== loxdb ESP32-S3 benchmark start (profile=%s) ===\n", P()->name); + + deinit_rc = lox_deinit(&g_db); + open_rc = db_open(true, false); + if (deinit_rc != LOX_OK || open_rc != LOX_OK) { + MDB_CONSOLE.printf("[ERR] pre-run reset/open failed: deinit=%s (%d), open=%s (%d)\n", + lox_err_to_string(deinit_rc), + (int)deinit_rc, + lox_err_to_string(open_rc), + (int)open_rc); + return; + } + print_effective_capacity(); + + ok = run_kv_bench(); + MDB_CONSOLE.printf("[CHECK] KV benchmark: %s\n", ok ? "PASS" : "FAIL"); + if (!ok) return; + if (stage_flush) (void)lox_flush(&g_db); + ok = run_ts_bench(); + MDB_CONSOLE.printf("[CHECK] TS benchmark: %s\n", ok ? "PASS" : "FAIL"); + if (!ok) return; + if (stage_flush) (void)lox_flush(&g_db); + ok = run_rel_bench(); + MDB_CONSOLE.printf("[CHECK] REL benchmark: %s\n", ok ? "PASS" : "FAIL"); + if (!ok) return; + if (stage_flush) (void)lox_flush(&g_db); + ok = run_wal_compact_bench(); + MDB_CONSOLE.printf("[CHECK] WAL compact: %s\n", ok ? "PASS" : "FAIL"); + if (!ok) return; + ok = run_reopen_check(); + MDB_CONSOLE.printf("[CHECK] Reopen integrity: %s\n", ok ? "PASS" : "FAIL"); + if (!ok) return; + ok = run_migration_check(); + MDB_CONSOLE.printf("[CHECK] Migration callback: %s\n", ok ? "PASS" : "FAIL"); + if (!ok) return; + ok = run_txn_check(); + MDB_CONSOLE.printf("[CHECK] TXN commit/rollback: %s\n", ok ? "PASS" : "FAIL"); + if (!ok) return; + + print_stats_snapshot(); + MDB_CONSOLE.println("=== loxdb ESP32-S3 benchmark end ==="); +} + +static void print_help(void) { + MDB_CONSOLE.println("Commands:"); + MDB_CONSOLE.println(" help - show commands"); + MDB_CONSOLE.println(" run - run full benchmark suite (fresh DB)"); + MDB_CONSOLE.println(" run_real - run real-data integration smoke suite"); + MDB_CONSOLE.println(" kv/ts/rel/wal - run single benchmark stage"); + MDB_CONSOLE.println(" reopenchk - run reopen latency + integrity check"); + MDB_CONSOLE.println(" migrate - run schema migration check"); + MDB_CONSOLE.println(" txn - run txn check"); + MDB_CONSOLE.println(" stats - print inspect snapshot"); + MDB_CONSOLE.println(" metrics - print last captured metrics"); + MDB_CONSOLE.println(" config - print active config"); + MDB_CONSOLE.println(" profiles - list profiles"); + MDB_CONSOLE.println(" profile - show active profile"); + MDB_CONSOLE.println(" profile - switch profile and reopen DB (wipe)"); + MDB_CONSOLE.println(" run_det - deterministic profile + paced OFF + run (recommended)"); + MDB_CONSOLE.println(" run_det_paced - deterministic profile + paced ON + run"); + MDB_CONSOLE.println(" note: run_det validates deterministic profile latency, not all profiles/workloads"); + MDB_CONSOLE.println(" paced - print paced mode"); + MDB_CONSOLE.println(" paced on|off - toggle paced mode"); + MDB_CONSOLE.println(" resetdb - wipe storage + reopen DB"); + MDB_CONSOLE.println(" reopen - reopen DB without wipe"); +} + +static void prompt(void) { MDB_CONSOLE.print("loxdb-bench> "); } + +static void reset_db_and_open(bool wipe) { + lox_err_t d = lox_deinit(&g_db); + if (d != LOX_OK) MDB_CONSOLE.printf("[WARN] deinit returned %d\n", (int)d); + { + lox_err_t o = db_open(wipe, false); + if (o != LOX_OK) + MDB_CONSOLE.printf("[ERR] db_open failed: %s (%d)\n", lox_err_to_string(o), (int)o); + else + MDB_CONSOLE.printf("[OK] DB ready (wipe=%u, profile=%s)\n", wipe ? 1u : 0u, P()->name); + } +} + +static void execute_command(char *line) { + char *cmd = strtok(line, " \t"); + char *arg = strtok(NULL, " \t"); + if (cmd == NULL) return; + + if (strcmp(cmd, "help") == 0) { + print_help(); + } else if (try_profile_shortcut(cmd)) { + return; + } else if (strcmp(cmd, "run_det") == 0) { + if (!set_profile("deterministic")) { + MDB_CONSOLE.println("[ERR] deterministic profile is not available"); + return; + } + MDB_CONSOLE.println("[NOTE] run_det validates deterministic profile latency, not all profiles/workloads."); + set_paced_mode(false); + reset_db_and_open(true); + run_full_bench_once(); + } else if (strcmp(cmd, "run_det_paced") == 0) { + if (!set_profile("deterministic")) { + MDB_CONSOLE.println("[ERR] deterministic profile is not available"); + return; + } + set_paced_mode(true); + reset_db_and_open(true); + run_full_bench_once(); + } else if (strcmp(cmd, "run") == 0) { + run_full_bench_once(); + } else if (strcmp(cmd, "run_real") == 0) { + (void)run_real_data_suite(); + } else if (strcmp(cmd, "kv") == 0) { + clear_metrics(); + MDB_CONSOLE.printf("[CHECK] KV benchmark: %s\n", run_kv_bench() ? "PASS" : "FAIL"); + } else if (strcmp(cmd, "ts") == 0) { + clear_metrics(); + MDB_CONSOLE.printf("[CHECK] TS benchmark: %s\n", run_ts_bench() ? "PASS" : "FAIL"); + } else if (strcmp(cmd, "rel") == 0) { + clear_metrics(); + MDB_CONSOLE.printf("[CHECK] REL benchmark: %s\n", run_rel_bench() ? "PASS" : "FAIL"); + } else if (strcmp(cmd, "wal") == 0) { + clear_metrics(); + MDB_CONSOLE.printf("[CHECK] WAL compact: %s\n", run_wal_compact_bench() ? "PASS" : "FAIL"); + } else if (strcmp(cmd, "reopenchk") == 0) { + clear_metrics(); + MDB_CONSOLE.printf("[CHECK] Reopen integrity: %s\n", run_reopen_check() ? "PASS" : "FAIL"); + } else if (strcmp(cmd, "migrate") == 0) { + MDB_CONSOLE.printf("[CHECK] Migration callback: %s\n", run_migration_check() ? "PASS" : "FAIL"); + } else if (strcmp(cmd, "txn") == 0) { + MDB_CONSOLE.printf("[CHECK] TXN commit/rollback: %s\n", run_txn_check() ? "PASS" : "FAIL"); + } else if (strcmp(cmd, "stats") == 0) { + print_stats_snapshot(); + } else if (strcmp(cmd, "metrics") == 0) { + print_last_metrics(); + } else if (strcmp(cmd, "config") == 0) { + print_config(); + } else if (strcmp(cmd, "profiles") == 0) { + print_profiles(); + } else if (strcmp(cmd, "profile") == 0) { + if (arg == NULL) { + MDB_CONSOLE.printf("[PROFILE] active=%s paced=%s\n", P()->name, g_paced_mode ? "ON" : "OFF"); + } else if (set_profile(arg)) { + MDB_CONSOLE.printf("[PROFILE] switched to %s paced=%s\n", P()->name, g_paced_mode ? "ON" : "OFF"); + reset_db_and_open(true); + } else { + MDB_CONSOLE.printf("[ERR] unknown profile: %s\n", arg); + print_profiles(); + } + } else if (strcmp(cmd, "paced") == 0) { + if (arg == NULL) { + MDB_CONSOLE.printf("[PACED] mode=%s\n", g_paced_mode ? "ON" : "OFF"); + } else if (strcmp(arg, "on") == 0) { + set_paced_mode(true); + } else if (strcmp(arg, "off") == 0) { + set_paced_mode(false); + } else { + MDB_CONSOLE.printf("[ERR] unknown paced arg: %s (use on/off)\n", arg); + } + } else if (strcmp(cmd, "resetdb") == 0) { + reset_db_and_open(true); + } else if (strcmp(cmd, "reopen") == 0) { + reset_db_and_open(false); + } else { + MDB_CONSOLE.printf("[ERR] unknown command: %s\n", cmd); + MDB_CONSOLE.println("Type 'help' for available commands."); + } +} + +void setup(void) { + lox_err_t err; + MDB_CONSOLE.begin(115200); + delay(1200); + + memset(&g_store_ctx, 0, sizeof(g_store_ctx)); + if (!storage_alloc()) { + MDB_CONSOLE.println("[FATAL] storage alloc failed"); + return; + } + storage_reset(); + err = db_open(true, false); + if (err != LOX_OK) { + MDB_CONSOLE.printf("[FATAL] lox_init failed: %s (%d)\n", lox_err_to_string(err), (int)err); + return; + } + + MDB_CONSOLE.println(); + MDB_CONSOLE.println("loxdb ESP32-S3 terminal bench is ready."); + MDB_CONSOLE.println("Tests do NOT run automatically at power-on."); + print_config(); + print_help(); + prompt(); +} + +void loop(void) { + static char line[96]; + static size_t line_len = 0u; + while (MDB_CONSOLE.available() > 0) { + char ch = (char)MDB_CONSOLE.read(); + if (ch == '\r') continue; + if (ch == '\n') { + line[line_len] = '\0'; + execute_command(line); + line_len = 0u; + prompt(); + continue; + } + if (line_len < (sizeof(line) - 1u)) line[line_len++] = ch; + } +} diff --git a/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/lox_import_export.h b/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/lox_import_export.h new file mode 100644 index 0000000..5e0809e --- /dev/null +++ b/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/lox_import_export.h @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: MIT +#ifndef LOX_IMPORT_EXPORT_H +#define LOX_IMPORT_EXPORT_H + +#include "lox.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + /* When 0, existing keys are skipped during import. */ + uint8_t overwrite_existing; + /* When 1, malformed items are skipped instead of aborting import. */ + uint8_t skip_invalid_items; +} lox_ie_options_t; + +typedef struct { + const char *name; + lox_ts_type_t type; + size_t raw_size; +} lox_ie_ts_stream_desc_t; + +typedef struct { + const char *name; + size_t row_size; +} lox_ie_rel_table_desc_t; + +lox_ie_options_t lox_ie_default_options(void); + +/* Exports selected KV keys into JSON: + * {"format":"loxdb.kv.v1","items":[{"key":"...","ttl":N,"value_hex":"..."}]} + */ +lox_err_t lox_ie_export_kv_json(lox_t *db, + const char *const *keys, + size_t key_count, + char *out_json, + size_t out_json_len, + size_t *out_used, + uint32_t *out_exported); + +/* Imports KV items from the same format produced by lox_ie_export_kv_json. */ +lox_err_t lox_ie_import_kv_json(lox_t *db, + const char *json, + const lox_ie_options_t *options, + uint32_t *out_imported, + uint32_t *out_skipped); + +/* Exports selected TS streams: + * {"format":"loxdb.ts.v1","items":[{"stream":"...","type":"u32","ts":1,"value_hex":"..."}]} + */ +lox_err_t lox_ie_export_ts_json(lox_t *db, + const lox_ie_ts_stream_desc_t *streams, + size_t stream_count, + lox_timestamp_t from, + lox_timestamp_t to, + char *out_json, + size_t out_json_len, + size_t *out_used, + uint32_t *out_exported); + +lox_err_t lox_ie_import_ts_json(lox_t *db, + const char *json, + const lox_ie_ts_stream_desc_t *streams, + size_t stream_count, + const lox_ie_options_t *options, + uint32_t *out_imported, + uint32_t *out_skipped); + +/* Exports selected REL tables: + * {"format":"loxdb.rel.v1","items":[{"table":"...","row_hex":"..."}]} + */ +lox_err_t lox_ie_export_rel_json(lox_t *db, + const lox_ie_rel_table_desc_t *tables, + size_t table_count, + char *out_json, + size_t out_json_len, + size_t *out_used, + uint32_t *out_exported); + +lox_err_t lox_ie_import_rel_json(lox_t *db, + const char *json, + const lox_ie_rel_table_desc_t *tables, + size_t table_count, + const lox_ie_options_t *options, + uint32_t *out_imported, + uint32_t *out_skipped); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/lox_json_wrapper.h b/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/lox_json_wrapper.h new file mode 100644 index 0000000..7c02c0b --- /dev/null +++ b/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/lox_json_wrapper.h @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: MIT +#ifndef LOX_JSON_WRAPPER_H +#define LOX_JSON_WRAPPER_H + +#include "lox.h" + +#ifdef __cplusplus +extern "C" { +#endif + +lox_err_t lox_json_kv_set_u32(lox_t *db, const char *key, uint32_t value, uint32_t ttl); +lox_err_t lox_json_kv_get_u32(lox_t *db, const char *key, uint32_t *out_value); + +lox_err_t lox_json_kv_set_i32(lox_t *db, const char *key, int32_t value, uint32_t ttl); +lox_err_t lox_json_kv_get_i32(lox_t *db, const char *key, int32_t *out_value); + +lox_err_t lox_json_kv_set_bool(lox_t *db, const char *key, bool value, uint32_t ttl); +lox_err_t lox_json_kv_get_bool(lox_t *db, const char *key, bool *out_value); + +lox_err_t lox_json_kv_set_cstr(lox_t *db, const char *key, const char *value, uint32_t ttl); +lox_err_t lox_json_kv_get_cstr(lox_t *db, const char *key, char *out_buf, size_t out_buf_len, size_t *out_len); + +/* Encodes a record as: + * {"key":"...","ttl":123,"value_hex":"A1B2..."} + */ +lox_err_t lox_json_encode_kv_record(const char *key, + const void *value, + size_t value_len, + uint32_t ttl, + char *out_json, + size_t out_json_len, + size_t *out_used); + +/* Decodes the same schema produced by lox_json_encode_kv_record. */ +lox_err_t lox_json_decode_kv_record(const char *json, + char *key_out, + size_t key_out_len, + uint8_t *value_out, + size_t value_out_len, + size_t *value_len_out, + uint32_t *ttl_out); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/src/lox_arena.h b/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/src/lox_arena.h new file mode 100644 index 0000000..e565642 --- /dev/null +++ b/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/src/lox_arena.h @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: MIT +#ifndef LOX_ARENA_H +#define LOX_ARENA_H + +#include "lox.h" + +static inline void lox_arena_init(lox_arena_t *arena, uint8_t *base, size_t capacity) { + arena->base = base; + arena->capacity = capacity; + arena->used = 0; +} + +static inline void *lox_arena_alloc(lox_arena_t *arena, size_t size, size_t align) { + size_t aligned = (arena->used + (align - 1u)) & ~(align - 1u); + void *ptr; + + if (aligned + size > arena->capacity) { + return NULL; + } + + ptr = arena->base + aligned; + arena->used = aligned + size; + return ptr; +} + +static inline size_t lox_arena_remaining(const lox_arena_t *arena) { + return arena->capacity - arena->used; +} + +#endif diff --git a/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/src/lox_crc.c b/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/src/lox_crc.c new file mode 100644 index 0000000..039c652 --- /dev/null +++ b/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/src/lox_crc.c @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: MIT +#include "lox_crc.h" + +uint32_t lox_crc32(uint32_t crc, const void *data, size_t len) { + const uint8_t *p = (const uint8_t *)data; + size_t i; + + crc = ~crc; + while (len-- != 0u) { + crc ^= *p++; + for (i = 0; i < 8u; ++i) { + if ((crc & 1u) != 0u) { + crc = 0xEDB88320u ^ (crc >> 1u); + } else { + crc >>= 1u; + } + } + } + + return ~crc; +} diff --git a/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/src/lox_crc.h b/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/src/lox_crc.h new file mode 100644 index 0000000..914dc52 --- /dev/null +++ b/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/src/lox_crc.h @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: MIT +#ifndef LOX_CRC_H +#define LOX_CRC_H + +#include +#include + +uint32_t lox_crc32(uint32_t crc, const void *data, size_t len); + +#define LOX_CRC32(data, len) lox_crc32(0xFFFFFFFFu, (data), (len)) + +#endif diff --git a/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/src/lox_import_export.c b/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/src/lox_import_export.c new file mode 100644 index 0000000..85ce852 --- /dev/null +++ b/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/src/lox_import_export.c @@ -0,0 +1,902 @@ +// SPDX-License-Identifier: MIT +#include "lox_import_export.h" +#include "lox_json_wrapper.h" + +#include +#include +#include +#include + +typedef struct { + const char *target_key; + uint32_t ttl; + uint8_t found; +} ie_ttl_lookup_t; + +typedef struct { + char *out; + size_t out_len; + size_t *pos; + const lox_ie_ts_stream_desc_t *desc; + uint32_t *exported; + lox_err_t rc; +} ie_ts_export_ctx_t; + +typedef struct { + char *out; + size_t out_len; + size_t *pos; + const char *table_name; + size_t row_size; + uint32_t *exported; + lox_err_t rc; +} ie_rel_export_ctx_t; + +static bool ie_find_ttl_cb(const char *key, const void *val, size_t val_len, uint32_t ttl_remaining, void *ctx) { + ie_ttl_lookup_t *q = (ie_ttl_lookup_t *)ctx; + (void)val; + (void)val_len; + if (strcmp(key, q->target_key) == 0) { + q->ttl = ttl_remaining; + q->found = 1u; + return false; + } + return true; +} + +static lox_err_t ie_lookup_ttl(lox_t *db, const char *key, uint32_t *out_ttl) { + ie_ttl_lookup_t q; + lox_err_t rc; + q.target_key = key; + q.ttl = 0u; + q.found = 0u; + rc = lox_kv_iter(db, ie_find_ttl_cb, &q); + if (rc != LOX_OK) return rc; + if (!q.found) return LOX_ERR_NOT_FOUND; + *out_ttl = q.ttl; + return LOX_OK; +} + +static lox_err_t ie_append(char *out, size_t out_len, size_t *pos, const char *s) { + size_t n = strlen(s); + if (*pos + n > out_len) return LOX_ERR_OVERFLOW; + memcpy(out + *pos, s, n); + *pos += n; + return LOX_OK; +} + +static lox_err_t ie_append_char(char *out, size_t out_len, size_t *pos, char c) { + if (*pos >= out_len) return LOX_ERR_OVERFLOW; + out[*pos] = c; + (*pos)++; + return LOX_OK; +} + +static lox_err_t ie_append_u32(char *out, size_t out_len, size_t *pos, uint32_t v) { + char buf[16]; + int n = snprintf(buf, sizeof(buf), "%u", (unsigned)v); + if (n <= 0 || (size_t)n >= sizeof(buf)) return LOX_ERR_INVALID; + return ie_append(out, out_len, pos, buf); +} + +static lox_err_t ie_append_hex(char *out, size_t out_len, size_t *pos, const uint8_t *buf, size_t len) { + static const char hx[] = "0123456789ABCDEF"; + size_t i; + if (*pos + (len * 2u) > out_len) return LOX_ERR_OVERFLOW; + for (i = 0u; i < len; ++i) { + out[*pos + i * 2u] = hx[(buf[i] >> 4) & 0x0Fu]; + out[*pos + i * 2u + 1u] = hx[buf[i] & 0x0Fu]; + } + *pos += len * 2u; + return LOX_OK; +} + +static void ie_skip_ws(const char **p) { + while (**p != '\0' && isspace((unsigned char)**p)) (*p)++; +} + +static const char *ie_find_items_array(const char *json) { + const char *items = strstr(json, "\"items\""); + if (items == NULL) return NULL; + items = strchr(items, '['); + return items; +} + +static const char *ie_find_obj_end(const char *p) { + int depth = 0; + int in_string = 0; + int esc = 0; + while (*p != '\0') { + char c = *p; + if (in_string) { + if (esc) { + esc = 0; + } else if (c == '\\') { + esc = 1; + } else if (c == '"') { + in_string = 0; + } + } else { + if (c == '"') { + in_string = 1; + } else if (c == '{') { + depth++; + } else if (c == '}') { + depth--; + if (depth == 0) return p; + } + } + p++; + } + return NULL; +} + +static int ie_is_hex_char(char c) { + return ((c >= '0' && c <= '9') || + (c >= 'a' && c <= 'f') || + (c >= 'A' && c <= 'F')); +} + +static uint8_t ie_hex_val(char c) { + if (c >= '0' && c <= '9') return (uint8_t)(c - '0'); + if (c >= 'a' && c <= 'f') return (uint8_t)(10 + (c - 'a')); + return (uint8_t)(10 + (c - 'A')); +} + +static lox_err_t ie_parse_json_string(const char **p, char *out, size_t out_len) { + size_t pos = 0u; + ie_skip_ws(p); + if (**p != '"') return LOX_ERR_INVALID; + (*p)++; + while (**p != '\0') { + char c = **p; + if (c == '"') { + (*p)++; + if (pos >= out_len) return LOX_ERR_OVERFLOW; + out[pos] = '\0'; + return LOX_OK; + } + if (c == '\\') { + (*p)++; + c = **p; + if (c == '\0') return LOX_ERR_INVALID; + switch (c) { + case '"': + case '\\': + case '/': + break; + case 'b': + c = '\b'; + break; + case 'f': + c = '\f'; + break; + case 'n': + c = '\n'; + break; + case 'r': + c = '\r'; + break; + case 't': + c = '\t'; + break; + case 'u': + if ((*p)[1] == '0' && (*p)[2] == '0' && + ie_is_hex_char((*p)[3]) && ie_is_hex_char((*p)[4])) { + c = (char)((ie_hex_val((*p)[3]) << 4) | ie_hex_val((*p)[4])); + (*p) += 4; + } else { + return LOX_ERR_INVALID; + } + break; + default: + return LOX_ERR_INVALID; + } + } + if (pos + 1u >= out_len) return LOX_ERR_OVERFLOW; + out[pos++] = c; + (*p)++; + } + return LOX_ERR_INVALID; +} + +static lox_err_t ie_parse_json_u32(const char **p, uint32_t *out) { + unsigned long v; + char *end = NULL; + ie_skip_ws(p); + if (**p == '\0' || !isdigit((unsigned char)**p)) return LOX_ERR_INVALID; + v = strtoul(*p, &end, 10); + if (end == NULL || end == *p || v > 0xFFFFFFFFul) return LOX_ERR_INVALID; + *p = end; + *out = (uint32_t)v; + return LOX_OK; +} + +static lox_err_t ie_parse_json_hex_string(const char **p, uint8_t *out, size_t out_len, size_t *out_used) { + size_t n = 0u; + ie_skip_ws(p); + if (**p != '"') return LOX_ERR_INVALID; + (*p)++; + while (**p != '\0' && **p != '"') { + if (!ie_is_hex_char((*p)[0]) || !ie_is_hex_char((*p)[1])) return LOX_ERR_INVALID; + if (n >= out_len) return LOX_ERR_OVERFLOW; + out[n++] = (uint8_t)((ie_hex_val((*p)[0]) << 4) | ie_hex_val((*p)[1])); + (*p) += 2; + } + if (**p != '"') return LOX_ERR_INVALID; + (*p)++; + *out_used = n; + return LOX_OK; +} + +static lox_err_t ie_parse_object_field_string(const char *obj, const char *field_name, char *out, size_t out_len) { + char needle[64]; + const char *p; + if (strlen(field_name) + 4u >= sizeof(needle)) return LOX_ERR_INVALID; + snprintf(needle, sizeof(needle), "\"%s\"", field_name); + p = strstr(obj, needle); + if (p == NULL) return LOX_ERR_NOT_FOUND; + p += strlen(needle); + ie_skip_ws(&p); + if (*p != ':') return LOX_ERR_INVALID; + p++; + return ie_parse_json_string(&p, out, out_len); +} + +static lox_err_t ie_parse_object_field_u32(const char *obj, const char *field_name, uint32_t *out) { + char needle[64]; + const char *p; + if (strlen(field_name) + 4u >= sizeof(needle)) return LOX_ERR_INVALID; + snprintf(needle, sizeof(needle), "\"%s\"", field_name); + p = strstr(obj, needle); + if (p == NULL) return LOX_ERR_NOT_FOUND; + p += strlen(needle); + ie_skip_ws(&p); + if (*p != ':') return LOX_ERR_INVALID; + p++; + return ie_parse_json_u32(&p, out); +} + +static lox_err_t ie_parse_object_field_hex(const char *obj, const char *field_name, uint8_t *out, size_t out_len, size_t *out_used) { + char needle[64]; + const char *p; + if (strlen(field_name) + 4u >= sizeof(needle)) return LOX_ERR_INVALID; + snprintf(needle, sizeof(needle), "\"%s\"", field_name); + p = strstr(obj, needle); + if (p == NULL) return LOX_ERR_NOT_FOUND; + p += strlen(needle); + ie_skip_ws(&p); + if (*p != ':') return LOX_ERR_INVALID; + p++; + return ie_parse_json_hex_string(&p, out, out_len, out_used); +} + +static const lox_ie_ts_stream_desc_t *ie_find_ts_desc(const lox_ie_ts_stream_desc_t *streams, + size_t stream_count, + const char *name) { + size_t i; + for (i = 0u; i < stream_count; ++i) { + if (streams[i].name != NULL && strcmp(streams[i].name, name) == 0) return &streams[i]; + } + return NULL; +} + +static const lox_ie_rel_table_desc_t *ie_find_rel_desc(const lox_ie_rel_table_desc_t *tables, + size_t table_count, + const char *name) { + size_t i; + for (i = 0u; i < table_count; ++i) { + if (tables[i].name != NULL && strcmp(tables[i].name, name) == 0) return &tables[i]; + } + return NULL; +} + +static const char *ie_ts_type_to_str(lox_ts_type_t type) { + switch (type) { + case LOX_TS_F32: return "f32"; + case LOX_TS_I32: return "i32"; + case LOX_TS_U32: return "u32"; + case LOX_TS_RAW: return "raw"; + default: return NULL; + } +} + +static lox_err_t ie_ts_type_from_str(const char *s, lox_ts_type_t *out) { + if (strcmp(s, "f32") == 0) { + *out = LOX_TS_F32; + return LOX_OK; + } + if (strcmp(s, "i32") == 0) { + *out = LOX_TS_I32; + return LOX_OK; + } + if (strcmp(s, "u32") == 0) { + *out = LOX_TS_U32; + return LOX_OK; + } + if (strcmp(s, "raw") == 0) { + *out = LOX_TS_RAW; + return LOX_OK; + } + return LOX_ERR_INVALID; +} + +static lox_err_t ie_ts_value_size(const lox_ie_ts_stream_desc_t *desc, size_t *out_size) { + if (desc == NULL || out_size == NULL) return LOX_ERR_INVALID; + switch (desc->type) { + case LOX_TS_F32: + case LOX_TS_I32: + case LOX_TS_U32: + *out_size = sizeof(uint32_t); + return LOX_OK; + case LOX_TS_RAW: + if (desc->raw_size == 0u || desc->raw_size > LOX_TS_RAW_MAX) return LOX_ERR_INVALID; + *out_size = desc->raw_size; + return LOX_OK; + default: + return LOX_ERR_INVALID; + } +} + +static bool ie_ts_export_cb(const lox_ts_sample_t *sample, void *ctx) { + ie_ts_export_ctx_t *x = (ie_ts_export_ctx_t *)ctx; + uint8_t bytes[LOX_TS_RAW_MAX]; + size_t value_size = 0u; + const char *type_name; + lox_err_t rc; + + if (x->rc != LOX_OK) return false; + rc = ie_ts_value_size(x->desc, &value_size); + if (rc != LOX_OK) { + x->rc = rc; + return false; + } + + switch (x->desc->type) { + case LOX_TS_F32: + memcpy(bytes, &sample->v.f32, sizeof(float)); + break; + case LOX_TS_I32: + memcpy(bytes, &sample->v.i32, sizeof(int32_t)); + break; + case LOX_TS_U32: + memcpy(bytes, &sample->v.u32, sizeof(uint32_t)); + break; + case LOX_TS_RAW: + memcpy(bytes, sample->v.raw, value_size); + break; + default: + x->rc = LOX_ERR_INVALID; + return false; + } + + if (*(x->exported) > 0u) { + rc = ie_append_char(x->out, x->out_len, x->pos, ','); + if (rc != LOX_OK) { + x->rc = rc; + return false; + } + } + + type_name = ie_ts_type_to_str(x->desc->type); + if (type_name == NULL) { + x->rc = LOX_ERR_INVALID; + return false; + } + + rc = ie_append(x->out, x->out_len, x->pos, "{\"stream\":\""); + if (rc != LOX_OK) { x->rc = rc; return false; } + rc = ie_append(x->out, x->out_len, x->pos, x->desc->name); + if (rc != LOX_OK) { x->rc = rc; return false; } + rc = ie_append(x->out, x->out_len, x->pos, "\",\"type\":\""); + if (rc != LOX_OK) { x->rc = rc; return false; } + rc = ie_append(x->out, x->out_len, x->pos, type_name); + if (rc != LOX_OK) { x->rc = rc; return false; } + rc = ie_append(x->out, x->out_len, x->pos, "\",\"ts\":"); + if (rc != LOX_OK) { x->rc = rc; return false; } + rc = ie_append_u32(x->out, x->out_len, x->pos, (uint32_t)sample->ts); + if (rc != LOX_OK) { x->rc = rc; return false; } + rc = ie_append(x->out, x->out_len, x->pos, ",\"value_hex\":\""); + if (rc != LOX_OK) { x->rc = rc; return false; } + rc = ie_append_hex(x->out, x->out_len, x->pos, bytes, value_size); + if (rc != LOX_OK) { x->rc = rc; return false; } + rc = ie_append(x->out, x->out_len, x->pos, "\"}"); + if (rc != LOX_OK) { x->rc = rc; return false; } + + (*(x->exported))++; + return true; +} + +static bool ie_rel_export_cb(const void *row_buf, void *ctx) { + ie_rel_export_ctx_t *x = (ie_rel_export_ctx_t *)ctx; + lox_err_t rc; + if (x->rc != LOX_OK) return false; + + if (*(x->exported) > 0u) { + rc = ie_append_char(x->out, x->out_len, x->pos, ','); + if (rc != LOX_OK) { x->rc = rc; return false; } + } + + rc = ie_append(x->out, x->out_len, x->pos, "{\"table\":\""); + if (rc != LOX_OK) { x->rc = rc; return false; } + rc = ie_append(x->out, x->out_len, x->pos, x->table_name); + if (rc != LOX_OK) { x->rc = rc; return false; } + rc = ie_append(x->out, x->out_len, x->pos, "\",\"row_hex\":\""); + if (rc != LOX_OK) { x->rc = rc; return false; } + rc = ie_append_hex(x->out, x->out_len, x->pos, (const uint8_t *)row_buf, x->row_size); + if (rc != LOX_OK) { x->rc = rc; return false; } + rc = ie_append(x->out, x->out_len, x->pos, "\"}"); + if (rc != LOX_OK) { x->rc = rc; return false; } + + (*(x->exported))++; + return true; +} + +lox_ie_options_t lox_ie_default_options(void) { + lox_ie_options_t o; + o.overwrite_existing = 0u; + o.skip_invalid_items = 0u; + return o; +} + +lox_err_t lox_ie_export_kv_json(lox_t *db, + const char *const *keys, + size_t key_count, + char *out_json, + size_t out_json_len, + size_t *out_used, + uint32_t *out_exported) { + size_t pos = 0u; + size_t i; + uint32_t exported = 0u; + lox_err_t rc; + if (db == NULL || keys == NULL || out_json == NULL || out_json_len == 0u || out_used == NULL || out_exported == NULL) { + return LOX_ERR_INVALID; + } + + rc = ie_append(out_json, out_json_len, &pos, "{\"format\":\"loxdb.kv.v1\",\"items\":["); + if (rc != LOX_OK) return rc; + + for (i = 0u; i < key_count; ++i) { + const char *key = keys[i]; + uint8_t value_buf[LOX_KV_VAL_MAX_LEN]; + size_t value_len = 0u; + uint32_t ttl = 0u; + char rec[1024]; + size_t rec_used = 0u; + + if (key == NULL || key[0] == '\0') return LOX_ERR_INVALID; + rc = lox_kv_get(db, key, value_buf, sizeof(value_buf), &value_len); + if (rc != LOX_OK) return rc; + rc = ie_lookup_ttl(db, key, &ttl); + if (rc != LOX_OK) return rc; + if (ttl == UINT32_MAX) { + /* kv_iter uses UINT32_MAX as "no expiry" sentinel; JSON IE uses ttl=0 for persistent keys. */ + ttl = 0u; + } + + rc = lox_json_encode_kv_record(key, value_buf, value_len, ttl, rec, sizeof(rec), &rec_used); + if (rc != LOX_OK) return rc; + if (exported > 0u) { + rc = ie_append(out_json, out_json_len, &pos, ","); + if (rc != LOX_OK) return rc; + } + rc = ie_append(out_json, out_json_len, &pos, rec); + if (rc != LOX_OK) return rc; + exported++; + } + + rc = ie_append(out_json, out_json_len, &pos, "]}"); + if (rc != LOX_OK) return rc; + if (pos >= out_json_len) return LOX_ERR_OVERFLOW; + out_json[pos] = '\0'; + *out_used = pos; + *out_exported = exported; + return LOX_OK; +} + +lox_err_t lox_ie_import_kv_json(lox_t *db, + const char *json, + const lox_ie_options_t *options, + uint32_t *out_imported, + uint32_t *out_skipped) { + const lox_ie_options_t default_opts = lox_ie_default_options(); + const lox_ie_options_t *opts = options != NULL ? options : &default_opts; + const char *p; + uint32_t imported = 0u; + uint32_t skipped = 0u; + + if (db == NULL || json == NULL || out_imported == NULL || out_skipped == NULL) return LOX_ERR_INVALID; + p = ie_find_items_array(json); + if (p == NULL || *p != '[') return LOX_ERR_INVALID; + p++; + + for (;;) { + lox_err_t rc; + char obj[1024]; + const char *obj_end; + size_t obj_len; + char key[LOX_KV_KEY_MAX_LEN]; + uint8_t value[LOX_KV_VAL_MAX_LEN]; + size_t value_len = 0u; + uint32_t ttl = 0u; + lox_err_t exists_rc; + + ie_skip_ws(&p); + if (*p == ']') { + p++; + break; + } + if (*p != '{') return LOX_ERR_INVALID; + obj_end = ie_find_obj_end(p); + if (obj_end == NULL) return LOX_ERR_INVALID; + obj_len = (size_t)(obj_end - p + 1); + if (obj_len >= sizeof(obj)) return LOX_ERR_OVERFLOW; + memcpy(obj, p, obj_len); + obj[obj_len] = '\0'; + p = obj_end + 1; + + rc = lox_json_decode_kv_record(obj, key, sizeof(key), value, sizeof(value), &value_len, &ttl); + if (rc != LOX_OK) { + if (opts->skip_invalid_items) { + skipped++; + } else { + return rc; + } + } else { + exists_rc = lox_kv_exists(db, key); + if (!opts->overwrite_existing && exists_rc == LOX_OK) { + skipped++; + } else { + rc = lox_kv_set(db, key, value, value_len, ttl); + if (rc != LOX_OK) { + if (opts->skip_invalid_items) { + skipped++; + } else { + return rc; + } + } else { + imported++; + } + } + } + + ie_skip_ws(&p); + if (*p == ',') { + p++; + continue; + } + if (*p == ']') { + p++; + break; + } + return LOX_ERR_INVALID; + } + + ie_skip_ws(&p); + if (*p == '}') { + p++; + ie_skip_ws(&p); + } + if (*p != '\0') return LOX_ERR_INVALID; + *out_imported = imported; + *out_skipped = skipped; + return LOX_OK; +} + +lox_err_t lox_ie_export_ts_json(lox_t *db, + const lox_ie_ts_stream_desc_t *streams, + size_t stream_count, + lox_timestamp_t from, + lox_timestamp_t to, + char *out_json, + size_t out_json_len, + size_t *out_used, + uint32_t *out_exported) { + size_t pos = 0u; + size_t i; + uint32_t exported = 0u; + lox_err_t rc; + + if (db == NULL || streams == NULL || out_json == NULL || out_json_len == 0u || out_used == NULL || out_exported == NULL) { + return LOX_ERR_INVALID; + } + + rc = ie_append(out_json, out_json_len, &pos, "{\"format\":\"loxdb.ts.v1\",\"items\":["); + if (rc != LOX_OK) return rc; + + for (i = 0u; i < stream_count; ++i) { + ie_ts_export_ctx_t ctx; + if (streams[i].name == NULL || streams[i].name[0] == '\0') return LOX_ERR_INVALID; + rc = ie_ts_value_size(&streams[i], &ctx.out_len); + if (rc != LOX_OK) return rc; + + ctx.out = out_json; + ctx.out_len = out_json_len; + ctx.pos = &pos; + ctx.desc = &streams[i]; + ctx.exported = &exported; + ctx.rc = LOX_OK; + + rc = lox_ts_query(db, streams[i].name, from, to, ie_ts_export_cb, &ctx); + if (rc != LOX_OK) return rc; + if (ctx.rc != LOX_OK) return ctx.rc; + } + + rc = ie_append(out_json, out_json_len, &pos, "]}"); + if (rc != LOX_OK) return rc; + if (pos >= out_json_len) return LOX_ERR_OVERFLOW; + out_json[pos] = '\0'; + *out_used = pos; + *out_exported = exported; + return LOX_OK; +} + +lox_err_t lox_ie_import_ts_json(lox_t *db, + const char *json, + const lox_ie_ts_stream_desc_t *streams, + size_t stream_count, + const lox_ie_options_t *options, + uint32_t *out_imported, + uint32_t *out_skipped) { + const lox_ie_options_t default_opts = lox_ie_default_options(); + const lox_ie_options_t *opts = options != NULL ? options : &default_opts; + const char *p; + uint32_t imported = 0u; + uint32_t skipped = 0u; + + if (db == NULL || json == NULL || streams == NULL || out_imported == NULL || out_skipped == NULL) return LOX_ERR_INVALID; + p = ie_find_items_array(json); + if (p == NULL || *p != '[') return LOX_ERR_INVALID; + p++; + + for (;;) { + char obj[1024]; + const char *obj_end; + size_t obj_len; + char stream_name[LOX_TS_STREAM_NAME_LEN + 1u]; + char type_name[16]; + uint32_t ts = 0u; + uint8_t bytes[LOX_TS_RAW_MAX]; + size_t bytes_len = 0u; + lox_ts_type_t item_type; + const lox_ie_ts_stream_desc_t *desc; + size_t expected_len = 0u; + lox_err_t rc; + + ie_skip_ws(&p); + if (*p == ']') { + p++; + break; + } + if (*p != '{') return LOX_ERR_INVALID; + obj_end = ie_find_obj_end(p); + if (obj_end == NULL) return LOX_ERR_INVALID; + obj_len = (size_t)(obj_end - p + 1); + if (obj_len >= sizeof(obj)) return LOX_ERR_OVERFLOW; + memcpy(obj, p, obj_len); + obj[obj_len] = '\0'; + p = obj_end + 1; + + rc = ie_parse_object_field_string(obj, "stream", stream_name, sizeof(stream_name)); + if (rc != LOX_OK) goto ts_item_error; + rc = ie_parse_object_field_string(obj, "type", type_name, sizeof(type_name)); + if (rc != LOX_OK) goto ts_item_error; + rc = ie_parse_object_field_u32(obj, "ts", &ts); + if (rc != LOX_OK) goto ts_item_error; + rc = ie_parse_object_field_hex(obj, "value_hex", bytes, sizeof(bytes), &bytes_len); + if (rc != LOX_OK) goto ts_item_error; + + rc = ie_ts_type_from_str(type_name, &item_type); + if (rc != LOX_OK) goto ts_item_error; + desc = ie_find_ts_desc(streams, stream_count, stream_name); + if (desc == NULL) { + rc = LOX_ERR_NOT_FOUND; + goto ts_item_error; + } + if (desc->type != item_type) { + rc = LOX_ERR_SCHEMA; + goto ts_item_error; + } + rc = ie_ts_value_size(desc, &expected_len); + if (rc != LOX_OK) goto ts_item_error; + if (bytes_len != expected_len) { + rc = LOX_ERR_INVALID; + goto ts_item_error; + } + + rc = lox_ts_insert(db, stream_name, (lox_timestamp_t)ts, bytes); + if (rc != LOX_OK) goto ts_item_error; + imported++; + goto ts_item_next; + + ts_item_error: + if (opts->skip_invalid_items) { + skipped++; + } else { + return rc; + } + + ts_item_next: + ie_skip_ws(&p); + if (*p == ',') { + p++; + continue; + } + if (*p == ']') { + p++; + break; + } + return LOX_ERR_INVALID; + } + + ie_skip_ws(&p); + if (*p == '}') { + p++; + ie_skip_ws(&p); + } + if (*p != '\0') return LOX_ERR_INVALID; + *out_imported = imported; + *out_skipped = skipped; + return LOX_OK; +} + +lox_err_t lox_ie_export_rel_json(lox_t *db, + const lox_ie_rel_table_desc_t *tables, + size_t table_count, + char *out_json, + size_t out_json_len, + size_t *out_used, + uint32_t *out_exported) { + size_t pos = 0u; + size_t i; + uint32_t exported = 0u; + lox_err_t rc; + + if (db == NULL || tables == NULL || out_json == NULL || out_json_len == 0u || out_used == NULL || out_exported == NULL) { + return LOX_ERR_INVALID; + } + + rc = ie_append(out_json, out_json_len, &pos, "{\"format\":\"loxdb.rel.v1\",\"items\":["); + if (rc != LOX_OK) return rc; + + for (i = 0u; i < table_count; ++i) { + lox_table_t *table = NULL; + size_t actual_row_size; + ie_rel_export_ctx_t ctx; + + if (tables[i].name == NULL || tables[i].name[0] == '\0') return LOX_ERR_INVALID; + rc = lox_table_get(db, tables[i].name, &table); + if (rc != LOX_OK) return rc; + actual_row_size = lox_table_row_size(table); + if (actual_row_size == 0u || actual_row_size > 1024u) return LOX_ERR_INVALID; + if (tables[i].row_size != 0u && tables[i].row_size != actual_row_size) return LOX_ERR_SCHEMA; + + ctx.out = out_json; + ctx.out_len = out_json_len; + ctx.pos = &pos; + ctx.table_name = tables[i].name; + ctx.row_size = actual_row_size; + ctx.exported = &exported; + ctx.rc = LOX_OK; + + rc = lox_rel_iter(db, table, ie_rel_export_cb, &ctx); + if (rc != LOX_OK) return rc; + if (ctx.rc != LOX_OK) return ctx.rc; + } + + rc = ie_append(out_json, out_json_len, &pos, "]}"); + if (rc != LOX_OK) return rc; + if (pos >= out_json_len) return LOX_ERR_OVERFLOW; + out_json[pos] = '\0'; + *out_used = pos; + *out_exported = exported; + return LOX_OK; +} + +lox_err_t lox_ie_import_rel_json(lox_t *db, + const char *json, + const lox_ie_rel_table_desc_t *tables, + size_t table_count, + const lox_ie_options_t *options, + uint32_t *out_imported, + uint32_t *out_skipped) { + const lox_ie_options_t default_opts = lox_ie_default_options(); + const lox_ie_options_t *opts = options != NULL ? options : &default_opts; + const char *p; + uint32_t imported = 0u; + uint32_t skipped = 0u; + + if (db == NULL || json == NULL || tables == NULL || out_imported == NULL || out_skipped == NULL) return LOX_ERR_INVALID; + p = ie_find_items_array(json); + if (p == NULL || *p != '[') return LOX_ERR_INVALID; + p++; + + for (;;) { + char obj[2048]; + const char *obj_end; + size_t obj_len; + char table_name[LOX_REL_TABLE_NAME_LEN + 1u]; + uint8_t row[1024]; + size_t row_len = 0u; + const lox_ie_rel_table_desc_t *desc; + lox_table_t *table = NULL; + size_t actual_row_size; + lox_err_t rc; + + ie_skip_ws(&p); + if (*p == ']') { + p++; + break; + } + if (*p != '{') return LOX_ERR_INVALID; + obj_end = ie_find_obj_end(p); + if (obj_end == NULL) return LOX_ERR_INVALID; + obj_len = (size_t)(obj_end - p + 1); + if (obj_len >= sizeof(obj)) return LOX_ERR_OVERFLOW; + memcpy(obj, p, obj_len); + obj[obj_len] = '\0'; + p = obj_end + 1; + + rc = ie_parse_object_field_string(obj, "table", table_name, sizeof(table_name)); + if (rc != LOX_OK) goto rel_item_error; + rc = ie_parse_object_field_hex(obj, "row_hex", row, sizeof(row), &row_len); + if (rc != LOX_OK) goto rel_item_error; + + desc = ie_find_rel_desc(tables, table_count, table_name); + if (desc == NULL) { + rc = LOX_ERR_NOT_FOUND; + goto rel_item_error; + } + + rc = lox_table_get(db, table_name, &table); + if (rc != LOX_OK) goto rel_item_error; + actual_row_size = lox_table_row_size(table); + if (desc->row_size != 0u && desc->row_size != actual_row_size) { + rc = LOX_ERR_SCHEMA; + goto rel_item_error; + } + if (row_len != actual_row_size) { + rc = LOX_ERR_INVALID; + goto rel_item_error; + } + + rc = lox_rel_insert(db, table, row); + if (rc == LOX_ERR_EXISTS) { + skipped++; + goto rel_item_next; + } + if (rc != LOX_OK) goto rel_item_error; + imported++; + goto rel_item_next; + + rel_item_error: + if (opts->skip_invalid_items) { + skipped++; + } else { + return rc; + } + + rel_item_next: + ie_skip_ws(&p); + if (*p == ',') { + p++; + continue; + } + if (*p == ']') { + p++; + break; + } + return LOX_ERR_INVALID; + } + + ie_skip_ws(&p); + if (*p == '}') { + p++; + ie_skip_ws(&p); + } + if (*p != '\0') return LOX_ERR_INVALID; + *out_imported = imported; + *out_skipped = skipped; + return LOX_OK; +} diff --git a/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/src/lox_internal.h b/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/src/lox_internal.h new file mode 100644 index 0000000..a41b25a --- /dev/null +++ b/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/src/lox_internal.h @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: MIT +#ifndef LOX_INTERNAL_H +#define LOX_INTERNAL_H + +#include "lox.h" + +#define LOX_MAGIC 0x4D444230u + +typedef struct { + uint8_t *base; + size_t used; + size_t capacity; +} lox_arena_t; + +typedef struct { + uint8_t state; + uint32_t key_hash; + char key[LOX_KV_KEY_MAX_LEN]; + uint32_t val_offset; + uint32_t val_len; + uint32_t expires_at; + uint32_t last_access; +} lox_kv_bucket_t; + +typedef struct { + lox_kv_bucket_t *buckets; + uint32_t bucket_count; + uint32_t entry_count; + uint32_t collision_count; + uint32_t eviction_count; + uint8_t *value_store; + uint32_t value_capacity; + uint32_t value_used; + uint32_t live_value_bytes; + uint32_t access_clock; +} lox_kv_state_t; + +typedef struct { + char key[LOX_KV_KEY_MAX_LEN]; + void *val_ptr; + size_t val_len; + uint32_t expires_at; + uint8_t op; + uint8_t val_buf[LOX_KV_VAL_MAX_LEN]; +} lox_txn_stage_entry_t; + +typedef struct { + char name[LOX_TS_STREAM_NAME_LEN]; + lox_ts_type_t type; + size_t raw_size; + uint32_t sample_stride; + uint32_t head; + uint32_t tail; + uint32_t count; + uint32_t capacity; + uint8_t *buf; + bool registered; +} lox_ts_stream_t; + +typedef struct { + lox_ts_stream_t streams[LOX_TS_MAX_STREAMS]; + uint32_t registered_streams; + uint32_t mutation_seq; +} lox_ts_state_t; + +typedef struct { + char name[LOX_REL_COL_NAME_LEN]; + lox_col_type_t type; + size_t size; + size_t offset; + bool is_index; +} lox_col_desc_t; + +typedef struct { + uint8_t key_bytes[LOX_REL_INDEX_KEY_MAX]; + uint32_t row_idx; +} lox_index_entry_t; + +struct lox_table_s { + char name[LOX_REL_TABLE_NAME_LEN]; + uint16_t schema_version; + lox_col_desc_t cols[LOX_REL_MAX_COLS]; + uint32_t col_count; + uint32_t max_rows; + size_t row_size; + uint32_t index_col; + size_t index_key_size; + uint8_t *rows; + uint8_t *alive_bitmap; + lox_index_entry_t *index; + uint32_t *order; + uint32_t live_count; + uint32_t index_count; + uint32_t order_count; + uint32_t mutation_seq; + bool registered; +}; + +typedef struct { + struct lox_table_s tables[LOX_REL_MAX_TABLES]; + uint32_t registered_tables; +} lox_rel_state_t; + +typedef struct { + uint32_t wal_offset; + uint32_t wal_size; + uint32_t super_a_offset; + uint32_t super_b_offset; + uint32_t super_size; + uint32_t bank_a_offset; + uint32_t bank_b_offset; + uint32_t bank_size; + uint32_t kv_size; + uint32_t ts_size; + uint32_t rel_size; + uint32_t total_size; + uint32_t active_bank; + uint32_t active_generation; +} lox_storage_layout_t; + +typedef struct { + uint32_t magic; + uint8_t *heap; + size_t heap_size; + size_t live_bytes; + lox_storage_t *storage; + lox_timestamp_t (*now)(void); + void (*lock)(void *hdl); + void (*unlock)(void *hdl); + void (*lock_destroy)(void *hdl); + void *lock_handle; + uint32_t storage_bytes_written; + uint32_t compact_count; + uint32_t reopen_count; + uint32_t recovery_count; + lox_err_t last_runtime_error; + lox_err_t last_recovery_status; + bool wal_enabled; + lox_arena_t arena; + lox_arena_t kv_arena; + lox_arena_t ts_arena; + lox_arena_t rel_arena; + lox_kv_state_t kv; + lox_txn_stage_entry_t *txn_stage; + uint8_t txn_active; + uint32_t txn_stage_count; + lox_ts_state_t ts; + lox_rel_state_t rel; + lox_storage_layout_t layout; + uint32_t wal_sequence; + uint32_t wal_entry_count; + uint32_t wal_used; + uint8_t wal_compact_auto; + uint8_t wal_compact_threshold_pct; + uint8_t wal_sync_mode; + lox_err_t (*on_migrate)(lox_t *db, const char *table_name, uint16_t old_version, uint16_t new_version); + bool storage_loading; + bool wal_replaying; + uint32_t ts_dropped_samples; + bool migration_in_progress; +} lox_core_t; + +typedef struct { + char name[LOX_REL_TABLE_NAME_LEN]; + uint16_t schema_version; + lox_col_desc_t cols[LOX_REL_MAX_COLS]; + uint32_t col_count; + uint32_t max_rows; + size_t row_size; + uint32_t index_col; + bool sealed; +} lox_schema_impl_t; + +LOX_STATIC_ASSERT(core_size_fits, sizeof(lox_core_t) <= sizeof(((lox_t *)0)->_opaque)); +LOX_STATIC_ASSERT(schema_size_fits, sizeof(lox_schema_impl_t) <= sizeof(((lox_schema_t *)0)->_opaque)); +LOX_STATIC_ASSERT(table_size_fits, sizeof(struct lox_table_s) >= (LOX_REL_TABLE_NAME_LEN + sizeof(size_t))); + +lox_core_t *lox_core(lox_t *db); +const lox_core_t *lox_core_const(const lox_t *db); +lox_err_t lox_kv_init(lox_t *db); +lox_err_t lox_ts_init(lox_t *db); +size_t lox_kv_live_bytes(const lox_t *db); +lox_err_t lox_kv_set_at(lox_t *db, const char *key, const void *val, size_t len, uint32_t expires_at); +lox_err_t lox_storage_bootstrap(lox_t *db); +lox_err_t lox_storage_flush(lox_t *db); +lox_err_t lox_persist_kv_set(lox_t *db, const char *key, const void *val, size_t len, uint32_t expires_at); +lox_err_t lox_persist_kv_del(lox_t *db, const char *key); +lox_err_t lox_persist_kv_clear(lox_t *db); +lox_err_t lox_persist_kv_set_txn(lox_t *db, const char *key, const void *val, size_t len, uint32_t expires_at); +lox_err_t lox_persist_kv_del_txn(lox_t *db, const char *key); +lox_err_t lox_persist_txn_commit(lox_t *db); +lox_err_t lox_persist_ts_insert(lox_t *db, const char *name, lox_timestamp_t ts, const void *val, size_t val_len); +lox_err_t lox_persist_ts_register(lox_t *db, const char *name, lox_ts_type_t type, size_t raw_size); +lox_err_t lox_persist_ts_clear(lox_t *db, const char *name); +lox_err_t lox_persist_rel_insert(lox_t *db, const lox_table_t *table, const void *row_buf); +lox_err_t lox_persist_rel_delete(lox_t *db, const lox_table_t *table, const void *search_val); +lox_err_t lox_persist_rel_table_create(lox_t *db, const lox_schema_t *schema); +lox_err_t lox_persist_rel_clear(lox_t *db, const lox_table_t *table); + +static inline void lox__maybe_compact(lox_t *db) { + lox_core_t *core = lox_core(db); + uint32_t wal_total; + uint32_t wal_used; + uint32_t wal_fill_pct; + uint32_t threshold; + + if (!core->wal_enabled || core->layout.wal_size <= 32u || core->wal_compact_auto == 0u) { + return; + } + + wal_total = core->layout.wal_size - 32u; + wal_used = (core->wal_used > 32u) ? (core->wal_used - 32u) : 0u; + wal_fill_pct = (wal_total == 0u) ? 0u : ((wal_used * 100u) / wal_total); + threshold = (core->wal_compact_threshold_pct != 0u) ? core->wal_compact_threshold_pct : 75u; + if (wal_fill_pct >= threshold) { + (void)lox_storage_flush(db); + } +} + +static inline void lox_record_error(lox_core_t *core, lox_err_t err) { + if (core != NULL && err != LOX_OK) { + core->last_runtime_error = err; + } +} + +#endif diff --git a/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/src/lox_json_wrapper.c b/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/src/lox_json_wrapper.c new file mode 100644 index 0000000..e8329ff --- /dev/null +++ b/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/src/lox_json_wrapper.c @@ -0,0 +1,347 @@ +// SPDX-License-Identifier: MIT +#include "lox_json_wrapper.h" + +#include +#include +#include +#include + +static int json_is_hex_char(char c) { + return ((c >= '0' && c <= '9') || + (c >= 'a' && c <= 'f') || + (c >= 'A' && c <= 'F')); +} + +static uint8_t json_hex_val(char c) { + if (c >= '0' && c <= '9') return (uint8_t)(c - '0'); + if (c >= 'a' && c <= 'f') return (uint8_t)(10 + (c - 'a')); + return (uint8_t)(10 + (c - 'A')); +} + +static lox_err_t json_append_char(char *out, size_t out_len, size_t *pos, char c) { + if (*pos >= out_len) return LOX_ERR_OVERFLOW; + out[*pos] = c; + (*pos)++; + return LOX_OK; +} + +static lox_err_t json_append_str(char *out, size_t out_len, size_t *pos, const char *s) { + size_t n = strlen(s); + if (*pos + n > out_len) return LOX_ERR_OVERFLOW; + memcpy(out + *pos, s, n); + *pos += n; + return LOX_OK; +} + +static lox_err_t json_append_escaped(char *out, size_t out_len, size_t *pos, const char *s) { + static const char hx[] = "0123456789ABCDEF"; + while (*s != '\0') { + unsigned char c = (unsigned char)*s++; + lox_err_t rc; + if (c == '"' || c == '\\') { + rc = json_append_char(out, out_len, pos, '\\'); + if (rc != LOX_OK) return rc; + rc = json_append_char(out, out_len, pos, (char)c); + if (rc != LOX_OK) return rc; + } else if (c < 0x20u) { + rc = json_append_str(out, out_len, pos, "\\u00"); + if (rc != LOX_OK) return rc; + rc = json_append_char(out, out_len, pos, hx[(c >> 4) & 0x0Fu]); + if (rc != LOX_OK) return rc; + rc = json_append_char(out, out_len, pos, hx[c & 0x0Fu]); + if (rc != LOX_OK) return rc; + } else { + rc = json_append_char(out, out_len, pos, (char)c); + if (rc != LOX_OK) return rc; + } + } + return LOX_OK; +} + +static lox_err_t json_append_hex(char *out, size_t out_len, size_t *pos, const uint8_t *buf, size_t len) { + static const char hx[] = "0123456789ABCDEF"; + size_t i; + if (*pos + (len * 2u) > out_len) return LOX_ERR_OVERFLOW; + for (i = 0u; i < len; ++i) { + out[*pos + i * 2u] = hx[(buf[i] >> 4) & 0x0Fu]; + out[*pos + i * 2u + 1u] = hx[buf[i] & 0x0Fu]; + } + *pos += len * 2u; + return LOX_OK; +} + +static void json_skip_ws(const char **p) { + while (**p != '\0' && isspace((unsigned char)**p)) (*p)++; +} + +static lox_err_t json_parse_u32(const char **p, uint32_t *out) { + unsigned long v; + char *end = NULL; + json_skip_ws(p); + if (**p == '\0' || !isdigit((unsigned char)**p)) return LOX_ERR_INVALID; + v = strtoul(*p, &end, 10); + if (end == NULL || end == *p || v > 0xFFFFFFFFul) return LOX_ERR_INVALID; + *p = end; + *out = (uint32_t)v; + return LOX_OK; +} + +static lox_err_t json_parse_string(const char **p, char *out, size_t out_len) { + size_t pos = 0u; + json_skip_ws(p); + if (**p != '"') return LOX_ERR_INVALID; + (*p)++; + while (**p != '\0') { + char c = **p; + if (c == '"') { + (*p)++; + if (pos >= out_len) return LOX_ERR_OVERFLOW; + out[pos] = '\0'; + return LOX_OK; + } + if (c == '\\') { + (*p)++; + c = **p; + if (c == '\0') return LOX_ERR_INVALID; + switch (c) { + case '"': + case '\\': + case '/': + break; + case 'b': + c = '\b'; + break; + case 'f': + c = '\f'; + break; + case 'n': + c = '\n'; + break; + case 'r': + c = '\r'; + break; + case 't': + c = '\t'; + break; + case 'u': + if ((*p)[1] == '0' && (*p)[2] == '0' && + json_is_hex_char((*p)[3]) && json_is_hex_char((*p)[4])) { + c = (char)((json_hex_val((*p)[3]) << 4) | json_hex_val((*p)[4])); + (*p) += 4; + } else { + return LOX_ERR_INVALID; + } + break; + default: + return LOX_ERR_INVALID; + } + } + if (pos + 1u >= out_len) return LOX_ERR_OVERFLOW; + out[pos++] = c; + (*p)++; + } + return LOX_ERR_INVALID; +} + +static lox_err_t json_parse_hex_string(const char **p, uint8_t *out, size_t out_len, size_t *out_len_used) { + size_t n = 0u; + json_skip_ws(p); + if (**p != '"') return LOX_ERR_INVALID; + (*p)++; + while (**p != '\0' && **p != '"') { + if (!json_is_hex_char((*p)[0]) || !json_is_hex_char((*p)[1])) return LOX_ERR_INVALID; + if (n >= out_len) return LOX_ERR_OVERFLOW; + out[n++] = (uint8_t)((json_hex_val((*p)[0]) << 4) | json_hex_val((*p)[1])); + (*p) += 2; + } + if (**p != '"') return LOX_ERR_INVALID; + (*p)++; + *out_len_used = n; + return LOX_OK; +} + +lox_err_t lox_json_kv_set_u32(lox_t *db, const char *key, uint32_t value, uint32_t ttl) { + char buf[16]; + int n = snprintf(buf, sizeof(buf), "%u", (unsigned)value); + if (n <= 0 || (size_t)n >= sizeof(buf)) return LOX_ERR_INVALID; + return lox_kv_set(db, key, buf, (size_t)n, ttl); +} + +lox_err_t lox_json_kv_get_u32(lox_t *db, const char *key, uint32_t *out_value) { + char buf[16]; + size_t out_len = 0u; + char *end = NULL; + unsigned long v; + lox_err_t rc; + if (out_value == NULL) return LOX_ERR_INVALID; + rc = lox_kv_get(db, key, buf, sizeof(buf) - 1u, &out_len); + if (rc != LOX_OK) return rc; + buf[out_len] = '\0'; + v = strtoul(buf, &end, 10); + if (end == NULL || *end != '\0' || v > 0xFFFFFFFFul) return LOX_ERR_INVALID; + *out_value = (uint32_t)v; + return LOX_OK; +} + +lox_err_t lox_json_kv_set_i32(lox_t *db, const char *key, int32_t value, uint32_t ttl) { + char buf[16]; + int n = snprintf(buf, sizeof(buf), "%d", (int)value); + if (n <= 0 || (size_t)n >= sizeof(buf)) return LOX_ERR_INVALID; + return lox_kv_set(db, key, buf, (size_t)n, ttl); +} + +lox_err_t lox_json_kv_get_i32(lox_t *db, const char *key, int32_t *out_value) { + char buf[16]; + size_t out_len = 0u; + char *end = NULL; + long v; + lox_err_t rc; + if (out_value == NULL) return LOX_ERR_INVALID; + rc = lox_kv_get(db, key, buf, sizeof(buf) - 1u, &out_len); + if (rc != LOX_OK) return rc; + buf[out_len] = '\0'; + v = strtol(buf, &end, 10); + if (end == NULL || *end != '\0' || v < (-2147483647L - 1L) || v > 2147483647L) return LOX_ERR_INVALID; + *out_value = (int32_t)v; + return LOX_OK; +} + +lox_err_t lox_json_kv_set_bool(lox_t *db, const char *key, bool value, uint32_t ttl) { + const char *s = value ? "true" : "false"; + return lox_kv_set(db, key, s, strlen(s), ttl); +} + +lox_err_t lox_json_kv_get_bool(lox_t *db, const char *key, bool *out_value) { + char buf[8]; + size_t out_len = 0u; + lox_err_t rc; + if (out_value == NULL) return LOX_ERR_INVALID; + rc = lox_kv_get(db, key, buf, sizeof(buf) - 1u, &out_len); + if (rc != LOX_OK) return rc; + buf[out_len] = '\0'; + if (strcmp(buf, "true") == 0) { + *out_value = true; + return LOX_OK; + } + if (strcmp(buf, "false") == 0) { + *out_value = false; + return LOX_OK; + } + return LOX_ERR_INVALID; +} + +lox_err_t lox_json_kv_set_cstr(lox_t *db, const char *key, const char *value, uint32_t ttl) { + if (value == NULL) return LOX_ERR_INVALID; + return lox_kv_set(db, key, value, strlen(value), ttl); +} + +lox_err_t lox_json_kv_get_cstr(lox_t *db, const char *key, char *out_buf, size_t out_buf_len, size_t *out_len) { + lox_err_t rc; + size_t n = 0u; + if (out_buf == NULL || out_buf_len == 0u) return LOX_ERR_INVALID; + rc = lox_kv_get(db, key, out_buf, out_buf_len - 1u, &n); + if (rc != LOX_OK) return rc; + out_buf[n] = '\0'; + if (out_len != NULL) *out_len = n; + return LOX_OK; +} + +lox_err_t lox_json_encode_kv_record(const char *key, + const void *value, + size_t value_len, + uint32_t ttl, + char *out_json, + size_t out_json_len, + size_t *out_used) { + size_t pos = 0u; + lox_err_t rc; + if (key == NULL || value == NULL || out_json == NULL || out_json_len == 0u || out_used == NULL) return LOX_ERR_INVALID; + rc = json_append_char(out_json, out_json_len, &pos, '{'); + if (rc != LOX_OK) return rc; + rc = json_append_str(out_json, out_json_len, &pos, "\"key\":\""); + if (rc != LOX_OK) return rc; + rc = json_append_escaped(out_json, out_json_len, &pos, key); + if (rc != LOX_OK) return rc; + rc = json_append_str(out_json, out_json_len, &pos, "\",\"ttl\":"); + if (rc != LOX_OK) return rc; + { + char num[16]; + int n = snprintf(num, sizeof(num), "%u", (unsigned)ttl); + if (n <= 0 || (size_t)n >= sizeof(num)) return LOX_ERR_INVALID; + rc = json_append_str(out_json, out_json_len, &pos, num); + if (rc != LOX_OK) return rc; + } + rc = json_append_str(out_json, out_json_len, &pos, ",\"value_hex\":\""); + if (rc != LOX_OK) return rc; + rc = json_append_hex(out_json, out_json_len, &pos, (const uint8_t *)value, value_len); + if (rc != LOX_OK) return rc; + rc = json_append_str(out_json, out_json_len, &pos, "\"}"); + if (rc != LOX_OK) return rc; + if (pos >= out_json_len) return LOX_ERR_OVERFLOW; + out_json[pos] = '\0'; + *out_used = pos; + return LOX_OK; +} + +lox_err_t lox_json_decode_kv_record(const char *json, + char *key_out, + size_t key_out_len, + uint8_t *value_out, + size_t value_out_len, + size_t *value_len_out, + uint32_t *ttl_out) { + const char *p = json; + char field[32]; + uint8_t seen_key = 0u; + uint8_t seen_ttl = 0u; + uint8_t seen_val = 0u; + if (json == NULL || key_out == NULL || key_out_len == 0u || + value_out == NULL || value_len_out == NULL || ttl_out == NULL) { + return LOX_ERR_INVALID; + } + json_skip_ws(&p); + if (*p != '{') return LOX_ERR_INVALID; + p++; + for (;;) { + lox_err_t rc; + json_skip_ws(&p); + if (*p == '}') { + p++; + break; + } + rc = json_parse_string(&p, field, sizeof(field)); + if (rc != LOX_OK) return rc; + json_skip_ws(&p); + if (*p != ':') return LOX_ERR_INVALID; + p++; + if (strcmp(field, "key") == 0) { + rc = json_parse_string(&p, key_out, key_out_len); + if (rc != LOX_OK) return rc; + seen_key = 1u; + } else if (strcmp(field, "ttl") == 0) { + rc = json_parse_u32(&p, ttl_out); + if (rc != LOX_OK) return rc; + seen_ttl = 1u; + } else if (strcmp(field, "value_hex") == 0) { + rc = json_parse_hex_string(&p, value_out, value_out_len, value_len_out); + if (rc != LOX_OK) return rc; + seen_val = 1u; + } else { + return LOX_ERR_INVALID; + } + json_skip_ws(&p); + if (*p == ',') { + p++; + continue; + } + if (*p == '}') { + p++; + break; + } + return LOX_ERR_INVALID; + } + json_skip_ws(&p); + if (*p != '\0') return LOX_ERR_INVALID; + if (!(seen_key && seen_ttl && seen_val)) return LOX_ERR_INVALID; + return LOX_OK; +} diff --git a/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/src/lox_kv.c b/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/src/lox_kv.c new file mode 100644 index 0000000..57e4714 --- /dev/null +++ b/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/src/lox_kv.c @@ -0,0 +1,1060 @@ +// SPDX-License-Identifier: MIT +#include "lox_internal.h" +#include "lox_lock.h" +#include "lox_arena.h" + +#include + +enum { + LOX_KV_BUCKET_EMPTY = 0, + LOX_KV_BUCKET_LIVE = 1, + LOX_KV_BUCKET_TOMBSTONE = 2, + LOX_TXN_OP_PUT = 0, + LOX_TXN_OP_DEL = 1 +}; + +static uint32_t lox_kv_entry_limit(void) { + return LOX_KV_MAX_KEYS - LOX_TXN_STAGE_KEYS; +} + +static uint32_t lox_align4_u32(uint32_t value) { + return (value + 3u) & ~3u; +} + +static uint32_t lox_kv_hash(const char *key) { + uint32_t hash = 2166136261u; + size_t i; + + for (i = 0; key[i] != '\0'; ++i) { + hash ^= (uint8_t)key[i]; + hash *= 16777619u; + } + + return hash; +} + +static uint32_t lox_kv_bucket_count(void) { + uint32_t key_limit = lox_kv_entry_limit(); + uint32_t required = (key_limit * 4u + 2u) / 3u; + uint32_t buckets = 1u; + + while (buckets < required) { + buckets <<= 1u; + } + + return buckets; +} + +static bool lox_kv_key_valid(const char *key) { + size_t len; + + if (key == NULL || key[0] == '\0') { + return false; + } + + len = strlen(key); + return len < LOX_KV_KEY_MAX_LEN; +} + +static lox_timestamp_t lox_now(const lox_core_t *core) { + if (core->now == NULL) { + return 0; + } + + return core->now(); +} + +static bool lox_kv_expired(const lox_core_t *core, const lox_kv_bucket_t *bucket) { +#if LOX_KV_ENABLE_TTL + if (bucket->expires_at == 0u) { + return false; + } + + return lox_now(core) >= (lox_timestamp_t)bucket->expires_at; +#else + (void)core; + (void)bucket; + return false; +#endif +} + +static uint32_t lox_kv_live_value_bytes(const lox_core_t *core) { + return core->kv.live_value_bytes; +} + +static uint32_t lox_kv_fragmented_bytes(const lox_core_t *core) { + return core->kv.value_used - lox_kv_live_value_bytes(core); +} + +static bool lox_kv_should_compact(const lox_core_t *core) { + if (core->kv.value_used == 0u) { + return false; + } + + return lox_kv_fragmented_bytes(core) * 2u > core->kv.value_used; +} + +static void lox_kv_compact(lox_core_t *core) { + uint8_t *dst = core->kv.value_store; + uint32_t i; + + LOX_LOG("INFO", + "KV val_pool compaction: used=%u/%u live=%u", + (unsigned)core->kv.value_used, + (unsigned)core->kv.value_capacity, + (unsigned)core->kv.entry_count); + + for (i = 0; i < core->kv.bucket_count; ++i) { + lox_kv_bucket_t *bucket = &core->kv.buckets[i]; + if (bucket->state != LOX_KV_BUCKET_LIVE) { + continue; + } + + if (bucket->val_len != 0u && dst != &core->kv.value_store[bucket->val_offset]) { + memmove(dst, &core->kv.value_store[bucket->val_offset], bucket->val_len); + bucket->val_offset = (uint32_t)(dst - core->kv.value_store); + } + + dst += bucket->val_len; + } + + core->kv.value_used = (uint32_t)(dst - core->kv.value_store); +} + +static void lox_kv_maybe_compact(lox_core_t *core) { + if (lox_kv_should_compact(core)) { + lox_kv_compact(core); + } +} + +static lox_err_t lox_kv_find_slot(lox_core_t *core, + const char *key, + uint32_t *slot_out, + bool *found_out, + uint32_t *probe_collisions_out) { + uint32_t search_hash = lox_kv_hash(key); + uint32_t mask = core->kv.bucket_count - 1u; + uint32_t idx = search_hash & mask; + uint32_t tombstone = UINT32_MAX; + uint32_t probed; + uint32_t probe_collisions = 0u; + + for (probed = 0; probed < core->kv.bucket_count; ++probed) { + lox_kv_bucket_t *bucket = &core->kv.buckets[idx]; + + if (bucket->state == LOX_KV_BUCKET_EMPTY) { + *slot_out = (tombstone != UINT32_MAX) ? tombstone : idx; + *found_out = false; + return LOX_OK; + } + + if (bucket->state == LOX_KV_BUCKET_TOMBSTONE) { + if (tombstone == UINT32_MAX) { + tombstone = idx; + } + } else if (bucket->key_hash == search_hash && + strncmp(bucket->key, key, LOX_KV_KEY_MAX_LEN) == 0) { + *slot_out = idx; + *found_out = true; + if (probe_collisions_out != NULL) { + *probe_collisions_out = probe_collisions; + } + return LOX_OK; + } else { + probe_collisions++; + } + + idx = (idx + 1u) & mask; + } + + if (tombstone != UINT32_MAX) { + *slot_out = tombstone; + *found_out = false; + if (probe_collisions_out != NULL) { + *probe_collisions_out = probe_collisions; + } + return LOX_OK; + } + + if (probe_collisions_out != NULL) { + *probe_collisions_out = probe_collisions; + } + return LOX_ERR_FULL; +} + +static void lox_kv_normalize_access_clock(lox_core_t *core) { + uint32_t i; + + if (core->kv.access_clock != UINT32_MAX) { + return; + } + for (i = 0u; i < core->kv.bucket_count; ++i) { + lox_kv_bucket_t *bucket = &core->kv.buckets[i]; + if (bucket->state == LOX_KV_BUCKET_LIVE) { + bucket->last_access = 1u; + } + } + core->kv.access_clock = 2u; +} + +static uint32_t lox_kv_next_access_clock(lox_core_t *core) { + lox_kv_normalize_access_clock(core); + return core->kv.access_clock++; +} + +static void lox_kv_remove_slot(lox_core_t *core, uint32_t idx) { + lox_kv_bucket_t *bucket = &core->kv.buckets[idx]; + + if (bucket->state == LOX_KV_BUCKET_LIVE && core->kv.entry_count != 0u) { + core->kv.live_value_bytes -= bucket->val_len; + core->kv.entry_count--; + } + + bucket->state = LOX_KV_BUCKET_TOMBSTONE; + bucket->key_hash = 0u; + bucket->key[0] = '\0'; + bucket->val_offset = 0u; + bucket->val_len = 0u; + bucket->expires_at = 0u; + bucket->last_access = 0u; +} + +static void lox_kv_shift_offsets(lox_core_t *core, uint32_t start_offset, int32_t delta) { + uint32_t i; + + for (i = 0; i < core->kv.bucket_count; ++i) { + lox_kv_bucket_t *bucket = &core->kv.buckets[i]; + if (bucket->state == LOX_KV_BUCKET_LIVE && bucket->val_offset > start_offset) { + bucket->val_offset = (uint32_t)((int32_t)bucket->val_offset + delta); + } + } +} + +static void lox_kv_write_bytes(uint8_t *dst, const void *val, size_t len) { + if (len != 0u) { + memcpy(dst, val, len); + } +} + +static lox_err_t lox_kv_overwrite_value(lox_core_t *core, + lox_kv_bucket_t *bucket, + const void *val, + size_t len) { + uint32_t old_offset = bucket->val_offset; + uint32_t old_len = bucket->val_len; + uint32_t tail_offset = old_offset + old_len; + uint32_t tail_len = core->kv.value_used - tail_offset; + + if (len == old_len) { + lox_kv_write_bytes(&core->kv.value_store[old_offset], val, len); + return LOX_OK; + } + + if (len < old_len) { + lox_kv_write_bytes(&core->kv.value_store[old_offset], val, len); + if (tail_len != 0u) { + memmove(&core->kv.value_store[old_offset + len], + &core->kv.value_store[tail_offset], + tail_len); + } + lox_kv_shift_offsets(core, old_offset, -((int32_t)(old_len - len))); + core->kv.value_used -= (old_len - (uint32_t)len); + bucket->val_len = (uint32_t)len; + core->kv.live_value_bytes -= old_len; + core->kv.live_value_bytes += (uint32_t)len; + return LOX_OK; + } + + if (core->kv.value_used + (len - old_len) > core->kv.value_capacity) { + return LOX_ERR_NO_MEM; + } + + if (tail_len != 0u) { + memmove(&core->kv.value_store[old_offset + len], + &core->kv.value_store[tail_offset], + tail_len); + } + lox_kv_shift_offsets(core, old_offset, (int32_t)(len - old_len)); + core->kv.value_used += (uint32_t)(len - old_len); + lox_kv_write_bytes(&core->kv.value_store[old_offset], val, len); + bucket->val_len = (uint32_t)len; + core->kv.live_value_bytes -= old_len; + core->kv.live_value_bytes += (uint32_t)len; + return LOX_OK; +} + +static lox_err_t lox_kv_append_value(lox_core_t *core, + lox_kv_bucket_t *bucket, + const void *val, + size_t len) { + if (core->kv.value_used + len > core->kv.value_capacity) { + lox_kv_compact(core); + } + + if (core->kv.value_used + len > core->kv.value_capacity) { + return LOX_ERR_NO_MEM; + } + + lox_kv_write_bytes(&core->kv.value_store[core->kv.value_used], val, len); + bucket->val_offset = core->kv.value_used; + bucket->val_len = (uint32_t)len; + core->kv.value_used += (uint32_t)len; + core->kv.live_value_bytes += (uint32_t)len; + return LOX_OK; +} + +static lox_err_t lox_kv_evict_lru(lox_core_t *core) { + uint32_t i; + uint32_t best_idx = UINT32_MAX; + uint32_t best_clock = UINT32_MAX; + + for (i = 0; i < core->kv.bucket_count; ++i) { + lox_kv_bucket_t *bucket = &core->kv.buckets[i]; + if (bucket->state != LOX_KV_BUCKET_LIVE) { + continue; + } + + if (best_idx == UINT32_MAX || bucket->last_access < best_clock) { + best_idx = i; + best_clock = bucket->last_access; + } + } + + if (best_idx == UINT32_MAX) { + return LOX_ERR_FULL; + } + + LOX_LOG("WARN", + "KV LRU eviction: key=%s last_access=%u", + core->kv.buckets[best_idx].key, + (unsigned)core->kv.buckets[best_idx].last_access); + core->kv.eviction_count++; + lox_kv_remove_slot(core, best_idx); + lox_kv_maybe_compact(core); + return LOX_OK; +} + +lox_err_t lox_kv_init(lox_t *db) { + lox_core_t *core = lox_core(db); +#if LOX_ENABLE_KV + size_t bucket_bytes; + size_t stage_bytes; + uint32_t entry_limit; +#endif + + memset(&core->kv, 0, sizeof(core->kv)); + core->txn_stage = NULL; + core->txn_active = 0u; + core->txn_stage_count = 0u; + +#if LOX_ENABLE_KV + entry_limit = lox_kv_entry_limit(); + if (entry_limit == 0u) { + return LOX_ERR_NO_MEM; + } + core->kv.bucket_count = lox_kv_bucket_count(); + bucket_bytes = (size_t)core->kv.bucket_count * sizeof(lox_kv_bucket_t); + stage_bytes = (size_t)LOX_TXN_STAGE_KEYS * sizeof(lox_txn_stage_entry_t); + + core->kv.buckets = (lox_kv_bucket_t *)lox_arena_alloc(&core->kv_arena, bucket_bytes, 8u); + core->txn_stage = (lox_txn_stage_entry_t *)lox_arena_alloc(&core->kv_arena, stage_bytes, 8u); + if (core->kv.buckets == NULL || core->txn_stage == NULL) { + return LOX_ERR_NO_MEM; + } + + memset(core->kv.buckets, 0, bucket_bytes); + memset(core->txn_stage, 0, stage_bytes); + core->kv.value_store = core->kv_arena.base + core->kv_arena.used; + core->kv.value_capacity = (uint32_t)lox_arena_remaining(&core->kv_arena); + if (core->kv.value_capacity == 0u) { + return LOX_ERR_NO_MEM; + } + core->kv.access_clock = 1u; + core->kv.live_value_bytes = 0u; +#endif + + return LOX_OK; +} + +size_t lox_kv_live_bytes(const lox_t *db) { + const lox_core_t *core = lox_core_const(db); + return lox_kv_live_value_bytes(core) + ((size_t)core->kv.bucket_count * sizeof(lox_kv_bucket_t)); +} + +#if LOX_ENABLE_KV +static lox_err_t lox_kv_set_at_internal(lox_t *db, + const char *key, + const void *val, + size_t len, + uint32_t expires_at, + bool persist) { + lox_core_t *core; + lox_kv_bucket_t *bucket; + uint32_t slot; + uint32_t probe_collisions = 0u; + bool found; + lox_err_t err; + + if (db == NULL || val == NULL || !lox_kv_key_valid(key)) { + return LOX_ERR_INVALID; + } + + if (len > LOX_KV_VAL_MAX_LEN) { + return LOX_ERR_OVERFLOW; + } + + core = lox_core(db); + if (core->magic != LOX_MAGIC) { + return LOX_ERR_INVALID; + } + + err = lox_kv_find_slot(core, key, &slot, &found, &probe_collisions); + if (err != LOX_OK && err != LOX_ERR_FULL) { + return err; + } + + if (!found && core->kv.entry_count >= lox_kv_entry_limit()) { +#if LOX_KV_OVERFLOW_POLICY == LOX_KV_POLICY_REJECT + return LOX_ERR_FULL; +#else + err = lox_kv_evict_lru(core); + if (err != LOX_OK) { + return err; + } + err = lox_kv_find_slot(core, key, &slot, &found, &probe_collisions); + if (err != LOX_OK) { + return err; + } +#endif + } + + bucket = &core->kv.buckets[slot]; + if (!found) { + core->kv.collision_count += probe_collisions; + core->kv.entry_count++; + } + + if (found) { + err = lox_kv_overwrite_value(core, bucket, val, len); + } else { + err = lox_kv_append_value(core, bucket, val, len); + } + + if (err != LOX_OK) { + if (!found && core->kv.entry_count != 0u) { + core->kv.entry_count--; + } + return err; + } + + bucket->state = LOX_KV_BUCKET_LIVE; + bucket->key_hash = lox_kv_hash(key); + memcpy(bucket->key, key, strlen(key) + 1u); + bucket->expires_at = expires_at; + bucket->last_access = lox_kv_next_access_clock(core); + core->live_bytes = lox_kv_live_bytes(db); + if (persist) { + err = lox_persist_kv_set(db, key, val, len, expires_at); + if (err != LOX_OK) { + return err; + } + } + return LOX_OK; +} + +lox_err_t lox_kv_set_at(lox_t *db, const char *key, const void *val, size_t len, uint32_t expires_at) { + return lox_kv_set_at_internal(db, key, val, len, expires_at, true); +} + +lox_err_t lox_kv_set(lox_t *db, const char *key, const void *val, size_t len, uint32_t ttl) { + lox_core_t *core; + uint32_t expires_at = 0u; + lox_err_t rc = LOX_OK; + + if (db == NULL) { + return LOX_ERR_INVALID; + } + + LOX_LOCK(db); + core = lox_core(db); + if (core->magic != LOX_MAGIC) { + rc = LOX_ERR_INVALID; + goto unlock; + } + if (val == NULL || !lox_kv_key_valid(key)) { + rc = LOX_ERR_INVALID; + goto unlock; + } + if (len > LOX_KV_VAL_MAX_LEN) { + rc = LOX_ERR_OVERFLOW; + goto unlock; + } + +#if LOX_KV_ENABLE_TTL + if (ttl != 0u) { + expires_at = (uint32_t)(lox_now(core) + ttl); + } +#else + (void)ttl; +#endif + + if (core->txn_active == 1u) { + lox_txn_stage_entry_t *entry; + if (core->txn_stage_count >= LOX_TXN_STAGE_KEYS) { + rc = LOX_ERR_FULL; + goto unlock; + } + entry = &core->txn_stage[core->txn_stage_count]; + memset(entry, 0, sizeof(*entry)); + memcpy(entry->key, key, strlen(key) + 1u); + entry->val_len = len; + entry->expires_at = expires_at; + entry->op = LOX_TXN_OP_PUT; + if (len != 0u) { + memcpy(entry->val_buf, val, len); + } + entry->val_ptr = entry->val_buf; + core->txn_stage_count++; + rc = LOX_OK; + } else { + bool wal_first = core->wal_enabled && core->storage != NULL && !core->storage_loading && !core->wal_replaying; + if (wal_first) { + rc = lox_persist_kv_set(db, key, val, len, expires_at); + if (rc == LOX_OK) { + rc = lox_kv_set_at_internal(db, key, val, len, expires_at, false); + } + } else { + rc = lox_kv_set_at_internal(db, key, val, len, expires_at, true); + } + if (rc == LOX_OK) { + lox__maybe_compact(db); + } + } + +unlock: + LOX_UNLOCK(db); + return rc; +} + +lox_err_t lox_kv_get(lox_t *db, const char *key, void *buf, size_t buf_len, size_t *out_len) { + lox_core_t *core; + lox_kv_bucket_t *bucket; + uint32_t slot; + bool found; + lox_err_t err; + lox_err_t rc = LOX_OK; + + if (db == NULL || buf == NULL || !lox_kv_key_valid(key)) { + return LOX_ERR_INVALID; + } + + LOX_LOCK(db); + core = lox_core(db); + if (core->magic != LOX_MAGIC) { + rc = LOX_ERR_INVALID; + goto unlock; + } + if (core->txn_active == 1u) { + int32_t i; + for (i = (int32_t)core->txn_stage_count - 1; i >= 0; --i) { + lox_txn_stage_entry_t *entry = &core->txn_stage[i]; + if (strncmp(entry->key, key, LOX_KV_KEY_MAX_LEN) != 0) { + continue; + } + if (entry->op == LOX_TXN_OP_DEL) { + rc = LOX_ERR_NOT_FOUND; + goto unlock; + } + if (out_len != NULL) { + *out_len = entry->val_len; + } + if (buf_len < entry->val_len) { + rc = LOX_ERR_OVERFLOW; + goto unlock; + } + if (entry->val_len != 0u) { + memcpy(buf, entry->val_ptr, entry->val_len); + } + rc = LOX_OK; + goto unlock; + } + } + + err = lox_kv_find_slot(core, key, &slot, &found, NULL); + if (err != LOX_OK || !found) { + rc = LOX_ERR_NOT_FOUND; + goto unlock; + } + + bucket = &core->kv.buckets[slot]; + if (lox_kv_expired(core, bucket)) { + rc = LOX_ERR_EXPIRED; + goto unlock; + } + + if (out_len != NULL) { + *out_len = bucket->val_len; + } + + if (buf_len < bucket->val_len) { + rc = LOX_ERR_OVERFLOW; + goto unlock; + } + + if (bucket->val_len != 0u) { + memcpy(buf, &core->kv.value_store[bucket->val_offset], bucket->val_len); + } + + bucket->last_access = lox_kv_next_access_clock(core); + rc = LOX_OK; + +unlock: + LOX_UNLOCK(db); + return rc; +} + +static lox_err_t lox_kv_del_internal(lox_t *db, const char *key, bool persist) { + lox_core_t *core; + uint32_t slot; + bool found; + lox_err_t err; + + core = lox_core(db); + err = lox_kv_find_slot(core, key, &slot, &found, NULL); + if (err != LOX_OK || !found) { + return LOX_ERR_NOT_FOUND; + } + + lox_kv_remove_slot(core, slot); + lox_kv_maybe_compact(core); + core->live_bytes = lox_kv_live_bytes(db); + if (persist) { + return lox_persist_kv_del(db, key); + } + return LOX_OK; +} + +lox_err_t lox_kv_del(lox_t *db, const char *key) { + lox_core_t *core; + lox_err_t rc = LOX_OK; + + if (db == NULL || !lox_kv_key_valid(key)) { + return LOX_ERR_INVALID; + } + + LOX_LOCK(db); + core = lox_core(db); + if (core->magic != LOX_MAGIC) { + rc = LOX_ERR_INVALID; + goto unlock; + } + + if (core->txn_active == 1u) { + if (core->txn_stage_count >= LOX_TXN_STAGE_KEYS) { + rc = LOX_ERR_FULL; + goto unlock; + } + memset(&core->txn_stage[core->txn_stage_count], 0, sizeof(core->txn_stage[core->txn_stage_count])); + memcpy(core->txn_stage[core->txn_stage_count].key, key, strlen(key) + 1u); + core->txn_stage[core->txn_stage_count].op = LOX_TXN_OP_DEL; + core->txn_stage_count++; + rc = LOX_OK; + goto unlock; + } + + if (core->wal_enabled && core->storage != NULL && !core->storage_loading && !core->wal_replaying) { + rc = lox_persist_kv_del(db, key); + if (rc == LOX_OK) { + rc = lox_kv_del_internal(db, key, false); + } + } else { + rc = lox_kv_del_internal(db, key, true); + } + +unlock: + LOX_UNLOCK(db); + return rc; +} + +lox_err_t lox_kv_exists(lox_t *db, const char *key) { + lox_core_t *core; + lox_kv_bucket_t *bucket; + uint32_t slot; + bool found; + lox_err_t err; + + if (db == NULL || !lox_kv_key_valid(key)) { + return LOX_ERR_INVALID; + } + + LOX_LOCK(db); + core = lox_core(db); + if (core->magic != LOX_MAGIC) { + err = LOX_ERR_INVALID; + goto unlock; + } + + err = lox_kv_find_slot(core, key, &slot, &found, NULL); + if (err != LOX_OK || !found) { + err = LOX_ERR_NOT_FOUND; + goto unlock; + } + + bucket = &core->kv.buckets[slot]; + if (lox_kv_expired(core, bucket)) { + err = LOX_ERR_EXPIRED; + goto unlock; + } + + bucket->last_access = lox_kv_next_access_clock(core); + err = LOX_OK; + +unlock: + LOX_UNLOCK(db); + return err; +} + +lox_err_t lox_kv_iter(lox_t *db, lox_kv_iter_cb_t cb, void *ctx) { + uint32_t i = 0u; + bool done = false; + bool keep_running = true; + + if (db == NULL || cb == NULL) { + return LOX_ERR_INVALID; + } + + while (!done && keep_running) { + char key_copy[LOX_KV_KEY_MAX_LEN]; + uint8_t val_copy[LOX_KV_VAL_MAX_LEN]; + size_t val_len_copy = 0u; + uint32_t ttl_remaining = UINT32_MAX; + bool have_item = false; + lox_core_t *core; + + LOX_LOCK(db); + core = lox_core(db); + if (core->magic != LOX_MAGIC) { + LOX_UNLOCK(db); + return LOX_ERR_INVALID; + } + + while (i < core->kv.bucket_count) { + lox_kv_bucket_t *bucket = &core->kv.buckets[i++]; + if (bucket->state != LOX_KV_BUCKET_LIVE || lox_kv_expired(core, bucket)) { + continue; + } + +#if LOX_KV_ENABLE_TTL + if (bucket->expires_at != 0u) { + lox_timestamp_t now = lox_now(core); + ttl_remaining = bucket->expires_at > now ? (uint32_t)(bucket->expires_at - now) : 0u; + } +#endif + memcpy(key_copy, bucket->key, sizeof(key_copy)); + val_len_copy = bucket->val_len; + if (val_len_copy != 0u) { + memcpy(val_copy, &core->kv.value_store[bucket->val_offset], val_len_copy); + } + have_item = true; + break; + } + + done = (i >= core->kv.bucket_count) && !have_item; + /* Callback/lock invariant: user callback is invoked without DB lock held. */ + LOX_UNLOCK(db); + + if (have_item) { + keep_running = cb(key_copy, val_copy, val_len_copy, ttl_remaining, ctx); + } + } + + return LOX_OK; +} + +lox_err_t lox_kv_purge_expired(lox_t *db) { + lox_core_t *core; + uint32_t i; + bool wal_mode; + lox_err_t rc = LOX_OK; + + if (db == NULL) { + return LOX_ERR_INVALID; + } + + LOX_LOCK(db); + core = lox_core(db); + if (core->magic != LOX_MAGIC) { + rc = LOX_ERR_INVALID; + goto unlock; + } + + wal_mode = core->wal_enabled && core->storage != NULL && !core->storage_loading && !core->wal_replaying; + for (i = 0; i < core->kv.bucket_count; ++i) { + lox_kv_bucket_t *bucket = &core->kv.buckets[i]; + if (bucket->state == LOX_KV_BUCKET_LIVE && lox_kv_expired(core, bucket)) { + if (wal_mode) { + rc = lox_persist_kv_del(db, bucket->key); + if (rc != LOX_OK) { + goto unlock; + } + } + lox_kv_remove_slot(core, i); + } + } + + lox_kv_maybe_compact(core); + core->live_bytes = lox_kv_live_bytes(db); + if (!wal_mode) { + rc = lox_storage_flush(db); + } else { + rc = LOX_OK; + } + +unlock: + LOX_UNLOCK(db); + return rc; +} + +lox_err_t lox_kv_clear(lox_t *db) { + lox_core_t *core; + size_t bucket_bytes; + lox_err_t rc = LOX_OK; + + if (db == NULL) { + return LOX_ERR_INVALID; + } + + LOX_LOCK(db); + core = lox_core(db); + if (core->magic != LOX_MAGIC) { + rc = LOX_ERR_INVALID; + goto unlock; + } + + if (core->wal_enabled && core->storage != NULL && !core->storage_loading && !core->wal_replaying) { + rc = lox_persist_kv_clear(db); + if (rc != LOX_OK) { + goto unlock; + } + } + + bucket_bytes = (size_t)core->kv.bucket_count * sizeof(lox_kv_bucket_t); + memset(core->kv.buckets, 0, bucket_bytes); + core->kv.entry_count = 0u; + core->kv.value_used = 0u; + core->kv.live_value_bytes = 0u; + core->kv.access_clock = 1u; + core->live_bytes = lox_kv_live_bytes(db); + if (!(core->wal_enabled && core->storage != NULL && !core->storage_loading && !core->wal_replaying)) { + rc = lox_persist_kv_clear(db); + } + +unlock: + LOX_UNLOCK(db); + return rc; +} + +lox_err_t lox_txn_begin(lox_t *db) { + lox_core_t *core; + lox_err_t rc = LOX_OK; + + if (db == NULL) { + return LOX_ERR_INVALID; + } + + LOX_LOCK(db); + core = lox_core(db); + if (core->magic != LOX_MAGIC) { + rc = LOX_ERR_INVALID; + goto unlock; + } + if (core->txn_active == 1u) { + rc = LOX_ERR_TXN_ACTIVE; + goto unlock; + } + core->txn_active = 1u; + core->txn_stage_count = 0u; + +unlock: + LOX_UNLOCK(db); + return rc; +} + +lox_err_t lox_txn_commit(lox_t *db) { + lox_core_t *core; + uint32_t i; + lox_err_t rc = LOX_OK; + + if (db == NULL) { + return LOX_ERR_INVALID; + } + + LOX_LOCK(db); + core = lox_core(db); + if (core->magic != LOX_MAGIC) { + rc = LOX_ERR_INVALID; + goto unlock; + } + if (core->txn_active == 0u) { + rc = LOX_ERR_INVALID; + goto unlock; + } + + /* TXN visibility invariant: + * - stage entries are durable in WAL before commit marker. + * - staged entries become visible in live KV only after durable TXN_COMMIT marker. + */ + if (core->wal_enabled && core->storage != NULL && !core->storage_loading && !core->wal_replaying) { + uint32_t needed = 16u; /* TXN_COMMIT marker */ + for (i = 0u; i < core->txn_stage_count; ++i) { + lox_txn_stage_entry_t *entry = &core->txn_stage[i]; + uint32_t key_len = (uint32_t)strlen(entry->key); + uint32_t payload_len = (entry->op == LOX_TXN_OP_PUT) ? (1u + key_len + 4u + (uint32_t)entry->val_len + 4u) + : (1u + key_len); + needed += 16u + lox_align4_u32(payload_len); + } + if (core->wal_used + needed > core->layout.wal_size) { + rc = lox_storage_flush(db); + if (rc != LOX_OK) { + goto unlock; + } + } + } + + for (i = 0u; i < core->txn_stage_count; ++i) { + lox_txn_stage_entry_t *entry = &core->txn_stage[i]; + if (entry->op == LOX_TXN_OP_PUT) { + rc = lox_persist_kv_set_txn(db, entry->key, entry->val_ptr, entry->val_len, entry->expires_at); + if (rc != LOX_OK) { + goto unlock; + } + } else { + rc = lox_persist_kv_del_txn(db, entry->key); + if (rc != LOX_OK) { + goto unlock; + } + } + } + + rc = lox_persist_txn_commit(db); + if (rc != LOX_OK) { + goto unlock; + } + + for (i = 0u; i < core->txn_stage_count; ++i) { + lox_txn_stage_entry_t *entry = &core->txn_stage[i]; + if (entry->op == LOX_TXN_OP_PUT) { + rc = lox_kv_set_at_internal(db, entry->key, entry->val_ptr, entry->val_len, entry->expires_at, false); + if (rc != LOX_OK) { + goto unlock; + } + } else { + rc = lox_kv_del_internal(db, entry->key, false); + if (rc != LOX_OK && rc != LOX_ERR_NOT_FOUND) { + goto unlock; + } + } + } + + core->txn_active = 0u; + core->txn_stage_count = 0u; + rc = LOX_OK; + +unlock: + LOX_UNLOCK(db); + return rc; +} + +lox_err_t lox_txn_rollback(lox_t *db) { + lox_core_t *core; + lox_err_t rc = LOX_OK; + + if (db == NULL) { + return LOX_ERR_INVALID; + } + + LOX_LOCK(db); + core = lox_core(db); + if (core->magic != LOX_MAGIC) { + rc = LOX_ERR_INVALID; + goto unlock; + } + core->txn_active = 0u; + core->txn_stage_count = 0u; + +unlock: + LOX_UNLOCK(db); + return rc; +} +#else +lox_err_t lox_kv_set_at(lox_t *db, const char *key, const void *val, size_t len, uint32_t expires_at) { + (void)db; + (void)key; + (void)val; + (void)len; + (void)expires_at; + return LOX_ERR_DISABLED; +} + +lox_err_t lox_kv_set(lox_t *db, const char *key, const void *val, size_t len, uint32_t ttl) { + (void)db; + (void)key; + (void)val; + (void)len; + (void)ttl; + return LOX_ERR_DISABLED; +} + +lox_err_t lox_kv_get(lox_t *db, const char *key, void *buf, size_t buf_len, size_t *out_len) { + (void)db; + (void)key; + (void)buf; + (void)buf_len; + (void)out_len; + return LOX_ERR_DISABLED; +} + +lox_err_t lox_kv_del(lox_t *db, const char *key) { + (void)db; + (void)key; + return LOX_ERR_DISABLED; +} + +lox_err_t lox_kv_exists(lox_t *db, const char *key) { + (void)db; + (void)key; + return LOX_ERR_DISABLED; +} + +lox_err_t lox_kv_iter(lox_t *db, lox_kv_iter_cb_t cb, void *ctx) { + (void)db; + (void)cb; + (void)ctx; + return LOX_ERR_DISABLED; +} + +lox_err_t lox_kv_purge_expired(lox_t *db) { + (void)db; + return LOX_ERR_DISABLED; +} + +lox_err_t lox_kv_clear(lox_t *db) { + (void)db; + return LOX_ERR_DISABLED; +} + +lox_err_t lox_txn_begin(lox_t *db) { + (void)db; + return LOX_ERR_DISABLED; +} + +lox_err_t lox_txn_commit(lox_t *db) { + (void)db; + return LOX_ERR_DISABLED; +} + +lox_err_t lox_txn_rollback(lox_t *db) { + (void)db; + return LOX_ERR_DISABLED; +} +#endif diff --git a/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/src/lox_lock.h b/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/src/lox_lock.h new file mode 100644 index 0000000..65f926f --- /dev/null +++ b/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/src/lox_lock.h @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: MIT +#ifndef LOX_LOCK_H +#define LOX_LOCK_H + +#include "lox_internal.h" + +#if LOX_THREAD_SAFE +#define LOX_LOCK(db) \ + do { \ + lox_core_t *lox_lock_core__ = lox_core((db)); \ + if (lox_lock_core__->lock != NULL) { \ + lox_lock_core__->lock(lox_lock_core__->lock_handle); \ + } \ + } while (0) +#define LOX_UNLOCK(db) \ + do { \ + lox_core_t *lox_lock_core__ = lox_core((db)); \ + if (lox_lock_core__->unlock != NULL) { \ + lox_lock_core__->unlock(lox_lock_core__->lock_handle); \ + } \ + } while (0) +#else +#define LOX_LOCK(db) (void)(db) +#define LOX_UNLOCK(db) (void)(db) +#endif + +#endif diff --git a/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/src/lox_rel.c b/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/src/lox_rel.c new file mode 100644 index 0000000..44de7c6 --- /dev/null +++ b/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/src/lox_rel.c @@ -0,0 +1,1152 @@ +// SPDX-License-Identifier: MIT +#include "lox_internal.h" +#include "lox_lock.h" +#include "lox_arena.h" + +#include +#if defined(_MSC_VER) +#include +#endif + +#define LOX_REL_ROW_SCRATCH_MAX 1024u + +static size_t lox_rel_type_size(lox_col_type_t type) { + if (type == LOX_COL_U8 || type == LOX_COL_I8 || type == LOX_COL_BOOL) { + return 1u; + } + if (type == LOX_COL_U16 || type == LOX_COL_I16) { + return 2u; + } + if (type == LOX_COL_U32 || type == LOX_COL_I32 || type == LOX_COL_F32) { + return 4u; + } + if (type == LOX_COL_U64 || type == LOX_COL_I64 || type == LOX_COL_F64) { + return 8u; + } + return 0u; +} + +static lox_err_t lox_rel_validate_name(const char *name, size_t max_len) { + size_t len; + + if (name == NULL || name[0] == '\0') { + return LOX_ERR_INVALID; + } + + len = strlen(name); + if (len >= max_len) { + return LOX_ERR_INVALID; + } + + return LOX_OK; +} + +static lox_col_desc_t *lox_rel_find_col(lox_col_desc_t *cols, uint32_t col_count, const char *name) { + uint32_t i; + + for (i = 0; i < col_count; ++i) { + if (strcmp(cols[i].name, name) == 0) { + return &cols[i]; + } + } + + return NULL; +} + +static const lox_col_desc_t *lox_rel_find_col_const(const lox_col_desc_t *cols, uint32_t col_count, const char *name) { + uint32_t i; + + for (i = 0; i < col_count; ++i) { + if (strcmp(cols[i].name, name) == 0) { + return &cols[i]; + } + } + + return NULL; +} + +static size_t lox_rel_align_for_size(size_t size) { + if (size >= 8u) { + return 8u; + } + if (size >= 4u) { + return 4u; + } + if (size >= 2u) { + return 2u; + } + return 1u; +} + +static bool rel_is_alive(const uint8_t *bitmap, uint32_t row_idx) { + return ((bitmap[row_idx >> 3u] >> (row_idx & 7u)) & 1u) != 0u; +} + +static void rel_set_alive(uint8_t *bitmap, uint32_t row_idx, bool alive) { + if (alive) { + bitmap[row_idx >> 3u] |= (uint8_t)(1u << (row_idx & 7u)); + } else { + bitmap[row_idx >> 3u] &= (uint8_t)~(1u << (row_idx & 7u)); + } +} + +static const void *rel_row_ptr(const lox_table_t *table, uint32_t row_idx) { + return table->rows + ((size_t)row_idx * table->row_size); +} + +static void *rel_row_ptr_mut(lox_table_t *table, uint32_t row_idx) { + return table->rows + ((size_t)row_idx * table->row_size); +} + +static int rel_key_cmp(const void *a, const void *b, size_t size) { + return memcmp(a, b, size); +} + +static uint32_t rel_ctz_u32(uint32_t value) { +#if defined(_MSC_VER) + unsigned long idx; + _BitScanForward(&idx, value); + return (uint32_t)idx; +#elif defined(__GNUC__) || defined(__clang__) + return (uint32_t)__builtin_ctz(value); +#else + uint32_t idx = 0u; + while ((value & 1u) == 0u) { + value >>= 1u; + idx++; + } + return idx; +#endif +} + +static const lox_col_desc_t *rel_index_col(const lox_table_t *table); +static void rel_copy_column_to_index(uint8_t *dst, const lox_col_desc_t *col, const void *row_buf); + +static uint32_t rel_index_find_first(const lox_table_t *table, const void *key_bytes) { + int32_t lo = 0; + int32_t hi = (int32_t)table->index_count - 1; + int32_t result = -1; + + while (lo <= hi) { + int32_t mid = lo + (hi - lo) / 2; + int cmp = rel_key_cmp(table->index[mid].key_bytes, key_bytes, table->index_key_size); + if (cmp == 0) { + result = mid; + hi = mid - 1; + } else if (cmp < 0) { + lo = mid + 1; + } else { + hi = mid - 1; + } + } + + return (result >= 0) ? (uint32_t)result : UINT32_MAX; +} + +static void rel_index_insert(lox_table_t *table, uint32_t row_idx, const void *key_bytes) { + int32_t lo = 0; + int32_t hi = (int32_t)table->index_count - 1; + int32_t pos = (int32_t)table->index_count; + + while (lo <= hi) { + int32_t mid = lo + (hi - lo) / 2; + int cmp = rel_key_cmp(table->index[mid].key_bytes, key_bytes, table->index_key_size); + if (cmp < 0) { + lo = mid + 1; + } else { + pos = mid; + hi = mid - 1; + } + } + + memmove(&table->index[pos + 1], + &table->index[pos], + (table->index_count - (uint32_t)pos) * sizeof(lox_index_entry_t)); + memcpy(table->index[pos].key_bytes, key_bytes, table->index_key_size); + table->index[pos].row_idx = row_idx; + table->index_count++; +} + +static void rel_index_remove_row(lox_table_t *table, uint32_t row_idx) { + uint32_t i; + + for (i = 0; i < table->index_count; ++i) { + if (table->index[i].row_idx == row_idx) { + memmove(&table->index[i], + &table->index[i + 1u], + (table->index_count - i - 1u) * sizeof(lox_index_entry_t)); + table->index_count--; + return; + } + } +} + +static void rel_order_remove_row(lox_table_t *table, uint32_t row_idx) { + uint32_t i; + + for (i = 0; i < table->order_count; ++i) { + if (table->order[i] == row_idx) { + memmove(&table->order[i], + &table->order[i + 1u], + (table->order_count - i - 1u) * sizeof(uint32_t)); + table->order_count--; + return; + } + } +} + +static void rel_apply_insert_row(lox_table_t *table, uint32_t row_idx, const void *row_buf) { + const lox_col_desc_t *idx_col; + + memcpy(rel_row_ptr_mut(table, row_idx), row_buf, table->row_size); + rel_set_alive(table->alive_bitmap, row_idx, true); + table->order[table->order_count++] = row_idx; + table->live_count++; + + idx_col = rel_index_col(table); + if (idx_col != NULL) { + uint8_t key_bytes[LOX_REL_INDEX_KEY_MAX]; + rel_copy_column_to_index(key_bytes, idx_col, row_buf); + rel_index_insert(table, row_idx, key_bytes); + } + table->mutation_seq++; +} + +static void rel_apply_delete_row(lox_table_t *table, uint32_t row_idx) { + rel_set_alive(table->alive_bitmap, row_idx, false); + rel_index_remove_row(table, row_idx); + rel_order_remove_row(table, row_idx); + if (table->live_count != 0u) { + table->live_count--; + } + table->mutation_seq++; +} + +static bool rel_wal_mode(const lox_core_t *core) { + return core->wal_enabled && core->storage != NULL && !core->storage_loading && !core->wal_replaying; +} + +static bool rel_has_arena_space_for_table(const lox_core_t *core, const lox_schema_impl_t *impl) { + size_t need_rows = (size_t)impl->max_rows * impl->row_size; + size_t need_alive = (size_t)(impl->max_rows + 7u) / 8u; + size_t need_order = (size_t)impl->max_rows * sizeof(uint32_t); + size_t need_index = (impl->index_col != UINT32_MAX) ? ((size_t)impl->max_rows * sizeof(lox_index_entry_t)) : 0u; + size_t need_total = need_rows + need_alive + need_order + need_index + 32u; + return lox_arena_remaining((lox_arena_t *)&core->rel_arena) >= need_total; +} + +static uint32_t rel_find_free_row(const lox_table_t *table) { + uint32_t byte_idx; + uint32_t alive_bytes = (table->max_rows + 7u) / 8u; + + for (byte_idx = 0u; byte_idx < alive_bytes; ++byte_idx) { + uint32_t row_base = byte_idx * 8u; + uint8_t effective = table->alive_bitmap[byte_idx]; + if (row_base + 8u > table->max_rows) { + uint32_t valid_bits = table->max_rows - row_base; + uint8_t valid_mask = (uint8_t)((1u << valid_bits) - 1u); + effective |= (uint8_t)(~valid_mask); + } + if (effective == 0xFFu) { + continue; + } + return row_base + rel_ctz_u32((uint32_t)(uint8_t)(~effective)); + } + + return UINT32_MAX; +} + +static lox_table_t *rel_find_table(lox_core_t *core, const char *name) { + uint32_t i; + + for (i = 0; i < LOX_REL_MAX_TABLES; ++i) { + lox_table_t *table = &core->rel.tables[i]; + if (table->registered && strcmp(table->name, name) == 0) { + return table; + } + } + + return NULL; +} + +static const lox_col_desc_t *rel_index_col(const lox_table_t *table) { + if (table->index_col == UINT32_MAX) { + return NULL; + } + return &table->cols[table->index_col]; +} + +static const void *rel_index_key_ptr(const lox_table_t *table, const void *row_buf) { + const lox_col_desc_t *col = rel_index_col(table); + if (col == NULL) { + return NULL; + } + return (const uint8_t *)row_buf + col->offset; +} + +static void rel_copy_column_to_index(uint8_t *dst, const lox_col_desc_t *col, const void *row_buf) { + memset(dst, 0, LOX_REL_INDEX_KEY_MAX); + memcpy(dst, (const uint8_t *)row_buf + col->offset, col->size); +} + +static lox_err_t rel_validate_str_value(const char *str, size_t max_size) { + size_t i; + + for (i = 0; i < max_size; ++i) { + if (str[i] == '\0') { + return LOX_OK; + } + } + + return LOX_ERR_SCHEMA; +} + +static lox_err_t rel_validate_table_and_handle(lox_t *db, lox_table_t *table) { + if (db == NULL || table == NULL) { + return LOX_ERR_INVALID; + } + if (lox_core(db)->magic != LOX_MAGIC) { + return LOX_ERR_INVALID; + } + if (!table->registered) { + return LOX_ERR_INVALID; + } + return LOX_OK; +} + +#if LOX_ENABLE_REL +lox_err_t lox_schema_init(lox_schema_t *schema, const char *name, uint32_t max_rows) { + lox_schema_impl_t *impl; + lox_err_t err; + + if (schema == NULL || max_rows == 0u) { + return LOX_ERR_INVALID; + } + + err = lox_rel_validate_name(name, LOX_REL_TABLE_NAME_LEN); + if (err != LOX_OK) { + return err; + } + + memset(schema, 0, sizeof(*schema)); + schema->schema_version = 0u; + impl = (lox_schema_impl_t *)&schema->_opaque[0]; + memcpy(impl->name, name, strlen(name) + 1u); + impl->schema_version = schema->schema_version; + impl->max_rows = max_rows; + impl->index_col = UINT32_MAX; + return LOX_OK; +} + +lox_err_t lox_schema_add(lox_schema_t *schema, + const char *col_name, + lox_col_type_t type, + size_t size, + bool is_index) { + lox_schema_impl_t *impl; + lox_col_desc_t *col; + lox_err_t err; + size_t fixed_size; + + if (schema == NULL) { + return LOX_ERR_INVALID; + } + + err = lox_rel_validate_name(col_name, LOX_REL_COL_NAME_LEN); + if (err != LOX_OK) { + return err; + } + + impl = (lox_schema_impl_t *)&schema->_opaque[0]; + if (impl->sealed) { + return LOX_ERR_SEALED; + } + if (impl->col_count >= LOX_REL_MAX_COLS) { + return LOX_ERR_FULL; + } + if (lox_rel_find_col(impl->cols, impl->col_count, col_name) != NULL) { + return LOX_ERR_INVALID; + } + + fixed_size = lox_rel_type_size(type); + if (fixed_size != 0u) { + if (size != fixed_size) { + return LOX_ERR_INVALID; + } + } else if ((type == LOX_COL_STR || type == LOX_COL_BLOB) && size != 0u) { + if (size > LOX_REL_INDEX_KEY_MAX && is_index) { + return LOX_ERR_INVALID; + } + } else { + return LOX_ERR_INVALID; + } + + if (is_index && impl->index_col != UINT32_MAX) { + return LOX_ERR_INVALID; + } + + col = &impl->cols[impl->col_count]; + memset(col, 0, sizeof(*col)); + memcpy(col->name, col_name, strlen(col_name) + 1u); + col->type = type; + col->size = size; + col->is_index = is_index; + if (is_index) { + impl->index_col = impl->col_count; + } + impl->col_count++; + return LOX_OK; +} + +lox_err_t lox_schema_seal(lox_schema_t *schema) { + lox_schema_impl_t *impl; + size_t offset = 0u; + uint32_t i; + + if (schema == NULL) { + return LOX_ERR_INVALID; + } + + impl = (lox_schema_impl_t *)&schema->_opaque[0]; + if (impl->col_count == 0u) { + return LOX_ERR_INVALID; + } + if (impl->sealed) { + return LOX_OK; + } + + for (i = 0; i < impl->col_count; ++i) { + lox_col_desc_t *col = &impl->cols[i]; + size_t align = lox_rel_align_for_size(col->size); + offset = (offset + (align - 1u)) & ~(align - 1u); + col->offset = offset; + offset += col->size; + } + + impl->row_size = (offset + 3u) & ~3u; + /* schema_version is captured at seal-time and treated as immutable afterwards. + * Any post-seal mutation of schema->schema_version is rejected in table_create. + */ + impl->schema_version = schema->schema_version; + impl->sealed = true; + return LOX_OK; +} + +lox_err_t lox_table_create(lox_t *db, lox_schema_t *schema) { + lox_core_t *core; + lox_schema_impl_t *impl; + lox_table_t *table; + lox_table_t *existing; + uint32_t alive_bytes; + uint32_t i; + lox_err_t rc = LOX_OK; + bool need_migrate_cb = false; + uint16_t migrate_old = 0u; + uint16_t migrate_new = 0u; + char migrate_name[LOX_REL_TABLE_NAME_LEN]; + bool wal_mode; + + if (db == NULL || schema == NULL) { + return LOX_ERR_INVALID; + } + + LOX_LOCK(db); + core = lox_core(db); + if (core->magic != LOX_MAGIC) { + rc = LOX_ERR_INVALID; + goto unlock; + } + + impl = (lox_schema_impl_t *)&schema->_opaque[0]; + if (!impl->sealed) { + rc = LOX_ERR_INVALID; + goto unlock; + } + /* Defensive contract check: callers must set schema_version before seal. */ + if (schema->schema_version != impl->schema_version) { + rc = LOX_ERR_SCHEMA; + goto unlock; + } + wal_mode = rel_wal_mode(core); + existing = rel_find_table(core, impl->name); + if (existing != NULL) { + if (existing->schema_version == impl->schema_version) { + rc = LOX_OK; + goto unlock; + } + if (core->on_migrate == NULL) { + rc = LOX_ERR_SCHEMA; + goto unlock; + } + if (core->migration_in_progress) { + rc = LOX_ERR_SCHEMA; + goto unlock; + } + core->migration_in_progress = true; + need_migrate_cb = true; + migrate_old = existing->schema_version; + migrate_new = impl->schema_version; + memset(migrate_name, 0, sizeof(migrate_name)); + memcpy(migrate_name, impl->name, strlen(impl->name) + 1u); + goto unlock; + } + if (core->rel.registered_tables >= LOX_REL_MAX_TABLES) { + rc = LOX_ERR_FULL; + goto unlock; + } + if (!rel_has_arena_space_for_table(core, impl)) { + rc = LOX_ERR_NO_MEM; + goto unlock; + } + if (wal_mode) { + rc = lox_persist_rel_table_create(db, schema); + if (rc != LOX_OK) { + goto unlock; + } + } + + for (i = 0; i < LOX_REL_MAX_TABLES; ++i) { + table = &core->rel.tables[i]; + if (!table->registered) { + memset(table, 0, sizeof(*table)); + memcpy(table->name, impl->name, sizeof(table->name)); + memcpy(table->cols, impl->cols, sizeof(impl->cols)); + table->col_count = impl->col_count; + table->max_rows = impl->max_rows; + table->row_size = impl->row_size; + table->index_col = impl->index_col; + table->schema_version = impl->schema_version; + if (impl->index_col != UINT32_MAX) { + table->index_key_size = impl->cols[impl->index_col].size; + } + + table->rows = (uint8_t *)lox_arena_alloc(&core->rel_arena, + (size_t)table->max_rows * table->row_size, + 8u); + alive_bytes = (table->max_rows + 7u) / 8u; + table->alive_bitmap = (uint8_t *)lox_arena_alloc(&core->rel_arena, alive_bytes, 1u); + table->order = (uint32_t *)lox_arena_alloc(&core->rel_arena, + (size_t)table->max_rows * sizeof(uint32_t), + 4u); + if (table->index_key_size != 0u) { + table->index = (lox_index_entry_t *)lox_arena_alloc(&core->rel_arena, + (size_t)table->max_rows * sizeof(lox_index_entry_t), + 4u); + } + + if (table->rows == NULL || table->alive_bitmap == NULL || table->order == NULL || + (table->index_key_size != 0u && table->index == NULL)) { + memset(table, 0, sizeof(*table)); + rc = LOX_ERR_NO_MEM; + goto unlock; + } + + memset(table->rows, 0, (size_t)table->max_rows * table->row_size); + memset(table->alive_bitmap, 0, alive_bytes); + if (table->index != NULL) { + memset(table->index, 0, (size_t)table->max_rows * sizeof(lox_index_entry_t)); + } + memset(table->order, 0, (size_t)table->max_rows * sizeof(uint32_t)); + table->registered = true; + core->rel.registered_tables++; + if (!wal_mode) { + rc = lox_storage_flush(db); + } else { + rc = LOX_OK; + } + goto unlock; + } + } + + rc = LOX_ERR_FULL; + +unlock: + LOX_UNLOCK(db); + if (need_migrate_cb) { + rc = core->on_migrate(db, migrate_name, migrate_old, migrate_new); + LOX_LOCK(db); + core = lox_core(db); + if (core->magic != LOX_MAGIC) { + LOX_UNLOCK(db); + return LOX_ERR_INVALID; + } + core->migration_in_progress = false; + if (rc != LOX_OK) { + LOX_UNLOCK(db); + return rc; + } + existing = rel_find_table(core, migrate_name); + if (existing == NULL) { + LOX_UNLOCK(db); + return LOX_ERR_NOT_FOUND; + } + if (rel_wal_mode(core)) { + rc = lox_persist_rel_table_create(db, schema); + if (rc != LOX_OK) { + LOX_UNLOCK(db); + return rc; + } + } + existing->schema_version = migrate_new; + if (!rel_wal_mode(core)) { + rc = lox_storage_flush(db); + } else { + rc = LOX_OK; + } + LOX_UNLOCK(db); + return rc; + } + return rc; +} + +lox_err_t lox_table_get(lox_t *db, const char *name, lox_table_t **out_table) { + lox_core_t *core; + lox_table_t *table; + lox_err_t err; + lox_err_t rc = LOX_OK; + + if (db == NULL || out_table == NULL) { + return LOX_ERR_INVALID; + } + + err = lox_rel_validate_name(name, LOX_REL_TABLE_NAME_LEN); + if (err != LOX_OK) { + return err; + } + + LOX_LOCK(db); + core = lox_core(db); + if (core->magic != LOX_MAGIC) { + rc = LOX_ERR_INVALID; + goto unlock; + } + + table = rel_find_table(core, name); + if (table == NULL) { + rc = LOX_ERR_NOT_FOUND; + goto unlock; + } + + *out_table = table; + rc = LOX_OK; + +unlock: + LOX_UNLOCK(db); + return rc; +} + +size_t lox_table_row_size(const lox_table_t *table) { + if (table == NULL) { + return 0u; + } + return table->row_size; +} + +lox_err_t lox_row_set(const lox_table_t *table, void *row_buf, const char *col_name, const void *val) { + const lox_col_desc_t *col; + + if (table == NULL || row_buf == NULL || col_name == NULL || val == NULL) { + return LOX_ERR_INVALID; + } + + col = lox_rel_find_col_const(table->cols, table->col_count, col_name); + if (col == NULL) { + return LOX_ERR_NOT_FOUND; + } + + if (col->type == LOX_COL_STR) { + if (rel_validate_str_value((const char *)val, col->size) != LOX_OK) { + return LOX_ERR_SCHEMA; + } + memset((uint8_t *)row_buf + col->offset, 0, col->size); + memcpy((uint8_t *)row_buf + col->offset, val, strlen((const char *)val) + 1u); + return LOX_OK; + } + + memcpy((uint8_t *)row_buf + col->offset, val, col->size); + return LOX_OK; +} + +lox_err_t lox_row_get(const lox_table_t *table, + const void *row_buf, + const char *col_name, + void *out, + size_t *out_len) { + const lox_col_desc_t *col; + + if (table == NULL || row_buf == NULL || col_name == NULL || out == NULL) { + return LOX_ERR_INVALID; + } + + col = lox_rel_find_col_const(table->cols, table->col_count, col_name); + if (col == NULL) { + return LOX_ERR_NOT_FOUND; + } + + memcpy(out, (const uint8_t *)row_buf + col->offset, col->size); + if (out_len != NULL) { + *out_len = col->size; + } + return LOX_OK; +} + +lox_err_t lox_rel_insert(lox_t *db, lox_table_t *table, const void *row_buf) { + lox_err_t err; + uint32_t row_idx; + lox_err_t rc = LOX_OK; + + if (db == NULL) { + return LOX_ERR_INVALID; + } + LOX_LOCK(db); + + err = rel_validate_table_and_handle(db, table); + if (err != LOX_OK) { + rc = err; + goto unlock; + } + if (row_buf == NULL) { + rc = LOX_ERR_INVALID; + goto unlock; + } + if (table->live_count >= table->max_rows) { + rc = LOX_ERR_FULL; + goto unlock; + } + + row_idx = rel_find_free_row(table); + if (row_idx == UINT32_MAX) { + rc = LOX_ERR_FULL; + goto unlock; + } + + rc = lox_persist_rel_insert(db, table, row_buf); + if (rc == LOX_OK) { + rel_apply_insert_row(table, row_idx, row_buf); + lox__maybe_compact(db); + } + +unlock: + LOX_UNLOCK(db); + return rc; +} + +lox_err_t lox_rel_find(lox_t *db, + lox_table_t *table, + const void *search_val, + lox_rel_iter_cb_t cb, + void *ctx) { + uint32_t idx; + uint32_t snapshot_mutation_seq; + lox_err_t err; + lox_err_t rc = LOX_OK; + + if (db == NULL) { + return LOX_ERR_INVALID; + } + LOX_LOCK(db); + err = rel_validate_table_and_handle(db, table); + if (err != LOX_OK) { + rc = err; + goto unlock; + } + if (search_val == NULL || cb == NULL) { + rc = LOX_ERR_INVALID; + goto unlock; + } + if (table->index_col == UINT32_MAX) { + rc = LOX_ERR_INVALID; + goto unlock; + } + + idx = rel_index_find_first(table, search_val); + snapshot_mutation_seq = table->mutation_seq; + if (idx == UINT32_MAX) { + rc = LOX_OK; + goto unlock; + } + + while (idx < table->index_count && + rel_key_cmp(table->index[idx].key_bytes, search_val, table->index_key_size) == 0) { + uint8_t row_copy[LOX_REL_ROW_SCRATCH_MAX]; + uint32_t row_idx = table->index[idx].row_idx; + if (table->row_size > sizeof(row_copy)) { + rc = LOX_ERR_OVERFLOW; + goto unlock; + } + memcpy(row_copy, rel_row_ptr(table, row_idx), table->row_size); + idx++; + LOX_UNLOCK(db); + if (!cb(row_copy, ctx)) { + return LOX_OK; + } + LOX_LOCK(db); + err = rel_validate_table_and_handle(db, table); + if (err != LOX_OK) { + rc = err; + goto unlock; + } + if (table->mutation_seq != snapshot_mutation_seq) { + rc = LOX_ERR_INVALID; + goto unlock; + } + } + + rc = LOX_OK; + +unlock: + LOX_UNLOCK(db); + return rc; +} + +lox_err_t lox_rel_find_by(lox_t *db, + lox_table_t *table, + const char *col_name, + const void *search_val, + void *out_buf) { + const lox_col_desc_t *col; + uint32_t i; + lox_err_t err; + + lox_err_t rc = LOX_OK; + + if (db == NULL) { + return LOX_ERR_INVALID; + } + LOX_LOCK(db); + err = rel_validate_table_and_handle(db, table); + if (err != LOX_OK) { + rc = err; + goto unlock; + } + if (col_name == NULL || search_val == NULL || out_buf == NULL) { + rc = LOX_ERR_INVALID; + goto unlock; + } + + col = lox_rel_find_col_const(table->cols, table->col_count, col_name); + if (col == NULL) { + rc = LOX_ERR_NOT_FOUND; + goto unlock; + } + + for (i = 0; i < table->order_count; ++i) { + uint32_t row_idx = table->order[i]; + const uint8_t *row = (const uint8_t *)rel_row_ptr(table, row_idx); + bool match = false; + + if (!rel_is_alive(table->alive_bitmap, row_idx)) { + continue; + } + + if (col->type == LOX_COL_STR) { + match = strncmp((const char *)(row + col->offset), (const char *)search_val, col->size) == 0; + } else { + match = memcmp(row + col->offset, search_val, col->size) == 0; + } + + if (match) { + memcpy(out_buf, row, table->row_size); + rc = LOX_OK; + goto unlock; + } + } + + rc = LOX_ERR_NOT_FOUND; + +unlock: + LOX_UNLOCK(db); + return rc; +} + +lox_err_t lox_rel_delete(lox_t *db, lox_table_t *table, const void *search_val, uint32_t *out_deleted) { + uint32_t deleted = 0u; + uint32_t idx; + uint32_t match_count = 0u; + uint32_t m; + lox_err_t err; + lox_err_t rc = LOX_OK; + + if (db == NULL) { + return LOX_ERR_INVALID; + } + LOX_LOCK(db); + + err = rel_validate_table_and_handle(db, table); + if (err != LOX_OK) { + rc = err; + goto unlock; + } + if (search_val == NULL) { + rc = LOX_ERR_INVALID; + goto unlock; + } + if (table->index_col == UINT32_MAX) { + rc = LOX_ERR_INVALID; + goto unlock; + } + + idx = rel_index_find_first(table, search_val); + while (idx != UINT32_MAX && + idx < table->index_count && + rel_key_cmp(table->index[idx].key_bytes, search_val, table->index_key_size) == 0) { + match_count++; + idx++; + } + + if (match_count == 0u) { + if (out_deleted != NULL) { + *out_deleted = 0u; + } + rc = LOX_OK; + goto unlock; + } + + rc = lox_persist_rel_delete(db, table, search_val); + if (rc != LOX_OK) { + goto unlock; + } + + for (m = 0u; m < match_count; ++m) { + idx = rel_index_find_first(table, search_val); + if (idx == UINT32_MAX) { + break; + } + rel_apply_delete_row(table, table->index[idx].row_idx); + deleted++; + } + if (out_deleted != NULL) { + *out_deleted = deleted; + } + +unlock: + LOX_UNLOCK(db); + return rc; +} + +lox_err_t lox_rel_iter(lox_t *db, lox_table_t *table, lox_rel_iter_cb_t cb, void *ctx) { + uint32_t i; + lox_err_t err; + lox_err_t rc = LOX_OK; + + if (db == NULL) { + return LOX_ERR_INVALID; + } + LOX_LOCK(db); + err = rel_validate_table_and_handle(db, table); + if (err != LOX_OK) { + rc = err; + goto unlock; + } + if (cb == NULL) { + rc = LOX_ERR_INVALID; + goto unlock; + } + + for (i = 0; i < table->order_count; ++i) { + uint32_t row_idx = table->order[i]; + if (rel_is_alive(table->alive_bitmap, row_idx)) { + uint8_t row_copy[LOX_REL_ROW_SCRATCH_MAX]; + if (table->row_size > sizeof(row_copy)) { + rc = LOX_ERR_OVERFLOW; + goto unlock; + } + memcpy(row_copy, rel_row_ptr(table, row_idx), table->row_size); + LOX_UNLOCK(db); + if (!cb(row_copy, ctx)) { + return LOX_OK; + } + LOX_LOCK(db); + err = rel_validate_table_and_handle(db, table); + if (err != LOX_OK) { + rc = err; + goto unlock; + } + } + } + + rc = LOX_OK; + +unlock: + LOX_UNLOCK(db); + return rc; +} + +lox_err_t lox_rel_count(const lox_table_t *table, uint32_t *out_count) { + if (table == NULL || out_count == NULL) { + return LOX_ERR_INVALID; + } + if (!table->registered) { + return LOX_ERR_INVALID; + } + *out_count = table->live_count; + return LOX_OK; +} + +lox_err_t lox_rel_clear(lox_t *db, lox_table_t *table) { + uint32_t alive_bytes; + lox_err_t err; + lox_err_t rc = LOX_OK; + + if (db == NULL) { + return LOX_ERR_INVALID; + } + LOX_LOCK(db); + err = rel_validate_table_and_handle(db, table); + if (err != LOX_OK) { + rc = err; + goto unlock; + } + if (rel_wal_mode(lox_core(db))) { + rc = lox_persist_rel_clear(db, table); + if (rc != LOX_OK) { + goto unlock; + } + } + + alive_bytes = (table->max_rows + 7u) / 8u; + memset(table->alive_bitmap, 0, alive_bytes); + if (table->index != NULL) { + memset(table->index, 0, (size_t)table->max_rows * sizeof(lox_index_entry_t)); + } + memset(table->order, 0, (size_t)table->max_rows * sizeof(uint32_t)); + table->live_count = 0u; + table->index_count = 0u; + table->order_count = 0u; + table->mutation_seq++; + if (!rel_wal_mode(lox_core(db))) { + rc = lox_storage_flush(db); + } else { + rc = LOX_OK; + } + +unlock: + LOX_UNLOCK(db); + return rc; +} +#else +lox_err_t lox_schema_init(lox_schema_t *schema, const char *name, uint32_t max_rows) { + (void)schema; + (void)name; + (void)max_rows; + return LOX_ERR_DISABLED; +} + +lox_err_t lox_schema_add(lox_schema_t *schema, + const char *col_name, + lox_col_type_t type, + size_t size, + bool is_index) { + (void)schema; + (void)col_name; + (void)type; + (void)size; + (void)is_index; + return LOX_ERR_DISABLED; +} + +lox_err_t lox_schema_seal(lox_schema_t *schema) { + (void)schema; + return LOX_ERR_DISABLED; +} + +lox_err_t lox_table_create(lox_t *db, lox_schema_t *schema) { + (void)db; + (void)schema; + return LOX_ERR_DISABLED; +} + +lox_err_t lox_table_get(lox_t *db, const char *name, lox_table_t **out_table) { + (void)db; + (void)name; + (void)out_table; + return LOX_ERR_DISABLED; +} + +size_t lox_table_row_size(const lox_table_t *table) { + (void)table; + return 0u; +} + +lox_err_t lox_row_set(const lox_table_t *table, void *row_buf, const char *col_name, const void *val) { + (void)table; + (void)row_buf; + (void)col_name; + (void)val; + return LOX_ERR_DISABLED; +} + +lox_err_t lox_row_get(const lox_table_t *table, + const void *row_buf, + const char *col_name, + void *out, + size_t *out_len) { + (void)table; + (void)row_buf; + (void)col_name; + (void)out; + (void)out_len; + return LOX_ERR_DISABLED; +} + +lox_err_t lox_rel_insert(lox_t *db, lox_table_t *table, const void *row_buf) { + (void)db; + (void)table; + (void)row_buf; + return LOX_ERR_DISABLED; +} + +lox_err_t lox_rel_find(lox_t *db, + lox_table_t *table, + const void *search_val, + lox_rel_iter_cb_t cb, + void *ctx) { + (void)db; + (void)table; + (void)search_val; + (void)cb; + (void)ctx; + return LOX_ERR_DISABLED; +} + +lox_err_t lox_rel_find_by(lox_t *db, + lox_table_t *table, + const char *col_name, + const void *search_val, + void *out_buf) { + (void)db; + (void)table; + (void)col_name; + (void)search_val; + (void)out_buf; + return LOX_ERR_DISABLED; +} + +lox_err_t lox_rel_delete(lox_t *db, lox_table_t *table, const void *search_val, uint32_t *out_deleted) { + (void)db; + (void)table; + (void)search_val; + (void)out_deleted; + return LOX_ERR_DISABLED; +} + +lox_err_t lox_rel_iter(lox_t *db, lox_table_t *table, lox_rel_iter_cb_t cb, void *ctx) { + (void)db; + (void)table; + (void)cb; + (void)ctx; + return LOX_ERR_DISABLED; +} + +lox_err_t lox_rel_count(const lox_table_t *table, uint32_t *out_count) { + (void)table; + (void)out_count; + return LOX_ERR_DISABLED; +} + +lox_err_t lox_rel_clear(lox_t *db, lox_table_t *table) { + (void)db; + (void)table; + return LOX_ERR_DISABLED; +} +#endif diff --git a/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/src/lox_ts.c b/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/src/lox_ts.c new file mode 100644 index 0000000..991503b --- /dev/null +++ b/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/src/lox_ts.c @@ -0,0 +1,701 @@ +// SPDX-License-Identifier: MIT +#include "lox_internal.h" +#include "lox_lock.h" + +#include + +static lox_err_t lox_ts_validate_name(const char *name) { + size_t len; + + if (name == NULL || name[0] == '\0') { + return LOX_ERR_INVALID; + } + + len = strlen(name); + if (len >= LOX_TS_STREAM_NAME_LEN) { + return LOX_ERR_INVALID; + } + + return LOX_OK; +} + +static lox_ts_stream_t *lox_ts_find(lox_core_t *core, const char *name) { + uint32_t i; + + for (i = 0; i < LOX_TS_MAX_STREAMS; ++i) { + lox_ts_stream_t *stream = &core->ts.streams[i]; + if (stream->registered && strcmp(stream->name, name) == 0) { + return stream; + } + } + + return NULL; +} + +static uint32_t lox_ts_stream_val_size(const lox_ts_stream_t *stream) { + return (stream->type == LOX_TS_RAW) ? (uint32_t)stream->raw_size : 4u; +} + +static uint32_t lox_ts_stream_stride(const lox_ts_stream_t *stream) { + return (uint32_t)sizeof(lox_timestamp_t) + lox_ts_stream_val_size(stream); +} + +static uint8_t *lox_ts_sample_ptr(lox_ts_stream_t *stream, uint32_t idx) { + return stream->buf + (idx * stream->sample_stride); +} + +static const uint8_t *lox_ts_sample_ptr_const(const lox_ts_stream_t *stream, uint32_t idx) { + return stream->buf + (idx * stream->sample_stride); +} + +static void lox_ts_read_sample(const lox_ts_stream_t *stream, uint32_t idx, lox_ts_sample_t *out) { + const uint8_t *slot = lox_ts_sample_ptr_const(stream, idx); + uint32_t val_len = lox_ts_stream_val_size(stream); + + memset(out, 0, sizeof(*out)); + memcpy(&out->ts, slot, sizeof(out->ts)); + memcpy(&out->v, slot + sizeof(out->ts), val_len); +} + +static void lox_ts_write_sample(const lox_ts_stream_t *stream, uint32_t idx, const lox_ts_sample_t *sample) { + uint8_t *slot = lox_ts_sample_ptr((lox_ts_stream_t *)stream, idx); + uint32_t val_len = lox_ts_stream_val_size(stream); + + memcpy(slot, &sample->ts, sizeof(sample->ts)); + memcpy(slot + sizeof(sample->ts), &sample->v, val_len); +} + +static void lox_ts_copy_sample_slot(const lox_ts_stream_t *stream, uint32_t dst_idx, uint32_t src_idx) { + uint8_t *dst = lox_ts_sample_ptr((lox_ts_stream_t *)stream, dst_idx); + const uint8_t *src = lox_ts_sample_ptr_const(stream, src_idx); + memcpy(dst, src, stream->sample_stride); +} + +static lox_err_t lox_ts_register_apply(lox_core_t *core, + const char *name, + lox_ts_type_t type, + size_t raw_size) { + uint32_t i; + + if (lox_ts_find(core, name) != NULL) { + return LOX_ERR_EXISTS; + } + if (core->ts.registered_streams >= LOX_TS_MAX_STREAMS) { + return LOX_ERR_FULL; + } + + for (i = 0; i < LOX_TS_MAX_STREAMS; ++i) { + lox_ts_stream_t *stream = &core->ts.streams[i]; + if (!stream->registered) { + memset(stream->name, 0, sizeof(stream->name)); + memcpy(stream->name, name, strlen(name) + 1u); + stream->type = type; + stream->raw_size = (type == LOX_TS_RAW) ? raw_size : 0u; + stream->sample_stride = lox_ts_stream_stride(stream); + if (stream->sample_stride == 0u) { + return LOX_ERR_INVALID; + } + stream->capacity = (uint32_t)((core->ts_arena.capacity / LOX_TS_MAX_STREAMS) / stream->sample_stride); + if (stream->capacity < 4u) { + return LOX_ERR_NO_MEM; + } + stream->head = 0u; + stream->tail = 0u; + stream->count = 0u; + stream->registered = true; + core->ts.registered_streams++; + core->ts.mutation_seq++; + return LOX_OK; + } + } + + return LOX_ERR_FULL; +} + +static lox_err_t lox_ts_stream_bytes(const lox_core_t *core, uint32_t *bytes_out) { + size_t bytes_per_stream; + + bytes_per_stream = core->ts_arena.capacity / LOX_TS_MAX_STREAMS; + if (bytes_per_stream < (sizeof(lox_timestamp_t) + 4u) * 4u) { + return LOX_ERR_NO_MEM; + } + + *bytes_out = (uint32_t)bytes_per_stream; + return LOX_OK; +} + +static void lox_ts_set_value(lox_ts_stream_t *stream, lox_ts_sample_t *sample, const void *val) { + if (stream->type == LOX_TS_F32) { + memcpy(&sample->v.f32, val, sizeof(sample->v.f32)); + } else if (stream->type == LOX_TS_I32) { + memcpy(&sample->v.i32, val, sizeof(sample->v.i32)); + } else if (stream->type == LOX_TS_U32) { + memcpy(&sample->v.u32, val, sizeof(sample->v.u32)); + } else { + memcpy(sample->v.raw, val, stream->raw_size); + } +} + +static void lox_ts_rb_insert(lox_ts_stream_t *stream, const lox_ts_sample_t *sample) { + if (stream->count == stream->capacity) { +#if LOX_TS_OVERFLOW_POLICY == LOX_TS_POLICY_DROP_OLDEST + lox_ts_sample_t dropped; + lox_ts_read_sample(stream, stream->tail, &dropped); + LOX_LOG("WARN", + "TS stream '%s' full: dropping oldest sample ts=%u", + stream->name, + (unsigned)dropped.ts); +#endif + } + + lox_ts_write_sample(stream, stream->head, sample); + stream->head = (stream->head + 1u) % stream->capacity; + + if (stream->count < stream->capacity) { + stream->count++; + } else { + stream->tail = (stream->tail + 1u) % stream->capacity; + } +} + +static void lox_ts_downsample_oldest(lox_ts_stream_t *stream) { + uint32_t i0 = stream->tail; + uint32_t i1 = (stream->tail + 1u) % stream->capacity; + uint32_t idx; + uint32_t next; + lox_ts_sample_t a; + lox_ts_sample_t b; + + lox_ts_read_sample(stream, i0, &a); + lox_ts_read_sample(stream, i1, &b); + + LOX_LOG("INFO", + "TS stream '%s' downsampling oldest two samples", + stream->name); + + a.ts = (a.ts / 2u) + (b.ts / 2u); + + if (stream->type == LOX_TS_F32) { + a.v.f32 = (a.v.f32 + b.v.f32) * 0.5f; + } else if (stream->type == LOX_TS_I32) { + a.v.i32 = (a.v.i32 / 2) + (b.v.i32 / 2); + } else if (stream->type == LOX_TS_U32) { + a.v.u32 = (a.v.u32 / 2u) + (b.v.u32 / 2u); + } else { + size_t i; + for (i = 0u; i < stream->raw_size; ++i) { + uint16_t merged = (uint16_t)a.v.raw[i] + (uint16_t)b.v.raw[i]; + a.v.raw[i] = (uint8_t)(merged / 2u); + } + } + lox_ts_write_sample(stream, i0, &a); + + idx = i1; + while (idx != stream->head) { + next = (idx + 1u) % stream->capacity; + if (next == stream->head) { + break; + } + lox_ts_copy_sample_slot(stream, idx, next); + idx = next; + } + + stream->head = (stream->head + stream->capacity - 1u) % stream->capacity; + stream->count--; +} + +lox_err_t lox_ts_init(lox_t *db) { + lox_core_t *core = lox_core(db); +#if LOX_ENABLE_TS + uint32_t stream_bytes; + uint32_t i; +#endif + + memset(&core->ts, 0, sizeof(core->ts)); + +#if LOX_ENABLE_TS + if (lox_ts_stream_bytes(core, &stream_bytes) != LOX_OK) { + return LOX_ERR_NO_MEM; + } + + for (i = 0; i < LOX_TS_MAX_STREAMS; ++i) { + core->ts.streams[i].buf = core->ts_arena.base + (i * stream_bytes); + core->ts.streams[i].sample_stride = 0u; + core->ts.streams[i].capacity = 0u; + } +#endif + + return LOX_OK; +} + +#if LOX_ENABLE_TS +lox_err_t lox_ts_register(lox_t *db, const char *name, lox_ts_type_t type, size_t raw_size) { + lox_core_t *core; + lox_err_t err; + lox_err_t rc = LOX_OK; + uint32_t before_registered = 0u; + uint32_t restore_index = UINT32_MAX; + lox_ts_stream_t restore_stream; + + if (db == NULL) { + return LOX_ERR_INVALID; + } + + err = lox_ts_validate_name(name); + if (err != LOX_OK) { + return err; + } + + LOX_LOCK(db); + core = lox_core(db); + if (core->magic != LOX_MAGIC) { + rc = LOX_ERR_INVALID; + goto unlock; + } + + if (type == LOX_TS_RAW && (raw_size == 0u || raw_size > LOX_TS_RAW_MAX)) { + rc = LOX_ERR_INVALID; + goto unlock; + } + + if (core->wal_enabled && core->storage != NULL && !core->storage_loading && !core->wal_replaying) { + rc = lox_persist_ts_register(db, name, type, raw_size); + if (rc != LOX_OK) { + goto unlock; + } + rc = lox_ts_register_apply(core, name, type, raw_size); + goto unlock; + } + + before_registered = core->ts.registered_streams; + memset(&restore_stream, 0, sizeof(restore_stream)); + { + uint32_t i; + for (i = 0u; i < LOX_TS_MAX_STREAMS; ++i) { + if (!core->ts.streams[i].registered) { + restore_index = i; + restore_stream = core->ts.streams[i]; + break; + } + } + } + rc = lox_ts_register_apply(core, name, type, raw_size); + if (rc != LOX_OK) { + goto unlock; + } + rc = lox_storage_flush(db); + if (rc != LOX_OK) { + core->ts.registered_streams = before_registered; + if (restore_index != UINT32_MAX) { + core->ts.streams[restore_index] = restore_stream; + } + if (core->ts.mutation_seq != 0u) { + core->ts.mutation_seq--; + } + } + +unlock: + LOX_UNLOCK(db); + return rc; +} + +lox_err_t lox_ts_insert(lox_t *db, const char *name, lox_timestamp_t ts, const void *val) { + lox_core_t *core; + lox_ts_stream_t *stream; + lox_ts_sample_t sample; + lox_err_t rc = LOX_OK; + + if (db == NULL || val == NULL) { + return LOX_ERR_INVALID; + } + + LOX_LOCK(db); + core = lox_core(db); + if (core->magic != LOX_MAGIC) { + rc = LOX_ERR_INVALID; + goto unlock; + } + + stream = lox_ts_find(core, name); + if (stream == NULL) { + rc = LOX_ERR_NOT_FOUND; + goto unlock; + } + + if (stream->capacity == 0u) { + rc = LOX_ERR_NO_MEM; + goto unlock; + } + +#if LOX_TS_OVERFLOW_POLICY == LOX_TS_POLICY_REJECT + if (stream->count == stream->capacity) { + LOX_LOG("WARN", + "TS stream '%s' full: rejecting new sample (REJECT policy)", + stream->name); + core->ts_dropped_samples++; + rc = LOX_ERR_FULL; + goto unlock; + } +#elif LOX_TS_OVERFLOW_POLICY == LOX_TS_POLICY_DOWNSAMPLE +#endif + + memset(&sample, 0, sizeof(sample)); + sample.ts = ts; + lox_ts_set_value(stream, &sample, val); + rc = lox_persist_ts_insert(db, name, ts, val, (stream->type == LOX_TS_RAW) ? stream->raw_size : 4u); + if (rc == LOX_OK) { +#if LOX_TS_OVERFLOW_POLICY == LOX_TS_POLICY_DOWNSAMPLE + if (stream->count == stream->capacity) { + lox_ts_downsample_oldest(stream); + core->ts_dropped_samples++; + } +#elif LOX_TS_OVERFLOW_POLICY == LOX_TS_POLICY_DROP_OLDEST + if (stream->count == stream->capacity) { + core->ts_dropped_samples++; + } +#endif + lox_ts_rb_insert(stream, &sample); + core->ts.mutation_seq++; + lox__maybe_compact(db); + } + +unlock: + LOX_UNLOCK(db); + return rc; +} + +lox_err_t lox_ts_last(lox_t *db, const char *name, lox_ts_sample_t *out) { + lox_core_t *core; + lox_ts_stream_t *stream; + uint32_t idx; + lox_err_t rc = LOX_OK; + + if (db == NULL || out == NULL) { + return LOX_ERR_INVALID; + } + + LOX_LOCK(db); + core = lox_core(db); + if (core->magic != LOX_MAGIC) { + rc = LOX_ERR_INVALID; + goto unlock; + } + + stream = lox_ts_find(core, name); + if (stream == NULL || stream->count == 0u) { + rc = LOX_ERR_NOT_FOUND; + goto unlock; + } + + idx = (stream->head + stream->capacity - 1u) % stream->capacity; + lox_ts_read_sample(stream, idx, out); + rc = LOX_OK; + +unlock: + LOX_UNLOCK(db); + return rc; +} + +lox_err_t lox_ts_query(lox_t *db, + const char *name, + lox_timestamp_t from, + lox_timestamp_t to, + lox_ts_query_cb_t cb, + void *ctx) { + lox_core_t *core; + lox_ts_stream_t *stream; + uint32_t i; + uint32_t idx; + uint32_t snapshot_mutation_seq; + lox_err_t rc = LOX_OK; + + if (db == NULL || cb == NULL) { + return LOX_ERR_INVALID; + } + + LOX_LOCK(db); + core = lox_core(db); + if (core->magic != LOX_MAGIC) { + rc = LOX_ERR_INVALID; + goto unlock; + } + + stream = lox_ts_find(core, name); + if (stream == NULL) { + rc = LOX_ERR_NOT_FOUND; + goto unlock; + } + + { + uint32_t snapshot_tail = stream->tail; + uint32_t snapshot_count = stream->count; + uint32_t cap = stream->capacity; + snapshot_mutation_seq = core->ts.mutation_seq; + idx = snapshot_tail; + for (i = 0u; i < snapshot_count; ++i) { + lox_ts_sample_t sample; + lox_ts_read_sample(stream, idx, &sample); + bool in_range = (from <= to && sample.ts >= from && sample.ts <= to); + idx = (idx + 1u) % cap; + LOX_UNLOCK(db); + if (in_range && !cb(&sample, ctx)) { + return LOX_OK; + } + LOX_LOCK(db); + core = lox_core(db); + if (core->magic != LOX_MAGIC) { + rc = LOX_ERR_INVALID; + goto unlock; + } + stream = lox_ts_find(core, name); + if (stream == NULL) { + rc = LOX_ERR_NOT_FOUND; + goto unlock; + } + if (core->ts.mutation_seq != snapshot_mutation_seq) { + rc = LOX_ERR_MODIFIED; + goto unlock; + } + } + } + + rc = LOX_OK; + +unlock: + LOX_UNLOCK(db); + return rc; +} + +lox_err_t lox_ts_query_buf(lox_t *db, + const char *name, + lox_timestamp_t from, + lox_timestamp_t to, + lox_ts_sample_t *buf, + size_t max_count, + size_t *out_count) { + lox_core_t *core; + lox_ts_stream_t *stream; + uint32_t i; + uint32_t idx; + size_t written = 0u; + lox_err_t status = LOX_OK; + + if (db == NULL || buf == NULL) { + return LOX_ERR_INVALID; + } + + LOX_LOCK(db); + core = lox_core(db); + if (core->magic != LOX_MAGIC) { + status = LOX_ERR_INVALID; + goto unlock; + } + + stream = lox_ts_find(core, name); + if (stream == NULL) { + status = LOX_ERR_NOT_FOUND; + goto unlock; + } + + idx = stream->tail; + for (i = 0; i < stream->count; ++i) { + lox_ts_sample_t sample; + lox_ts_read_sample(stream, idx, &sample); + if (from <= to && sample.ts >= from && sample.ts <= to) { + if (written < max_count) { + buf[written] = sample; + } else { + status = LOX_ERR_OVERFLOW; + } + written++; + } + idx = (idx + 1u) % stream->capacity; + } + + if (out_count != NULL) { + *out_count = (written < max_count) ? written : max_count; + } + +unlock: + LOX_UNLOCK(db); + return status; +} + +lox_err_t lox_ts_count(lox_t *db, + const char *name, + lox_timestamp_t from, + lox_timestamp_t to, + size_t *out_count) { + lox_core_t *core; + lox_ts_stream_t *stream; + uint32_t i; + uint32_t idx; + size_t count = 0u; + lox_err_t rc = LOX_OK; + + if (db == NULL || out_count == NULL) { + return LOX_ERR_INVALID; + } + + LOX_LOCK(db); + core = lox_core(db); + if (core->magic != LOX_MAGIC) { + rc = LOX_ERR_INVALID; + goto unlock; + } + + stream = lox_ts_find(core, name); + if (stream == NULL) { + rc = LOX_ERR_NOT_FOUND; + goto unlock; + } + + idx = stream->tail; + for (i = 0; i < stream->count; ++i) { + lox_ts_sample_t sample; + lox_ts_read_sample(stream, idx, &sample); + if (from <= to && sample.ts >= from && sample.ts <= to) { + count++; + } + idx = (idx + 1u) % stream->capacity; + } + + *out_count = count; + rc = LOX_OK; + +unlock: + LOX_UNLOCK(db); + return rc; +} + +lox_err_t lox_ts_clear(lox_t *db, const char *name) { + lox_core_t *core; + lox_ts_stream_t *stream; + lox_err_t rc = LOX_OK; + uint32_t saved_head = 0u; + uint32_t saved_tail = 0u; + uint32_t saved_count = 0u; + + if (db == NULL) { + return LOX_ERR_INVALID; + } + + LOX_LOCK(db); + core = lox_core(db); + if (core->magic != LOX_MAGIC) { + rc = LOX_ERR_INVALID; + goto unlock; + } + + stream = lox_ts_find(core, name); + if (stream == NULL) { + rc = LOX_ERR_NOT_FOUND; + goto unlock; + } + + if (core->wal_enabled && core->storage != NULL && !core->storage_loading && !core->wal_replaying) { + rc = lox_persist_ts_clear(db, name); + if (rc != LOX_OK) { + goto unlock; + } + stream->head = 0u; + stream->tail = 0u; + stream->count = 0u; + core->ts.mutation_seq++; + goto unlock; + } + + saved_head = stream->head; + saved_tail = stream->tail; + saved_count = stream->count; + stream->head = 0u; + stream->tail = 0u; + stream->count = 0u; + core->ts.mutation_seq++; + rc = lox_storage_flush(db); + if (rc != LOX_OK) { + stream->head = saved_head; + stream->tail = saved_tail; + stream->count = saved_count; + core->ts.mutation_seq--; + } + +unlock: + LOX_UNLOCK(db); + return rc; +} +#else +lox_err_t lox_ts_register(lox_t *db, const char *name, lox_ts_type_t type, size_t raw_size) { + (void)db; + (void)name; + (void)type; + (void)raw_size; + return LOX_ERR_DISABLED; +} + +lox_err_t lox_ts_insert(lox_t *db, const char *name, lox_timestamp_t ts, const void *val) { + (void)db; + (void)name; + (void)ts; + (void)val; + return LOX_ERR_DISABLED; +} + +lox_err_t lox_ts_last(lox_t *db, const char *name, lox_ts_sample_t *out) { + (void)db; + (void)name; + (void)out; + return LOX_ERR_DISABLED; +} + +lox_err_t lox_ts_query(lox_t *db, + const char *name, + lox_timestamp_t from, + lox_timestamp_t to, + lox_ts_query_cb_t cb, + void *ctx) { + (void)db; + (void)name; + (void)from; + (void)to; + (void)cb; + (void)ctx; + return LOX_ERR_DISABLED; +} + +lox_err_t lox_ts_query_buf(lox_t *db, + const char *name, + lox_timestamp_t from, + lox_timestamp_t to, + lox_ts_sample_t *buf, + size_t max_count, + size_t *out_count) { + (void)db; + (void)name; + (void)from; + (void)to; + (void)buf; + (void)max_count; + (void)out_count; + return LOX_ERR_DISABLED; +} + +lox_err_t lox_ts_count(lox_t *db, + const char *name, + lox_timestamp_t from, + lox_timestamp_t to, + size_t *out_count) { + (void)db; + (void)name; + (void)from; + (void)to; + (void)out_count; + return LOX_ERR_DISABLED; +} + +lox_err_t lox_ts_clear(lox_t *db, const char *name) { + (void)db; + (void)name; + return LOX_ERR_DISABLED; +} +#endif diff --git a/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/src/lox_wal.c b/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/src/lox_wal.c new file mode 100644 index 0000000..14c48ff --- /dev/null +++ b/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/src/lox_wal.c @@ -0,0 +1,2414 @@ +// SPDX-License-Identifier: MIT +#include "lox_internal.h" +#include "lox_crc.h" +#include "lox_arena.h" +#include "lox_lock.h" + +#include + +#if !LOX_ENABLE_TS +static lox_err_t lox_ts_register_stub(lox_t *db, const char *name, lox_ts_type_t type, size_t raw_size) { + (void)db; + (void)name; + (void)type; + (void)raw_size; + return LOX_ERR_DISABLED; +} +static lox_err_t lox_ts_insert_stub(lox_t *db, const char *name, lox_timestamp_t ts, const void *val) { + (void)db; + (void)name; + (void)ts; + (void)val; + return LOX_ERR_DISABLED; +} +static lox_err_t lox_ts_clear_stub(lox_t *db, const char *name) { + (void)db; + (void)name; + return LOX_ERR_DISABLED; +} +#define lox_ts_register lox_ts_register_stub +#define lox_ts_insert lox_ts_insert_stub +#define lox_ts_clear lox_ts_clear_stub +#endif + +#if !LOX_ENABLE_REL +static lox_err_t lox_schema_init_stub(lox_schema_t *schema, const char *name, uint32_t max_rows) { + (void)schema; + (void)name; + (void)max_rows; + return LOX_ERR_DISABLED; +} +static lox_err_t lox_schema_add_stub(lox_schema_t *schema, + const char *col_name, + lox_col_type_t type, + size_t size, + bool is_index) { + (void)schema; + (void)col_name; + (void)type; + (void)size; + (void)is_index; + return LOX_ERR_DISABLED; +} +static lox_err_t lox_schema_seal_stub(lox_schema_t *schema) { + (void)schema; + return LOX_ERR_DISABLED; +} +static lox_err_t lox_table_create_stub(lox_t *db, lox_schema_t *schema) { + (void)db; + (void)schema; + return LOX_ERR_DISABLED; +} +static lox_err_t lox_table_get_stub(lox_t *db, const char *name, lox_table_t **out_table) { + (void)db; + (void)name; + (void)out_table; + return LOX_ERR_DISABLED; +} +static lox_err_t lox_rel_insert_stub(lox_t *db, lox_table_t *table, const void *row_buf) { + (void)db; + (void)table; + (void)row_buf; + return LOX_ERR_DISABLED; +} +static lox_err_t lox_rel_delete_stub(lox_t *db, lox_table_t *table, const void *search_val, uint32_t *out_deleted) { + (void)db; + (void)table; + (void)search_val; + (void)out_deleted; + return LOX_ERR_DISABLED; +} +static lox_err_t lox_rel_clear_stub(lox_t *db, lox_table_t *table) { + (void)db; + (void)table; + return LOX_ERR_DISABLED; +} +#define lox_schema_init lox_schema_init_stub +#define lox_schema_add lox_schema_add_stub +#define lox_schema_seal lox_schema_seal_stub +#define lox_table_create lox_table_create_stub +#define lox_table_get lox_table_get_stub +#define lox_rel_insert lox_rel_insert_stub +#define lox_rel_delete lox_rel_delete_stub +#define lox_rel_clear lox_rel_clear_stub +#endif + +enum { + LOX_WAL_MAGIC = 0x4D44424Cu, + LOX_WAL_VERSION = 0x00010000u, + LOX_SNAPSHOT_FORMAT_VERSION = 0x00020000u, + LOX_WAL_ENTRY_MAGIC = 0x454E5452u, + LOX_KV_PAGE_MAGIC = 0x4B565047u, + LOX_TS_PAGE_MAGIC = 0x54535047u, + LOX_REL_PAGE_MAGIC = 0x524C5047u, + LOX_SUPER_MAGIC = 0x53555052u, + LOX_WAL_ENGINE_KV = 0u, + LOX_WAL_ENGINE_TS = 1u, + LOX_WAL_ENGINE_REL = 2u, + LOX_WAL_ENGINE_TXN_KV = 3u, + LOX_WAL_ENGINE_META = 0xFFu, + LOX_WAL_OP_SET_INSERT = 0u, + LOX_WAL_OP_DEL = 1u, + LOX_WAL_OP_CLEAR = 2u, + LOX_WAL_OP_TXN_COMMIT = 5u, + LOX_WAL_OP_TS_REGISTER = 6u, + LOX_WAL_OP_REL_TABLE_CREATE = 7u +}; + +#define LOX_WAL_HEADER_SIZE 32u +#define LOX_PAGE_HEADER_SIZE 32u +#define LOX_SUPERBLOCK_SIZE 32u + +static uint32_t lox_align_u32(uint32_t value, uint32_t align) { + return (value + (align - 1u)) & ~(align - 1u); +} + +static uint32_t lox_kv_snapshot_payload_max(const lox_core_t *core) { + uint32_t max_entries; + uint32_t max_key_len = (LOX_KV_KEY_MAX_LEN > 0u) ? (LOX_KV_KEY_MAX_LEN - 1u) : 0u; + uint32_t per_entry = 1u + max_key_len + 4u + LOX_KV_VAL_MAX_LEN + 4u; + (void)core; + max_entries = (LOX_KV_MAX_KEYS > LOX_TXN_STAGE_KEYS) ? (LOX_KV_MAX_KEYS - LOX_TXN_STAGE_KEYS) : 0u; + return max_entries * per_entry; +} + +static void lox_put_u32(uint8_t *dst, uint32_t value) { + memcpy(dst, &value, sizeof(value)); +} + +static void lox_put_u16(uint8_t *dst, uint16_t value) { + memcpy(dst, &value, sizeof(value)); +} + +static uint32_t lox_get_u32(const uint8_t *src) { + uint32_t value = 0u; + memcpy(&value, src, sizeof(value)); + return value; +} + +static uint16_t lox_get_u16(const uint8_t *src) { + uint16_t value = 0u; + memcpy(&value, src, sizeof(value)); + return value; +} + +static uint32_t lox_bank_kv_offset(const lox_core_t *core, uint32_t bank) { + return ((bank == 0u) ? core->layout.bank_a_offset : core->layout.bank_b_offset); +} + +static uint32_t lox_bank_ts_offset(const lox_core_t *core, uint32_t bank) { + return lox_bank_kv_offset(core, bank) + core->layout.kv_size; +} + +static uint32_t lox_bank_rel_offset(const lox_core_t *core, uint32_t bank) { + return lox_bank_ts_offset(core, bank) + core->layout.ts_size; +} + +static bool lox_storage_ready(const lox_core_t *core) { + return core->storage != NULL && core->storage->read != NULL && core->storage->write != NULL && + core->storage->erase != NULL && core->storage->sync != NULL; +} + +static lox_err_t lox_storage_read_bytes(lox_core_t *core, uint32_t offset, void *buf, size_t len) { + lox_err_t err; + LOX_IO_BEFORE_READ(offset, len); + err = core->storage->read(core->storage->ctx, offset, buf, len); + LOX_IO_AFTER_READ(offset, len, err); + return err; +} + +static lox_err_t lox_storage_write_bytes(lox_core_t *core, uint32_t offset, const void *buf, size_t len) { + lox_err_t err; + LOX_IO_BEFORE_WRITE(offset, len); + err = core->storage->write(core->storage->ctx, offset, buf, len); + LOX_IO_AFTER_WRITE(offset, len, err); + if (err == LOX_OK) { + core->storage_bytes_written += (uint32_t)len; + } + return err; +} + +static lox_err_t lox_storage_erase_region(lox_core_t *core, uint32_t offset, uint32_t size) { + uint32_t pos; + + for (pos = 0u; pos < size; pos += core->storage->erase_size) { + LOX_IO_BEFORE_ERASE(offset + pos, core->storage->erase_size); + lox_err_t err = core->storage->erase(core->storage->ctx, offset + pos); + LOX_IO_AFTER_ERASE(offset + pos, core->storage->erase_size, err); + if (err != LOX_OK) { + return err; + } + } + + return LOX_OK; +} + +static lox_err_t lox_storage_sync_core(lox_core_t *core) { + lox_err_t err; + LOX_IO_BEFORE_SYNC(); + err = core->storage->sync(core->storage->ctx); + LOX_IO_AFTER_SYNC(err); + return err; +} + +static lox_err_t lox_write_wal_header(lox_core_t *core) { + uint8_t header[LOX_WAL_HEADER_SIZE]; + uint32_t crc; + + memset(header, 0, sizeof(header)); + lox_put_u32(header + 0u, LOX_WAL_MAGIC); + lox_put_u32(header + 4u, LOX_WAL_VERSION); + lox_put_u32(header + 8u, core->wal_entry_count); + lox_put_u32(header + 12u, core->wal_sequence); + crc = LOX_CRC32(header, 16u); + lox_put_u32(header + 16u, crc); + return lox_storage_write_bytes(core, core->layout.wal_offset, header, LOX_WAL_HEADER_SIZE); +} + +static lox_err_t lox_reset_wal(lox_core_t *core, uint32_t next_sequence) { + lox_err_t err; + + core->wal_sequence = next_sequence; + core->wal_entry_count = 0u; + core->wal_used = LOX_WAL_HEADER_SIZE; + + err = lox_storage_erase_region(core, core->layout.wal_offset, core->layout.wal_size); + if (err != LOX_OK) { + return err; + } + + err = lox_write_wal_header(core); + if (err != LOX_OK) { + return err; + } + + return lox_storage_sync_core(core); +} + +static lox_err_t lox_write_page_header(lox_core_t *core, + uint32_t offset, + uint32_t magic, + uint32_t generation, + uint32_t payload_length, + uint32_t entry_count, + uint32_t payload_crc) { + uint8_t header[LOX_PAGE_HEADER_SIZE]; + uint32_t header_crc; + + memset(header, 0, sizeof(header)); + lox_put_u32(header + 0u, magic); + lox_put_u32(header + 4u, LOX_SNAPSHOT_FORMAT_VERSION); + lox_put_u32(header + 8u, generation); + lox_put_u32(header + 12u, payload_length); + lox_put_u32(header + 16u, entry_count); + lox_put_u32(header + 20u, payload_crc); + header_crc = LOX_CRC32(header, 24u); + lox_put_u32(header + 24u, header_crc); + return lox_storage_write_bytes(core, offset, header, sizeof(header)); +} + +static bool lox_validate_page_header(const uint8_t *header, + uint32_t expected_magic, + uint32_t max_payload_len, + uint32_t *out_generation, + uint32_t *out_payload_len, + uint32_t *out_entry_count, + uint32_t *out_payload_crc) { + uint32_t header_crc = lox_get_u32(header + 24u); + uint32_t payload_len = lox_get_u32(header + 12u); + + if (lox_get_u32(header + 0u) != expected_magic) { + return false; + } + if (lox_get_u32(header + 4u) != LOX_SNAPSHOT_FORMAT_VERSION) { + return false; + } + if (LOX_CRC32(header, 24u) != header_crc) { + return false; + } + if (payload_len > max_payload_len) { + return false; + } + *out_generation = lox_get_u32(header + 8u); + *out_payload_len = payload_len; + *out_entry_count = lox_get_u32(header + 16u); + *out_payload_crc = lox_get_u32(header + 20u); + return true; +} + +static bool lox_validate_superblock(const uint8_t *super, + uint32_t *out_generation, + uint32_t *out_active_bank) { + uint32_t header_crc = lox_get_u32(super + 20u); + if (lox_get_u32(super + 0u) != LOX_SUPER_MAGIC) { + return false; + } + if (lox_get_u32(super + 4u) != LOX_SNAPSHOT_FORMAT_VERSION) { + return false; + } + if (LOX_CRC32(super, 20u) != header_crc) { + return false; + } + if (lox_get_u32(super + 16u) > 1u) { + return false; + } + *out_generation = lox_get_u32(super + 12u); + *out_active_bank = lox_get_u32(super + 16u); + return true; +} + +static lox_err_t lox_write_superblock(lox_core_t *core, uint32_t generation, uint32_t active_bank) { + uint8_t super[LOX_SUPERBLOCK_SIZE]; + uint32_t header_crc; + uint32_t offset; + + memset(super, 0, sizeof(super)); + lox_put_u32(super + 0u, LOX_SUPER_MAGIC); + lox_put_u32(super + 4u, LOX_SNAPSHOT_FORMAT_VERSION); + lox_put_u32(super + 8u, LOX_WAL_VERSION); + lox_put_u32(super + 12u, generation); + lox_put_u32(super + 16u, active_bank); + header_crc = LOX_CRC32(super, 20u); + lox_put_u32(super + 20u, header_crc); + + offset = (generation & 1u) == 0u ? core->layout.super_b_offset : core->layout.super_a_offset; + return lox_storage_write_bytes(core, offset, super, sizeof(super)); +} + +static lox_err_t lox_write_kv_page(lox_core_t *core, uint32_t bank, uint32_t generation) { + uint32_t count = 0u; + uint32_t page_offset = lox_bank_kv_offset(core, bank); + uint32_t offset = page_offset + LOX_PAGE_HEADER_SIZE; + uint32_t crc = 0xFFFFFFFFu; + uint32_t max_end = page_offset + core->layout.kv_size; + uint32_t i; + lox_err_t err; + + for (i = 0u; i < core->kv.bucket_count; ++i) { + const lox_kv_bucket_t *bucket = &core->kv.buckets[i]; + uint8_t key_len; + uint8_t header[4]; + + if (bucket->state != 1u) { + continue; + } + + key_len = (uint8_t)strlen(bucket->key); + if (offset + 1u + key_len + 4u + bucket->val_len + 4u > max_end) { + return LOX_ERR_STORAGE; + } + + err = lox_storage_write_bytes(core, offset, &key_len, 1u); + if (err != LOX_OK) { + return err; + } + crc = lox_crc32(crc, &key_len, 1u); + offset += 1u; + + err = lox_storage_write_bytes(core, offset, bucket->key, key_len); + if (err != LOX_OK) { + return err; + } + crc = lox_crc32(crc, bucket->key, key_len); + offset += key_len; + + lox_put_u32(header, bucket->val_len); + err = lox_storage_write_bytes(core, offset, header, 4u); + if (err != LOX_OK) { + return err; + } + crc = lox_crc32(crc, header, 4u); + offset += 4u; + + err = lox_storage_write_bytes(core, offset, &core->kv.value_store[bucket->val_offset], bucket->val_len); + if (err != LOX_OK) { + return err; + } + crc = lox_crc32(crc, &core->kv.value_store[bucket->val_offset], bucket->val_len); + offset += bucket->val_len; + + lox_put_u32(header, bucket->expires_at); + err = lox_storage_write_bytes(core, offset, header, 4u); + if (err != LOX_OK) { + return err; + } + crc = lox_crc32(crc, header, 4u); + offset += 4u; + count++; + } + + return lox_write_page_header(core, + page_offset, + LOX_KV_PAGE_MAGIC, + generation, + offset - (page_offset + LOX_PAGE_HEADER_SIZE), + count, + crc); +} + +static uint32_t lox_ts_stream_val_size(const lox_ts_stream_t *stream) { + return (stream->type == LOX_TS_RAW) ? (uint32_t)stream->raw_size : 4u; +} + +static const uint8_t *lox_ts_sample_ptr_const(const lox_ts_stream_t *stream, uint32_t idx) { + return stream->buf + (idx * stream->sample_stride); +} + +static lox_err_t lox_write_ts_page(lox_core_t *core, uint32_t bank, uint32_t generation) { + uint32_t stream_count = 0u; + uint32_t page_offset = lox_bank_ts_offset(core, bank); + uint32_t offset = page_offset + LOX_PAGE_HEADER_SIZE; + uint32_t crc = 0xFFFFFFFFu; + uint32_t max_end = page_offset + core->layout.ts_size; + uint32_t i; + lox_err_t err; + + for (i = 0u; i < LOX_TS_MAX_STREAMS; ++i) { + const lox_ts_stream_t *stream = &core->ts.streams[i]; + uint8_t name_len; + uint8_t one; + uint8_t u32buf[4]; + uint32_t j; + uint32_t idx; + + if (!stream->registered) { + continue; + } + + name_len = (uint8_t)strlen(stream->name); + if (offset + 1u + name_len + 1u + 4u + 4u > max_end) { + return LOX_ERR_STORAGE; + } + one = name_len; + err = lox_storage_write_bytes(core, offset, &one, 1u); + if (err != LOX_OK) { + return err; + } + crc = lox_crc32(crc, &one, 1u); + offset += 1u; + + err = lox_storage_write_bytes(core, offset, stream->name, name_len); + if (err != LOX_OK) { + return err; + } + crc = lox_crc32(crc, stream->name, name_len); + offset += name_len; + + one = (uint8_t)stream->type; + err = lox_storage_write_bytes(core, offset, &one, 1u); + if (err != LOX_OK) { + return err; + } + crc = lox_crc32(crc, &one, 1u); + offset += 1u; + + lox_put_u32(u32buf, (uint32_t)stream->raw_size); + err = lox_storage_write_bytes(core, offset, u32buf, 4u); + if (err != LOX_OK) { + return err; + } + crc = lox_crc32(crc, u32buf, 4u); + offset += 4u; + + lox_put_u32(u32buf, stream->count); + err = lox_storage_write_bytes(core, offset, u32buf, 4u); + if (err != LOX_OK) { + return err; + } + crc = lox_crc32(crc, u32buf, 4u); + offset += 4u; + + idx = stream->tail; + for (j = 0u; j < stream->count; ++j) { + const uint8_t *sample_ptr = lox_ts_sample_ptr_const(stream, idx); + lox_timestamp_t sample_ts = 0; + uint32_t val_len = lox_ts_stream_val_size(stream); + uint64_t ts; + uint32_t ts_low; + uint32_t ts_high; + + memcpy(&sample_ts, sample_ptr, sizeof(sample_ts)); + ts = (uint64_t)sample_ts; + ts_low = (uint32_t)(ts & 0xFFFFFFFFu); + ts_high = (uint32_t)(ts >> 32u); + + if (offset + 8u + val_len > max_end) { + return LOX_ERR_STORAGE; + } + lox_put_u32(u32buf, ts_low); + err = lox_storage_write_bytes(core, offset, u32buf, 4u); + if (err != LOX_OK) { + return err; + } + crc = lox_crc32(crc, u32buf, 4u); + offset += 4u; + + lox_put_u32(u32buf, ts_high); + err = lox_storage_write_bytes(core, offset, u32buf, 4u); + if (err != LOX_OK) { + return err; + } + crc = lox_crc32(crc, u32buf, 4u); + offset += 4u; + + err = lox_storage_write_bytes(core, offset, sample_ptr + sizeof(sample_ts), val_len); + if (err != LOX_OK) { + return err; + } + crc = lox_crc32(crc, sample_ptr + sizeof(sample_ts), val_len); + offset += val_len; + idx = (idx + 1u) % stream->capacity; + } + + stream_count++; + } + + return lox_write_page_header(core, + page_offset, + LOX_TS_PAGE_MAGIC, + generation, + offset - (page_offset + LOX_PAGE_HEADER_SIZE), + stream_count, + crc); +} + +static lox_err_t lox_write_rel_page(lox_core_t *core, uint32_t bank, uint32_t generation) { + uint32_t table_count = 0u; + uint32_t page_offset = lox_bank_rel_offset(core, bank); + uint32_t offset = page_offset + LOX_PAGE_HEADER_SIZE; + uint32_t crc = 0xFFFFFFFFu; + uint32_t max_end = page_offset + core->layout.rel_size; + uint32_t i; + lox_err_t err; + + for (i = 0u; i < LOX_REL_MAX_TABLES; ++i) { + const lox_table_t *table = &core->rel.tables[i]; + uint8_t name_len; + uint8_t meta[2]; + uint8_t u16buf[2]; + uint8_t u32buf[4]; + uint32_t j; + + if (!table->registered) { + continue; + } + + name_len = (uint8_t)strlen(table->name); + if (offset + 1u + name_len + 2u + 4u + 4u + 4u + 4u + 4u > max_end) { + return LOX_ERR_STORAGE; + } + err = lox_storage_write_bytes(core, offset, &name_len, 1u); + if (err != LOX_OK) { + return err; + } + crc = lox_crc32(crc, &name_len, 1u); + offset += 1u; + + err = lox_storage_write_bytes(core, offset, table->name, name_len); + if (err != LOX_OK) { + return err; + } + crc = lox_crc32(crc, table->name, name_len); + offset += name_len; + + lox_put_u16(u16buf, table->schema_version); + err = lox_storage_write_bytes(core, offset, u16buf, 2u); + if (err != LOX_OK) { + return err; + } + crc = lox_crc32(crc, u16buf, 2u); + offset += 2u; + + lox_put_u32(u32buf, table->max_rows); + err = lox_storage_write_bytes(core, offset, u32buf, 4u); + if (err != LOX_OK) { + return err; + } + crc = lox_crc32(crc, u32buf, 4u); + offset += 4u; + + lox_put_u32(u32buf, (uint32_t)table->row_size); + err = lox_storage_write_bytes(core, offset, u32buf, 4u); + if (err != LOX_OK) { + return err; + } + crc = lox_crc32(crc, u32buf, 4u); + offset += 4u; + + lox_put_u32(u32buf, table->col_count); + err = lox_storage_write_bytes(core, offset, u32buf, 4u); + if (err != LOX_OK) { + return err; + } + crc = lox_crc32(crc, u32buf, 4u); + offset += 4u; + + lox_put_u32(u32buf, table->index_col); + err = lox_storage_write_bytes(core, offset, u32buf, 4u); + if (err != LOX_OK) { + return err; + } + crc = lox_crc32(crc, u32buf, 4u); + offset += 4u; + + lox_put_u32(u32buf, table->live_count); + err = lox_storage_write_bytes(core, offset, u32buf, 4u); + if (err != LOX_OK) { + return err; + } + crc = lox_crc32(crc, u32buf, 4u); + offset += 4u; + + for (j = 0u; j < table->col_count; ++j) { + const lox_col_desc_t *col = &table->cols[j]; + uint8_t col_name_len = (uint8_t)strlen(col->name); + + if (offset + 1u + col_name_len + 2u + 4u > max_end) { + return LOX_ERR_STORAGE; + } + err = lox_storage_write_bytes(core, offset, &col_name_len, 1u); + if (err != LOX_OK) { + return err; + } + crc = lox_crc32(crc, &col_name_len, 1u); + offset += 1u; + + err = lox_storage_write_bytes(core, offset, col->name, col_name_len); + if (err != LOX_OK) { + return err; + } + crc = lox_crc32(crc, col->name, col_name_len); + offset += col_name_len; + + meta[0] = (uint8_t)col->type; + meta[1] = (uint8_t)(col->is_index ? 1u : 0u); + err = lox_storage_write_bytes(core, offset, meta, 2u); + if (err != LOX_OK) { + return err; + } + crc = lox_crc32(crc, meta, 2u); + offset += 2u; + + lox_put_u32(u32buf, (uint32_t)col->size); + err = lox_storage_write_bytes(core, offset, u32buf, 4u); + if (err != LOX_OK) { + return err; + } + crc = lox_crc32(crc, u32buf, 4u); + offset += 4u; + } + + for (j = 0u; j < table->order_count; ++j) { + uint32_t row_idx = table->order[j]; + if (((table->alive_bitmap[row_idx >> 3u] >> (row_idx & 7u)) & 1u) == 0u) { + continue; + } + if (offset + (uint32_t)table->row_size > max_end) { + return LOX_ERR_STORAGE; + } + err = lox_storage_write_bytes(core, offset, table->rows + ((size_t)row_idx * table->row_size), table->row_size); + if (err != LOX_OK) { + return err; + } + crc = lox_crc32(crc, table->rows + ((size_t)row_idx * table->row_size), table->row_size); + offset += (uint32_t)table->row_size; + } + + table_count++; + } + + return lox_write_page_header(core, + page_offset, + LOX_REL_PAGE_MAGIC, + generation, + offset - (page_offset + LOX_PAGE_HEADER_SIZE), + table_count, + crc); +} + +static lox_err_t lox_write_snapshot_bank(lox_core_t *core, uint32_t bank, uint32_t generation) { + lox_err_t err; + uint32_t bank_offset = (bank == 0u) ? core->layout.bank_a_offset : core->layout.bank_b_offset; + + err = lox_storage_erase_region(core, bank_offset, core->layout.bank_size); + if (err != LOX_OK) { + return err; + } + err = lox_write_kv_page(core, bank, generation); + if (err != LOX_OK) { + return err; + } + err = lox_write_ts_page(core, bank, generation); + if (err != LOX_OK) { + return err; + } + return lox_write_rel_page(core, bank, generation); +} + +static lox_err_t lox_load_kv_page(lox_t *db, uint32_t bank, uint32_t expected_generation) { + lox_core_t *core = lox_core(db); + uint8_t header[LOX_PAGE_HEADER_SIZE]; + uint32_t page_offset = lox_bank_kv_offset(core, bank); + uint32_t generation = 0u; + uint32_t payload_len = 0u; + uint32_t payload_crc = 0u; + uint32_t offset; + uint32_t payload_offset; + uint32_t count; + uint32_t payload_crc_calc = 0xFFFFFFFFu; + uint32_t i; + lox_err_t err; + + err = lox_storage_read_bytes(core, page_offset, header, sizeof(header)); + if (err != LOX_OK) { + return err; + } + if (!lox_validate_page_header(header, + LOX_KV_PAGE_MAGIC, + core->layout.kv_size - LOX_PAGE_HEADER_SIZE, + &generation, + &payload_len, + &count, + &payload_crc)) { + return LOX_ERR_CORRUPT; + } + if (generation != expected_generation) { + return LOX_ERR_CORRUPT; + } + + offset = page_offset + LOX_PAGE_HEADER_SIZE; + payload_offset = offset; + for (i = 0u; i < count; ++i) { + uint8_t key_len = 0u; + char key[LOX_KV_KEY_MAX_LEN]; + uint8_t u32buf[4]; + uint32_t val_len; + uint32_t expires_at; + uint8_t value[LOX_KV_VAL_MAX_LEN]; + + if (offset + 1u > payload_offset + payload_len) { + return LOX_ERR_CORRUPT; + } + err = lox_storage_read_bytes(core, offset, &key_len, 1u); + if (err != LOX_OK || key_len >= LOX_KV_KEY_MAX_LEN) { + return LOX_ERR_CORRUPT; + } + payload_crc_calc = lox_crc32(payload_crc_calc, &key_len, 1u); + offset += 1u; + + if (offset + key_len > payload_offset + payload_len) { + return LOX_ERR_CORRUPT; + } + err = lox_storage_read_bytes(core, offset, key, key_len); + if (err != LOX_OK) { + return LOX_ERR_CORRUPT; + } + payload_crc_calc = lox_crc32(payload_crc_calc, (const uint8_t *)key, key_len); + key[key_len] = '\0'; + offset += key_len; + + if (offset + 4u > payload_offset + payload_len) { + return LOX_ERR_CORRUPT; + } + err = lox_storage_read_bytes(core, offset, u32buf, 4u); + if (err != LOX_OK) { + return LOX_ERR_CORRUPT; + } + payload_crc_calc = lox_crc32(payload_crc_calc, u32buf, 4u); + val_len = lox_get_u32(u32buf); + offset += 4u; + if (val_len > LOX_KV_VAL_MAX_LEN) { + return LOX_ERR_CORRUPT; + } + + if (offset + val_len + 4u > payload_offset + payload_len) { + return LOX_ERR_CORRUPT; + } + err = lox_storage_read_bytes(core, offset, value, val_len); + if (err != LOX_OK) { + return LOX_ERR_CORRUPT; + } + payload_crc_calc = lox_crc32(payload_crc_calc, value, val_len); + offset += val_len; + + err = lox_storage_read_bytes(core, offset, u32buf, 4u); + if (err != LOX_OK) { + return LOX_ERR_CORRUPT; + } + payload_crc_calc = lox_crc32(payload_crc_calc, u32buf, 4u); + expires_at = lox_get_u32(u32buf); + offset += 4u; + + err = lox_kv_set_at(db, key, value, val_len, expires_at); + if (err != LOX_OK) { + return err; + } + } + + if (offset != payload_offset + payload_len) { + return LOX_ERR_CORRUPT; + } + if (payload_crc_calc != payload_crc) { + return LOX_ERR_CORRUPT; + } + + return LOX_OK; +} + +static lox_err_t lox_load_ts_page(lox_t *db, uint32_t bank, uint32_t expected_generation) { + lox_core_t *core = lox_core(db); + uint8_t header[LOX_PAGE_HEADER_SIZE]; + uint32_t page_offset = lox_bank_ts_offset(core, bank); + uint32_t generation = 0u; + uint32_t payload_len = 0u; + uint32_t payload_crc = 0u; + uint32_t offset; + uint32_t payload_offset; + uint32_t stream_count; + uint32_t payload_crc_calc = 0xFFFFFFFFu; + uint32_t i; + lox_err_t err; + + err = lox_storage_read_bytes(core, page_offset, header, sizeof(header)); + if (err != LOX_OK) { + return err; + } + if (!lox_validate_page_header(header, + LOX_TS_PAGE_MAGIC, + core->layout.ts_size - LOX_PAGE_HEADER_SIZE, + &generation, + &payload_len, + &stream_count, + &payload_crc)) { + return LOX_ERR_CORRUPT; + } + if (generation != expected_generation) { + return LOX_ERR_CORRUPT; + } + + offset = page_offset + LOX_PAGE_HEADER_SIZE; + payload_offset = offset; + for (i = 0u; i < stream_count; ++i) { + uint8_t name_len = 0u; + char name[LOX_TS_STREAM_NAME_LEN]; + uint8_t type_byte = 0u; + uint8_t u32buf[4]; + uint32_t raw_size; + uint32_t sample_count; + uint32_t j; + + if (offset + 1u > payload_offset + payload_len) { + return LOX_ERR_CORRUPT; + } + err = lox_storage_read_bytes(core, offset, &name_len, 1u); + if (err != LOX_OK || name_len >= LOX_TS_STREAM_NAME_LEN) { + return LOX_ERR_CORRUPT; + } + payload_crc_calc = lox_crc32(payload_crc_calc, &name_len, 1u); + offset += 1u; + + if (offset + name_len + 1u + 8u > payload_offset + payload_len) { + return LOX_ERR_CORRUPT; + } + err = lox_storage_read_bytes(core, offset, name, name_len); + if (err != LOX_OK) { + return LOX_ERR_CORRUPT; + } + payload_crc_calc = lox_crc32(payload_crc_calc, (const uint8_t *)name, name_len); + name[name_len] = '\0'; + offset += name_len; + + err = lox_storage_read_bytes(core, offset, &type_byte, 1u); + if (err != LOX_OK) { + return LOX_ERR_CORRUPT; + } + payload_crc_calc = lox_crc32(payload_crc_calc, &type_byte, 1u); + offset += 1u; + + err = lox_storage_read_bytes(core, offset, u32buf, 4u); + if (err != LOX_OK) { + return LOX_ERR_CORRUPT; + } + payload_crc_calc = lox_crc32(payload_crc_calc, u32buf, 4u); + raw_size = lox_get_u32(u32buf); + offset += 4u; + if ((lox_ts_type_t)type_byte == LOX_TS_RAW && (raw_size == 0u || raw_size > LOX_TS_RAW_MAX)) { + return LOX_ERR_CORRUPT; + } + + err = lox_storage_read_bytes(core, offset, u32buf, 4u); + if (err != LOX_OK) { + return LOX_ERR_CORRUPT; + } + payload_crc_calc = lox_crc32(payload_crc_calc, u32buf, 4u); + sample_count = lox_get_u32(u32buf); + offset += 4u; + + err = lox_ts_register(db, name, (lox_ts_type_t)type_byte, raw_size); + if (err != LOX_OK && err != LOX_ERR_EXISTS) { + return err; + } + + for (j = 0u; j < sample_count; ++j) { + uint32_t ts_low; + uint32_t ts_high; + uint8_t value[LOX_TS_RAW_MAX]; + uint32_t val_len = ((lox_ts_type_t)type_byte == LOX_TS_RAW) ? raw_size : 4u; + uint64_t full_ts; + + err = lox_storage_read_bytes(core, offset, u32buf, 4u); + if (err != LOX_OK) { + return LOX_ERR_CORRUPT; + } + payload_crc_calc = lox_crc32(payload_crc_calc, u32buf, 4u); + ts_low = lox_get_u32(u32buf); + offset += 4u; + + err = lox_storage_read_bytes(core, offset, u32buf, 4u); + if (err != LOX_OK) { + return LOX_ERR_CORRUPT; + } + payload_crc_calc = lox_crc32(payload_crc_calc, u32buf, 4u); + ts_high = lox_get_u32(u32buf); + offset += 4u; + + err = lox_storage_read_bytes(core, offset, value, val_len); + if (err != LOX_OK) { + return LOX_ERR_CORRUPT; + } + payload_crc_calc = lox_crc32(payload_crc_calc, value, val_len); + offset += val_len; + + full_ts = ((uint64_t)ts_high << 32u) | ts_low; + err = lox_ts_insert(db, name, (lox_timestamp_t)full_ts, value); + if (err != LOX_OK) { + return err; + } + } + } + + if (offset != payload_offset + payload_len) { + return LOX_ERR_CORRUPT; + } + if (payload_crc_calc != payload_crc) { + return LOX_ERR_CORRUPT; + } + + return LOX_OK; +} + +static lox_err_t lox_load_rel_page(lox_t *db, uint32_t bank, uint32_t expected_generation) { + lox_core_t *core = lox_core(db); + uint8_t header[LOX_PAGE_HEADER_SIZE]; + uint32_t page_offset = lox_bank_rel_offset(core, bank); + uint32_t generation = 0u; + uint32_t payload_len = 0u; + uint32_t payload_crc = 0u; + uint32_t offset; + uint32_t payload_offset; + uint32_t table_count; + uint32_t payload_crc_calc = 0xFFFFFFFFu; + uint32_t i; + lox_err_t err; + + err = lox_storage_read_bytes(core, page_offset, header, sizeof(header)); + if (err != LOX_OK) { + return err; + } + if (!lox_validate_page_header(header, + LOX_REL_PAGE_MAGIC, + core->layout.rel_size - LOX_PAGE_HEADER_SIZE, + &generation, + &payload_len, + &table_count, + &payload_crc)) { + return LOX_ERR_CORRUPT; + } + if (generation != expected_generation) { + return LOX_ERR_CORRUPT; + } + + offset = page_offset + LOX_PAGE_HEADER_SIZE; + payload_offset = offset; + for (i = 0u; i < table_count; ++i) { + lox_schema_t schema; + lox_table_t *table = NULL; + uint8_t name_len = 0u; + char table_name[LOX_REL_TABLE_NAME_LEN]; + uint8_t u16buf[2]; + uint8_t u32buf[4]; + uint16_t schema_version; + uint32_t max_rows; + uint32_t col_count; + uint32_t row_count; + uint32_t j; + + if (offset + 1u > payload_offset + payload_len) { + return LOX_ERR_CORRUPT; + } + err = lox_storage_read_bytes(core, offset, &name_len, 1u); + if (err != LOX_OK || name_len >= LOX_REL_TABLE_NAME_LEN) { + return LOX_ERR_CORRUPT; + } + payload_crc_calc = lox_crc32(payload_crc_calc, &name_len, 1u); + offset += 1u; + + if (offset + name_len + 2u + 4u + 4u + 4u + 4u + 4u > payload_offset + payload_len) { + return LOX_ERR_CORRUPT; + } + err = lox_storage_read_bytes(core, offset, table_name, name_len); + if (err != LOX_OK) { + return LOX_ERR_CORRUPT; + } + payload_crc_calc = lox_crc32(payload_crc_calc, (const uint8_t *)table_name, name_len); + table_name[name_len] = '\0'; + offset += name_len; + + err = lox_storage_read_bytes(core, offset, u16buf, 2u); + if (err != LOX_OK) { + return LOX_ERR_CORRUPT; + } + payload_crc_calc = lox_crc32(payload_crc_calc, u16buf, 2u); + schema_version = lox_get_u16(u16buf); + offset += 2u; + + err = lox_storage_read_bytes(core, offset, u32buf, 4u); + if (err != LOX_OK) { + return LOX_ERR_CORRUPT; + } + payload_crc_calc = lox_crc32(payload_crc_calc, u32buf, 4u); + max_rows = lox_get_u32(u32buf); + offset += 4u; + + if (offset + 4u > payload_offset + payload_len) { + return LOX_ERR_CORRUPT; + } + err = lox_storage_read_bytes(core, offset, u32buf, 4u); + if (err != LOX_OK) { + return LOX_ERR_CORRUPT; + } + payload_crc_calc = lox_crc32(payload_crc_calc, u32buf, 4u); + offset += 4u; + + err = lox_storage_read_bytes(core, offset, u32buf, 4u); + if (err != LOX_OK) { + return LOX_ERR_CORRUPT; + } + payload_crc_calc = lox_crc32(payload_crc_calc, u32buf, 4u); + col_count = lox_get_u32(u32buf); + offset += 4u; + if (col_count > LOX_REL_MAX_COLS) { + return LOX_ERR_CORRUPT; + } + + if (offset + 4u > payload_offset + payload_len) { + return LOX_ERR_CORRUPT; + } + err = lox_storage_read_bytes(core, offset, u32buf, 4u); + if (err != LOX_OK) { + return LOX_ERR_CORRUPT; + } + payload_crc_calc = lox_crc32(payload_crc_calc, u32buf, 4u); + offset += 4u; + + err = lox_storage_read_bytes(core, offset, u32buf, 4u); + if (err != LOX_OK) { + return LOX_ERR_CORRUPT; + } + payload_crc_calc = lox_crc32(payload_crc_calc, u32buf, 4u); + row_count = lox_get_u32(u32buf); + offset += 4u; + if (row_count > max_rows) { + return LOX_ERR_CORRUPT; + } + + err = lox_schema_init(&schema, table_name, max_rows); + if (err != LOX_OK) { + return err; + } + schema.schema_version = schema_version; + + for (j = 0u; j < col_count; ++j) { + uint8_t col_name_len = 0u; + char col_name[LOX_REL_COL_NAME_LEN]; + uint8_t meta[2]; + uint32_t col_size; + + err = lox_storage_read_bytes(core, offset, &col_name_len, 1u); + if (err != LOX_OK || col_name_len >= LOX_REL_COL_NAME_LEN) { + return LOX_ERR_CORRUPT; + } + payload_crc_calc = lox_crc32(payload_crc_calc, &col_name_len, 1u); + offset += 1u; + + err = lox_storage_read_bytes(core, offset, col_name, col_name_len); + if (err != LOX_OK) { + return LOX_ERR_CORRUPT; + } + payload_crc_calc = lox_crc32(payload_crc_calc, (const uint8_t *)col_name, col_name_len); + col_name[col_name_len] = '\0'; + offset += col_name_len; + + err = lox_storage_read_bytes(core, offset, meta, 2u); + if (err != LOX_OK) { + return LOX_ERR_CORRUPT; + } + payload_crc_calc = lox_crc32(payload_crc_calc, meta, 2u); + offset += 2u; + + err = lox_storage_read_bytes(core, offset, u32buf, 4u); + if (err != LOX_OK) { + return LOX_ERR_CORRUPT; + } + payload_crc_calc = lox_crc32(payload_crc_calc, u32buf, 4u); + col_size = lox_get_u32(u32buf); + offset += 4u; + + err = lox_schema_add(&schema, + col_name, + (lox_col_type_t)meta[0], + col_size, + meta[1] != 0u); + if (err != LOX_OK) { + return err; + } + } + + err = lox_schema_seal(&schema); + if (err != LOX_OK) { + return err; + } + err = lox_table_create(db, &schema); + if (err != LOX_OK) { + return err; + } + err = lox_table_get(db, table_name, &table); + if (err != LOX_OK) { + return err; + } + + for (j = 0u; j < row_count; ++j) { + uint8_t row_buf[1024]; + if (table->row_size > sizeof(row_buf)) { + return LOX_ERR_SCHEMA; + } + err = lox_storage_read_bytes(core, offset, row_buf, table->row_size); + if (err != LOX_OK) { + return LOX_ERR_CORRUPT; + } + payload_crc_calc = lox_crc32(payload_crc_calc, row_buf, table->row_size); + offset += (uint32_t)table->row_size; + err = lox_rel_insert(db, table, row_buf); + if (err != LOX_OK) { + return err; + } + } + } + + if (offset != payload_offset + payload_len) { + return LOX_ERR_CORRUPT; + } + if (payload_crc_calc != payload_crc) { + return LOX_ERR_CORRUPT; + } + + return LOX_OK; +} + +static lox_err_t lox_apply_wal_entry(lox_t *db, + uint8_t engine, + uint8_t op, + const uint8_t *data, + uint16_t data_len) { + if (engine == LOX_WAL_ENGINE_KV) { + if (op == LOX_WAL_OP_SET_INSERT) { + uint8_t key_len; + char key[LOX_KV_KEY_MAX_LEN]; + uint32_t val_len; + uint32_t expires_at; + const uint8_t *val; + + if (data_len < 1u) { + return LOX_OK; + } + key_len = data[0]; + if ((uint32_t)key_len >= LOX_KV_KEY_MAX_LEN || data_len < (uint16_t)(1u + key_len + 8u)) { + return LOX_OK; + } + memcpy(key, data + 1u, key_len); + key[key_len] = '\0'; + val_len = lox_get_u32(data + 1u + key_len); + if (val_len > LOX_KV_VAL_MAX_LEN || + data_len < (uint16_t)(1u + key_len + 4u + val_len + 4u)) { + return LOX_OK; + } + val = data + 1u + key_len + 4u; + expires_at = lox_get_u32(val + val_len); + return lox_kv_set_at(db, key, val, val_len, expires_at); + } + if (op == LOX_WAL_OP_DEL) { + uint8_t key_len; + char key[LOX_KV_KEY_MAX_LEN]; + + if (data_len < 1u) { + return LOX_OK; + } + key_len = data[0]; + if ((uint32_t)key_len >= LOX_KV_KEY_MAX_LEN || data_len < (uint16_t)(1u + key_len)) { + return LOX_OK; + } + memcpy(key, data + 1u, key_len); + key[key_len] = '\0'; + (void)lox_kv_del(db, key); + return LOX_OK; + } + if (op == LOX_WAL_OP_CLEAR) { + return lox_kv_clear(db); + } + } else if (engine == LOX_WAL_ENGINE_TS) { + if (op == LOX_WAL_OP_SET_INSERT) { + uint8_t name_len; + char name[LOX_TS_STREAM_NAME_LEN]; + uint32_t ts_low; + uint32_t ts_high; + uint8_t type_byte; + uint32_t value_len; + uint64_t ts; + lox_err_t err; + + if (data_len < 1u) { + return LOX_OK; + } + name_len = data[0]; + if ((uint32_t)name_len >= LOX_TS_STREAM_NAME_LEN || data_len < (uint16_t)(1u + name_len + 9u)) { + return LOX_OK; + } + memcpy(name, data + 1u, name_len); + name[name_len] = '\0'; + ts_low = lox_get_u32(data + 1u + name_len); + ts_high = lox_get_u32(data + 1u + name_len + 4u); + type_byte = data[1u + name_len + 8u]; + value_len = (uint32_t)data_len - (1u + name_len + 9u); + err = lox_ts_register(db, name, (lox_ts_type_t)type_byte, value_len); + if (err != LOX_OK && err != LOX_ERR_EXISTS) { + return err; + } + ts = ((uint64_t)ts_high << 32u) | ts_low; + return lox_ts_insert(db, name, (lox_timestamp_t)ts, data + 1u + name_len + 9u); + } + if (op == LOX_WAL_OP_TS_REGISTER) { + uint8_t name_len; + char name[LOX_TS_STREAM_NAME_LEN]; + uint8_t type_byte; + uint32_t raw_size; + if (data_len < 1u) { + return LOX_OK; + } + name_len = data[0]; + if ((uint32_t)name_len >= LOX_TS_STREAM_NAME_LEN || data_len < (uint16_t)(1u + name_len + 5u)) { + return LOX_OK; + } + memcpy(name, data + 1u, name_len); + name[name_len] = '\0'; + type_byte = data[1u + name_len]; + raw_size = lox_get_u32(data + 1u + name_len + 1u); + return lox_ts_register(db, name, (lox_ts_type_t)type_byte, raw_size); + } + if (op == LOX_WAL_OP_CLEAR) { + uint8_t name_len; + char name[LOX_TS_STREAM_NAME_LEN]; + if (data_len < 1u) { + return LOX_OK; + } + name_len = data[0]; + if ((uint32_t)name_len >= LOX_TS_STREAM_NAME_LEN || data_len < (uint16_t)(1u + name_len)) { + return LOX_OK; + } + memcpy(name, data + 1u, name_len); + name[name_len] = '\0'; + return lox_ts_clear(db, name); + } + } else if (engine == LOX_WAL_ENGINE_REL) { + if (op == LOX_WAL_OP_SET_INSERT) { + uint8_t name_len; + char table_name[LOX_REL_TABLE_NAME_LEN]; + uint32_t row_size; + lox_table_t *table = NULL; + + if (data_len < 1u) { + return LOX_OK; + } + name_len = data[0]; + if ((uint32_t)name_len >= LOX_REL_TABLE_NAME_LEN || data_len < (uint16_t)(1u + name_len + 4u)) { + return LOX_OK; + } + memcpy(table_name, data + 1u, name_len); + table_name[name_len] = '\0'; + row_size = lox_get_u32(data + 1u + name_len); + if (data_len < (uint16_t)(1u + name_len + 4u + row_size)) { + return LOX_OK; + } + if (lox_table_get(db, table_name, &table) != LOX_OK || table->row_size != row_size) { + return LOX_OK; + } + return lox_rel_insert(db, table, data + 1u + name_len + 4u); + } + if (op == LOX_WAL_OP_DEL) { + uint8_t name_len; + char table_name[LOX_REL_TABLE_NAME_LEN]; + lox_table_t *table = NULL; + + if (data_len < 1u) { + return LOX_OK; + } + name_len = data[0]; + if ((uint32_t)name_len >= LOX_REL_TABLE_NAME_LEN || data_len < (uint16_t)(1u + name_len)) { + return LOX_OK; + } + memcpy(table_name, data + 1u, name_len); + table_name[name_len] = '\0'; + if (lox_table_get(db, table_name, &table) != LOX_OK || table->index_key_size == 0u) { + return LOX_OK; + } + (void)lox_rel_delete(db, table, data + 1u + name_len, NULL); + return LOX_OK; + } + if (op == LOX_WAL_OP_CLEAR) { + uint8_t name_len; + char table_name[LOX_REL_TABLE_NAME_LEN]; + lox_table_t *table = NULL; + + if (data_len < 1u) { + return LOX_OK; + } + name_len = data[0]; + if ((uint32_t)name_len >= LOX_REL_TABLE_NAME_LEN || data_len < (uint16_t)(1u + name_len)) { + return LOX_OK; + } + memcpy(table_name, data + 1u, name_len); + table_name[name_len] = '\0'; + if (lox_table_get(db, table_name, &table) != LOX_OK) { + return LOX_OK; + } + return lox_rel_clear(db, table); + } + if (op == LOX_WAL_OP_REL_TABLE_CREATE) { + lox_schema_t schema; + uint8_t name_len; + char table_name[LOX_REL_TABLE_NAME_LEN]; + uint16_t schema_version; + uint32_t max_rows; + uint32_t col_count; + uint16_t off; + uint32_t c; + + if (data_len < 1u) { + return LOX_OK; + } + name_len = data[0]; + if ((uint32_t)name_len >= LOX_REL_TABLE_NAME_LEN || data_len < (uint16_t)(1u + name_len + 10u)) { + return LOX_OK; + } + memcpy(table_name, data + 1u, name_len); + table_name[name_len] = '\0'; + schema_version = lox_get_u16(data + 1u + name_len); + max_rows = lox_get_u32(data + 1u + name_len + 2u); + col_count = lox_get_u32(data + 1u + name_len + 6u); + off = (uint16_t)(1u + name_len + 10u); + + if (lox_schema_init(&schema, table_name, max_rows) != LOX_OK) { + return LOX_ERR_SCHEMA; + } + schema.schema_version = schema_version; + for (c = 0u; c < col_count; ++c) { + uint8_t col_name_len; + char col_name[LOX_REL_COL_NAME_LEN]; + lox_col_type_t type; + bool is_index; + uint32_t col_size; + if (off + 1u > data_len) { + return LOX_OK; + } + col_name_len = data[off++]; + if ((uint32_t)col_name_len >= LOX_REL_COL_NAME_LEN || off + col_name_len + 6u > data_len) { + return LOX_OK; + } + memcpy(col_name, data + off, col_name_len); + col_name[col_name_len] = '\0'; + off = (uint16_t)(off + col_name_len); + type = (lox_col_type_t)data[off++]; + is_index = data[off++] != 0u; + col_size = lox_get_u32(data + off); + off = (uint16_t)(off + 4u); + if (lox_schema_add(&schema, col_name, type, col_size, is_index) != LOX_OK) { + return LOX_ERR_SCHEMA; + } + } + if (lox_schema_seal(&schema) != LOX_OK) { + return LOX_ERR_SCHEMA; + } + return lox_table_create(db, &schema); + } + } + + return LOX_OK; +} + +static lox_err_t lox_replay_wal(lox_t *db, bool *out_had_entries, bool *out_header_reset) { + lox_core_t *core = lox_core(db); + uint8_t header[32]; + uint32_t stored_crc; + uint32_t entry_count; + uint32_t block_seq; + uint32_t offset = core->layout.wal_offset + LOX_WAL_HEADER_SIZE; + uint32_t i; + uint32_t replayed_count = 0u; + uint8_t txn_ops[LOX_TXN_STAGE_KEYS]; + uint16_t txn_lens[LOX_TXN_STAGE_KEYS]; + uint8_t txn_payloads[LOX_TXN_STAGE_KEYS][256]; + uint32_t txn_count = 0u; + lox_err_t err; + + *out_had_entries = false; + *out_header_reset = false; + + err = lox_storage_read_bytes(core, core->layout.wal_offset, header, sizeof(header)); + if (err != LOX_OK) { + return err; + } + + if (lox_get_u32(header + 0u) != LOX_WAL_MAGIC) { + LOX_LOG("ERROR", "WAL header corrupt: resetting WAL"); + *out_header_reset = true; + return LOX_OK; + } + + stored_crc = lox_get_u32(header + 16u); + if (LOX_CRC32(header, 16u) != stored_crc) { + LOX_LOG("ERROR", "WAL header corrupt: resetting WAL"); + *out_header_reset = true; + return LOX_OK; + } + + entry_count = lox_get_u32(header + 8u); + block_seq = lox_get_u32(header + 12u); + core->wal_sequence = block_seq; + + if (entry_count == 0u) { + core->wal_used = LOX_WAL_HEADER_SIZE; + return LOX_OK; + } + + *out_had_entries = true; + /* Recovery invariant: + * - TXN_KV WAL entries are staged only during replay. + * - Staged txn entries become visible only after durable TXN_COMMIT marker. + * - Corrupt/truncated WAL tail is ignored from first invalid entry onward. + */ + core->wal_replaying = true; + for (i = 0u; i < entry_count; ++i) { + uint8_t entry_header[16]; + uint8_t payload[1536]; + uint32_t entry_crc; + uint16_t data_len; + uint32_t aligned_len; + uint32_t crc; + + err = lox_storage_read_bytes(core, offset, entry_header, sizeof(entry_header)); + if (err != LOX_OK || lox_get_u32(entry_header + 0u) != LOX_WAL_ENTRY_MAGIC) { + break; + } + + data_len = lox_get_u16(entry_header + 10u); + aligned_len = lox_align_u32(data_len, 4u); + if (data_len > sizeof(payload) || offset + 16u + aligned_len > core->layout.wal_offset + core->layout.wal_size) { + break; + } + + err = lox_storage_read_bytes(core, offset + 16u, payload, aligned_len); + if (err != LOX_OK) { + break; + } + + entry_crc = lox_get_u32(entry_header + 12u); + crc = LOX_CRC32(entry_header, 12u); + crc = lox_crc32(crc, payload, data_len); + if (crc != entry_crc) { + LOX_LOG("ERROR", + "WAL corrupt entry at seq=%u: CRC mismatch, stopping replay", + (unsigned)lox_get_u32(entry_header + 4u)); + break; + } + + if (entry_header[8] == LOX_WAL_ENGINE_TXN_KV && + (entry_header[9] == LOX_WAL_OP_SET_INSERT || entry_header[9] == LOX_WAL_OP_DEL)) { + if (txn_count < LOX_TXN_STAGE_KEYS && data_len <= sizeof(txn_payloads[0])) { + txn_ops[txn_count] = entry_header[9]; + txn_lens[txn_count] = data_len; + memcpy(txn_payloads[txn_count], payload, data_len); + txn_count++; + } else { + txn_count = 0u; + } + offset += 16u + aligned_len; + replayed_count++; + continue; + } + if (entry_header[8] == LOX_WAL_ENGINE_META && entry_header[9] == LOX_WAL_OP_TXN_COMMIT) { + uint32_t t; + for (t = 0u; t < txn_count; ++t) { + err = lox_apply_wal_entry(db, LOX_WAL_ENGINE_KV, txn_ops[t], txn_payloads[t], txn_lens[t]); + if (err != LOX_OK) { + core->wal_replaying = false; + return err; + } + } + txn_count = 0u; + offset += 16u + aligned_len; + replayed_count++; + continue; + } + if (txn_count != 0u) { + txn_count = 0u; + } + + err = lox_apply_wal_entry(db, entry_header[8], entry_header[9], payload, data_len); + if (err != LOX_OK) { + core->wal_replaying = false; + return err; + } + + offset += 16u + aligned_len; + replayed_count++; + } + core->wal_replaying = false; + core->wal_used = offset - core->layout.wal_offset; + LOX_LOG("INFO", + "WAL recovery complete: replayed %u entries", + (unsigned)replayed_count); + return LOX_OK; +} + +static lox_err_t lox_crc_storage_region(lox_core_t *core, uint32_t offset, uint32_t len, uint32_t *out_crc) { + uint8_t chunk[128]; + uint32_t crc = 0xFFFFFFFFu; + uint32_t pos = 0u; + lox_err_t err; + + while (pos < len) { + uint32_t take = len - pos; + if (take > sizeof(chunk)) { + take = sizeof(chunk); + } + err = lox_storage_read_bytes(core, offset + pos, chunk, take); + if (err != LOX_OK) { + return err; + } + crc = lox_crc32(crc, chunk, take); + pos += take; + } + + *out_crc = crc; + return LOX_OK; +} + +static lox_err_t lox_validate_bank_pages(lox_core_t *core, uint32_t bank, uint32_t *out_generation) { + uint8_t header[LOX_PAGE_HEADER_SIZE]; + uint32_t gen = 0u; + uint32_t payload_len = 0u; + uint32_t entry_count = 0u; + uint32_t payload_crc = 0u; + uint32_t calc_crc = 0u; + lox_err_t err; + + err = lox_storage_read_bytes(core, lox_bank_kv_offset(core, bank), header, sizeof(header)); + if (err != LOX_OK) { + return err; + } + if (!lox_validate_page_header(header, + LOX_KV_PAGE_MAGIC, + core->layout.kv_size - LOX_PAGE_HEADER_SIZE, + &gen, + &payload_len, + &entry_count, + &payload_crc)) { + return LOX_ERR_CORRUPT; + } + (void)entry_count; + err = lox_crc_storage_region(core, lox_bank_kv_offset(core, bank) + LOX_PAGE_HEADER_SIZE, payload_len, &calc_crc); + if (err != LOX_OK || calc_crc != payload_crc) { + return LOX_ERR_CORRUPT; + } + + err = lox_storage_read_bytes(core, lox_bank_ts_offset(core, bank), header, sizeof(header)); + if (err != LOX_OK) { + return err; + } + { + uint32_t gen2 = 0u; + if (!lox_validate_page_header(header, + LOX_TS_PAGE_MAGIC, + core->layout.ts_size - LOX_PAGE_HEADER_SIZE, + &gen2, + &payload_len, + &entry_count, + &payload_crc)) { + return LOX_ERR_CORRUPT; + } + if (gen2 != gen) { + return LOX_ERR_CORRUPT; + } + } + err = lox_crc_storage_region(core, lox_bank_ts_offset(core, bank) + LOX_PAGE_HEADER_SIZE, payload_len, &calc_crc); + if (err != LOX_OK || calc_crc != payload_crc) { + return LOX_ERR_CORRUPT; + } + + err = lox_storage_read_bytes(core, lox_bank_rel_offset(core, bank), header, sizeof(header)); + if (err != LOX_OK) { + return err; + } + { + uint32_t gen3 = 0u; + if (!lox_validate_page_header(header, + LOX_REL_PAGE_MAGIC, + core->layout.rel_size - LOX_PAGE_HEADER_SIZE, + &gen3, + &payload_len, + &entry_count, + &payload_crc)) { + return LOX_ERR_CORRUPT; + } + if (gen3 != gen) { + return LOX_ERR_CORRUPT; + } + } + err = lox_crc_storage_region(core, lox_bank_rel_offset(core, bank) + LOX_PAGE_HEADER_SIZE, payload_len, &calc_crc); + if (err != LOX_OK || calc_crc != payload_crc) { + return LOX_ERR_CORRUPT; + } + + *out_generation = gen; + return LOX_OK; +} + +lox_err_t lox_storage_bootstrap(lox_t *db) { + lox_core_t *core = lox_core(db); + lox_err_t err; + bool had_entries = false; + bool reset_header = false; + bool super_a_valid = false; + bool super_b_valid = false; + uint8_t super_a[LOX_SUPERBLOCK_SIZE]; + uint8_t super_b[LOX_SUPERBLOCK_SIZE]; + uint32_t super_a_gen = 0u; + uint32_t super_b_gen = 0u; + uint32_t super_a_bank = 0u; + uint32_t super_b_bank = 0u; + uint32_t fallback_gen_a = 0u; + uint32_t fallback_gen_b = 0u; + bool fallback_a_valid = false; + bool fallback_b_valid = false; + uint32_t selected_bank = 0u; + uint32_t selected_gen = 0u; + bool have_selected = false; + uint32_t erase_size; + + memset(&core->layout, 0, sizeof(core->layout)); + core->wal_sequence = 0u; + core->wal_entry_count = 0u; + core->wal_used = LOX_WAL_HEADER_SIZE; + core->last_recovery_status = LOX_OK; + + if (!lox_storage_ready(core)) { + return LOX_OK; + } + if (core->storage->erase_size == 0u) { + LOX_LOG("ERROR", "Storage contract violation: erase_size must be > 0"); + return LOX_ERR_INVALID; + } + if (core->storage->write_size == 0u) { + LOX_LOG("ERROR", "Storage contract violation: write_size must be 1 (got 0)"); + return LOX_ERR_INVALID; + } + if (core->storage->write_size != 1u) { + LOX_LOG("ERROR", + "Storage contract violation: write_size=%u unsupported (only 1 is supported in this release)", + (unsigned)core->storage->write_size); + return LOX_ERR_INVALID; + } + + { + uint32_t wal_target; + uint32_t wal_min; + uint32_t fixed_bytes; + uint32_t need_without_wal; + uint32_t max_wal; + uint32_t max_wal_aligned; + + erase_size = core->storage->erase_size; + core->layout.wal_offset = 0u; + core->layout.super_size = erase_size; + wal_target = erase_size * 8u; + wal_min = erase_size * 2u; + core->layout.kv_size = + lox_align_u32(lox_kv_snapshot_payload_max(core) + LOX_PAGE_HEADER_SIZE, erase_size); + core->layout.ts_size = lox_align_u32((uint32_t)core->ts_arena.capacity + LOX_PAGE_HEADER_SIZE, erase_size); + core->layout.rel_size = lox_align_u32((uint32_t)core->rel_arena.capacity + LOX_PAGE_HEADER_SIZE, erase_size); + core->layout.bank_size = core->layout.kv_size + core->layout.ts_size + core->layout.rel_size; + + fixed_bytes = core->layout.super_size * 2u; + need_without_wal = fixed_bytes + (core->layout.bank_size * 2u); + if (core->storage->capacity < need_without_wal + wal_min) { + return LOX_ERR_STORAGE; + } + + max_wal = core->storage->capacity - need_without_wal; + max_wal_aligned = (max_wal / erase_size) * erase_size; + if (max_wal_aligned < wal_min) { + return LOX_ERR_STORAGE; + } + + core->layout.wal_size = wal_target; + if (core->layout.wal_size > max_wal_aligned) { + core->layout.wal_size = max_wal_aligned; + } + if (core->layout.wal_size < wal_min) { + core->layout.wal_size = wal_min; + } + } + core->layout.super_a_offset = core->layout.wal_offset + core->layout.wal_size; + core->layout.super_b_offset = core->layout.super_a_offset + core->layout.super_size; + core->layout.bank_a_offset = core->layout.super_b_offset + core->layout.super_size; + core->layout.bank_b_offset = core->layout.bank_a_offset + core->layout.bank_size; + core->layout.total_size = core->layout.bank_b_offset + core->layout.bank_size; + + if (core->storage->capacity < core->layout.total_size) { + return LOX_ERR_STORAGE; + } + + err = lox_storage_read_bytes(core, core->layout.super_a_offset, super_a, sizeof(super_a)); + if (err != LOX_OK) { + return err; + } + err = lox_storage_read_bytes(core, core->layout.super_b_offset, super_b, sizeof(super_b)); + if (err != LOX_OK) { + return err; + } + + super_a_valid = lox_validate_superblock(super_a, &super_a_gen, &super_a_bank); + super_b_valid = lox_validate_superblock(super_b, &super_b_gen, &super_b_bank); + + /* Boot selection invariant: + * 1) prefer newest valid superblock; + * 2) if no valid superblock exists, fallback to fully valid bank scan; + * 3) selected bank pages must all pass header+payload CRC validation. + */ + if (super_a_valid || super_b_valid) { + if (super_a_valid && (!super_b_valid || super_a_gen >= super_b_gen)) { + selected_bank = super_a_bank; + selected_gen = super_a_gen; + } else { + selected_bank = super_b_bank; + selected_gen = super_b_gen; + } + have_selected = true; + } else { + if (lox_validate_bank_pages(core, 0u, &fallback_gen_a) == LOX_OK) { + fallback_a_valid = true; + } + if (lox_validate_bank_pages(core, 1u, &fallback_gen_b) == LOX_OK) { + fallback_b_valid = true; + } + if (fallback_a_valid || fallback_b_valid) { + if (fallback_a_valid && (!fallback_b_valid || fallback_gen_a >= fallback_gen_b)) { + selected_bank = 0u; + selected_gen = fallback_gen_a; + } else { + selected_bank = 1u; + selected_gen = fallback_gen_b; + } + have_selected = true; + } + } + + if (have_selected) { + core->reopen_count++; + core->storage_loading = true; + err = lox_load_kv_page(db, selected_bank, selected_gen); + if (err == LOX_OK) { + err = lox_load_ts_page(db, selected_bank, selected_gen); + } + if (err == LOX_OK) { + err = lox_load_rel_page(db, selected_bank, selected_gen); + } + core->storage_loading = false; + if (err != LOX_OK) { + return err; + } + core->layout.active_bank = selected_bank; + core->layout.active_generation = selected_gen; + } else { + uint8_t probe[16]; + bool virgin = true; + err = lox_storage_read_bytes(core, core->layout.super_a_offset, probe, sizeof(probe)); + if (err != LOX_OK) { + return err; + } + for (uint32_t k = 0u; k < sizeof(probe); ++k) { + if (probe[k] != 0xFFu) { + virgin = false; + break; + } + } + if (virgin) { + err = lox_storage_read_bytes(core, core->layout.super_b_offset, probe, sizeof(probe)); + if (err != LOX_OK) { + return err; + } + for (uint32_t k = 0u; k < sizeof(probe); ++k) { + if (probe[k] != 0xFFu) { + virgin = false; + break; + } + } + } + if (!virgin) { + return LOX_ERR_CORRUPT; + } + /* Cold start: initialize first durable snapshot bank/superblock. */ + core->layout.active_bank = 0u; + core->layout.active_generation = 1u; + err = lox_write_snapshot_bank(core, 0u, core->layout.active_generation); + if (err != LOX_OK) { + return err; + } + err = lox_storage_sync_core(core); + if (err != LOX_OK) { + return err; + } + err = lox_write_superblock(core, core->layout.active_generation, 0u); + if (err != LOX_OK) { + return err; + } + err = lox_storage_sync_core(core); + if (err != LOX_OK) { + return err; + } + } + + if (!core->wal_enabled) { + return LOX_OK; + } + + err = lox_replay_wal(db, &had_entries, &reset_header); + if (err != LOX_OK) { + core->last_recovery_status = err; + lox_record_error(core, err); + return err; + } + + if (had_entries || reset_header) { + err = lox_storage_flush(db); + core->last_recovery_status = err; + if (err == LOX_OK) { + core->recovery_count++; + } else { + lox_record_error(core, err); + } + return err; + } + + err = lox_reset_wal(core, core->wal_sequence); + core->last_recovery_status = err; + if (err != LOX_OK) { + lox_record_error(core, err); + } + return err; +} + +static lox_err_t lox_append_wal_entry(lox_t *db, + uint8_t engine, + uint8_t op, + const uint8_t *payload, + uint16_t payload_len) { + lox_core_t *core = lox_core(db); + uint32_t aligned_len = lox_align_u32(payload_len, 4u); + uint32_t entry_len = 16u + aligned_len; + uint32_t offset = 0u; + uint32_t pad_len = aligned_len - payload_len; + uint8_t header[16]; + uint8_t pad[4] = { 0u, 0u, 0u, 0u }; + uint32_t crc; + lox_err_t err; + + if (core->wal_used + entry_len > core->layout.wal_size) { + err = lox_storage_flush(db); + if (err != LOX_OK) { + return err; + } + } + + if (core->wal_used + entry_len > core->layout.wal_size) { + return LOX_ERR_STORAGE; + } + + memset(header, 0, sizeof(header)); + lox_put_u32(header + 0u, LOX_WAL_ENTRY_MAGIC); + lox_put_u32(header + 4u, core->wal_entry_count + 1u); + header[8] = engine; + header[9] = op; + lox_put_u16(header + 10u, payload_len); + crc = LOX_CRC32(header, 12u); + if (payload_len != 0u) { + crc = lox_crc32(crc, payload, payload_len); + } + lox_put_u32(header + 12u, crc); + + offset = core->layout.wal_offset + core->wal_used; + err = lox_storage_write_bytes(core, offset, header, sizeof(header)); + if (err != LOX_OK) { + return err; + } + offset += (uint32_t)sizeof(header); + + if (payload_len != 0u) { + err = lox_storage_write_bytes(core, offset, payload, payload_len); + if (err != LOX_OK) { + return err; + } + offset += payload_len; + } + + if (pad_len != 0u) { + err = lox_storage_write_bytes(core, offset, pad, pad_len); + if (err != LOX_OK) { + return err; + } + } + + core->wal_used += entry_len; + core->wal_entry_count++; + err = lox_write_wal_header(core); + if (err != LOX_OK) { + return err; + } + if (core->wal_sync_mode == LOX_WAL_SYNC_ALWAYS) { + err = lox_storage_sync_core(core); + if (err != LOX_OK) { + return err; + } + } + + return LOX_OK; +} + +static lox_err_t lox_compact_nolock(lox_t *db) { + lox_core_t *core; + lox_err_t err; + uint32_t next_bank; + uint32_t next_generation; + + if (db == NULL) { + return LOX_ERR_INVALID; + } + + core = lox_core(db); + if (core->magic != LOX_MAGIC) { + return LOX_ERR_INVALID; + } + if (!lox_storage_ready(core)) { + return LOX_OK; + } + /* Durability invariant: + * - active bank is never erased in-place. + * - compact writes full snapshot into inactive bank, syncs, then switches superblock. + */ + next_bank = (core->layout.active_bank == 0u) ? 1u : 0u; + next_generation = core->layout.active_generation + 1u; + err = lox_write_snapshot_bank(core, next_bank, next_generation); + if (err != LOX_OK) { + return err; + } + err = lox_storage_sync_core(core); + if (err != LOX_OK) { + return err; + } + err = lox_write_superblock(core, next_generation, next_bank); + if (err != LOX_OK) { + return err; + } + err = lox_storage_sync_core(core); + if (err != LOX_OK) { + return err; + } + core->layout.active_bank = next_bank; + core->layout.active_generation = next_generation; + if (core->wal_enabled) { + err = lox_reset_wal(core, core->wal_sequence + 1u); + if (err != LOX_OK) { + return err; + } + core->compact_count++; + return LOX_OK; + } + core->compact_count++; + return LOX_OK; +} + +lox_err_t lox_compact(lox_t *db) { + lox_err_t rc; + lox_core_t *core; + + if (db == NULL) { + return LOX_ERR_INVALID; + } + + LOX_LOCK(db); + core = lox_core(db); + if (core->magic != LOX_MAGIC) { + LOX_UNLOCK(db); + return LOX_ERR_INVALID; + } + rc = lox_compact_nolock(db); + lox_record_error(core, rc); + LOX_UNLOCK(db); + return rc; +} + +lox_err_t lox_storage_flush(lox_t *db) { + lox_core_t *core = lox_core(db); + lox_err_t rc; + + /* Ordering invariant: + * - flush never runs during bootstrap load or WAL replay. + * - it must not serialize transient replay/load state. + */ + if (!lox_storage_ready(core) || core->storage_loading || core->wal_replaying) { + return LOX_OK; + } + + if (core->wal_enabled) { + LOX_LOG("INFO", + "WAL compaction triggered: entry_count=%u", + (unsigned)core->wal_entry_count); + } + + rc = lox_compact_nolock(db); + lox_record_error(core, rc); + return rc; +} + +lox_err_t lox_persist_kv_set(lox_t *db, const char *key, const void *val, size_t len, uint32_t expires_at) { + lox_core_t *core = lox_core(db); + uint8_t payload[256]; + size_t key_len; + + if (!lox_storage_ready(core) || core->storage_loading || core->wal_replaying) { + return LOX_OK; + } + if (!core->wal_enabled) { + return lox_storage_flush(db); + } + + key_len = strlen(key); + payload[0] = (uint8_t)key_len; + memcpy(payload + 1u, key, key_len); + lox_put_u32(payload + 1u + key_len, (uint32_t)len); + memcpy(payload + 1u + key_len + 4u, val, len); + lox_put_u32(payload + 1u + key_len + 4u + len, expires_at); + return lox_append_wal_entry(db, + LOX_WAL_ENGINE_KV, + LOX_WAL_OP_SET_INSERT, + payload, + (uint16_t)(1u + key_len + 4u + len + 4u)); +} + +lox_err_t lox_persist_kv_set_txn(lox_t *db, const char *key, const void *val, size_t len, uint32_t expires_at) { + lox_core_t *core = lox_core(db); + uint8_t payload[256]; + size_t key_len; + + if (!lox_storage_ready(core) || core->storage_loading || core->wal_replaying) { + return LOX_OK; + } + if (!core->wal_enabled) { + return lox_storage_flush(db); + } + + key_len = strlen(key); + payload[0] = (uint8_t)key_len; + memcpy(payload + 1u, key, key_len); + lox_put_u32(payload + 1u + key_len, (uint32_t)len); + memcpy(payload + 1u + key_len + 4u, val, len); + lox_put_u32(payload + 1u + key_len + 4u + len, expires_at); + return lox_append_wal_entry(db, + LOX_WAL_ENGINE_TXN_KV, + LOX_WAL_OP_SET_INSERT, + payload, + (uint16_t)(1u + key_len + 4u + len + 4u)); +} + +lox_err_t lox_persist_kv_del(lox_t *db, const char *key) { + lox_core_t *core = lox_core(db); + uint8_t payload[LOX_KV_KEY_MAX_LEN]; + size_t key_len; + + if (!lox_storage_ready(core) || core->storage_loading || core->wal_replaying) { + return LOX_OK; + } + if (!core->wal_enabled) { + return lox_storage_flush(db); + } + + key_len = strlen(key); + payload[0] = (uint8_t)key_len; + memcpy(payload + 1u, key, key_len); + return lox_append_wal_entry(db, + LOX_WAL_ENGINE_KV, + LOX_WAL_OP_DEL, + payload, + (uint16_t)(1u + key_len)); +} + +lox_err_t lox_persist_kv_del_txn(lox_t *db, const char *key) { + lox_core_t *core = lox_core(db); + uint8_t payload[LOX_KV_KEY_MAX_LEN]; + size_t key_len; + + if (!lox_storage_ready(core) || core->storage_loading || core->wal_replaying) { + return LOX_OK; + } + if (!core->wal_enabled) { + return lox_storage_flush(db); + } + + key_len = strlen(key); + payload[0] = (uint8_t)key_len; + memcpy(payload + 1u, key, key_len); + return lox_append_wal_entry(db, + LOX_WAL_ENGINE_TXN_KV, + LOX_WAL_OP_DEL, + payload, + (uint16_t)(1u + key_len)); +} + +lox_err_t lox_persist_kv_clear(lox_t *db) { + lox_core_t *core = lox_core(db); + + if (!lox_storage_ready(core) || core->storage_loading || core->wal_replaying) { + return LOX_OK; + } + if (!core->wal_enabled) { + return lox_storage_flush(db); + } + + return lox_append_wal_entry(db, LOX_WAL_ENGINE_KV, LOX_WAL_OP_CLEAR, NULL, 0u); +} + +lox_err_t lox_persist_txn_commit(lox_t *db) { + lox_core_t *core = lox_core(db); + + if (!lox_storage_ready(core) || core->storage_loading || core->wal_replaying) { + return LOX_OK; + } + if (!core->wal_enabled) { + return lox_storage_flush(db); + } + + return lox_append_wal_entry(db, LOX_WAL_ENGINE_META, LOX_WAL_OP_TXN_COMMIT, NULL, 0u); +} + +lox_err_t lox_persist_ts_insert(lox_t *db, const char *name, lox_timestamp_t ts, const void *val, size_t val_len) { + lox_core_t *core = lox_core(db); + uint8_t payload[256]; + size_t idx; + size_t name_len; + uint64_t full_ts = (uint64_t)ts; + lox_ts_stream_t *stream = NULL; + + if (!lox_storage_ready(core) || core->storage_loading || core->wal_replaying) { + return LOX_OK; + } + if (!core->wal_enabled) { + return lox_storage_flush(db); + } + + for (idx = 0u; idx < LOX_TS_MAX_STREAMS; ++idx) { + if (core->ts.streams[idx].registered && strcmp(core->ts.streams[idx].name, name) == 0) { + stream = &core->ts.streams[idx]; + break; + } + } + if (stream == NULL) { + return LOX_ERR_NOT_FOUND; + } + + name_len = strlen(name); + payload[0] = (uint8_t)name_len; + memcpy(payload + 1u, name, name_len); + lox_put_u32(payload + 1u + name_len, (uint32_t)(full_ts & 0xFFFFFFFFu)); + lox_put_u32(payload + 1u + name_len + 4u, (uint32_t)(full_ts >> 32u)); + payload[1u + name_len + 8u] = (uint8_t)stream->type; + memcpy(payload + 1u + name_len + 9u, val, val_len); + return lox_append_wal_entry(db, + LOX_WAL_ENGINE_TS, + LOX_WAL_OP_SET_INSERT, + payload, + (uint16_t)(1u + name_len + 9u + val_len)); +} + +lox_err_t lox_persist_ts_register(lox_t *db, const char *name, lox_ts_type_t type, size_t raw_size) { + lox_core_t *core = lox_core(db); + uint8_t payload[64]; + size_t name_len; + + if (!lox_storage_ready(core) || core->storage_loading || core->wal_replaying) { + return LOX_OK; + } + if (!core->wal_enabled) { + return lox_storage_flush(db); + } + + name_len = strlen(name); + if (name_len + 6u > sizeof(payload)) { + return LOX_ERR_INVALID; + } + payload[0] = (uint8_t)name_len; + memcpy(payload + 1u, name, name_len); + payload[1u + name_len] = (uint8_t)type; + lox_put_u32(payload + 1u + name_len + 1u, (uint32_t)raw_size); + return lox_append_wal_entry(db, + LOX_WAL_ENGINE_TS, + LOX_WAL_OP_TS_REGISTER, + payload, + (uint16_t)(1u + name_len + 1u + 4u)); +} + +lox_err_t lox_persist_ts_clear(lox_t *db, const char *name) { + lox_core_t *core = lox_core(db); + uint8_t payload[LOX_TS_STREAM_NAME_LEN]; + size_t name_len; + + if (!lox_storage_ready(core) || core->storage_loading || core->wal_replaying) { + return LOX_OK; + } + if (!core->wal_enabled) { + return lox_storage_flush(db); + } + + name_len = strlen(name); + if (name_len + 1u > sizeof(payload)) { + return LOX_ERR_INVALID; + } + payload[0] = (uint8_t)name_len; + memcpy(payload + 1u, name, name_len); + return lox_append_wal_entry(db, + LOX_WAL_ENGINE_TS, + LOX_WAL_OP_CLEAR, + payload, + (uint16_t)(1u + name_len)); +} + +lox_err_t lox_persist_rel_insert(lox_t *db, const lox_table_t *table, const void *row_buf) { + lox_core_t *core = lox_core(db); + uint8_t payload[1536]; + size_t name_len; + + if (!lox_storage_ready(core) || core->storage_loading || core->wal_replaying) { + return LOX_OK; + } + if (!core->wal_enabled) { + return lox_storage_flush(db); + } + if (table->row_size + LOX_REL_TABLE_NAME_LEN + 5u > sizeof(payload)) { + return LOX_ERR_STORAGE; + } + + name_len = strlen(table->name); + payload[0] = (uint8_t)name_len; + memcpy(payload + 1u, table->name, name_len); + lox_put_u32(payload + 1u + name_len, (uint32_t)table->row_size); + memcpy(payload + 1u + name_len + 4u, row_buf, table->row_size); + return lox_append_wal_entry(db, + LOX_WAL_ENGINE_REL, + LOX_WAL_OP_SET_INSERT, + payload, + (uint16_t)(1u + name_len + 4u + table->row_size)); +} + +lox_err_t lox_persist_rel_delete(lox_t *db, const lox_table_t *table, const void *search_val) { + lox_core_t *core = lox_core(db); + uint8_t payload[64]; + size_t name_len; + + if (!lox_storage_ready(core) || core->storage_loading || core->wal_replaying) { + return LOX_OK; + } + if (!core->wal_enabled) { + return lox_storage_flush(db); + } + + name_len = strlen(table->name); + payload[0] = (uint8_t)name_len; + memcpy(payload + 1u, table->name, name_len); + memcpy(payload + 1u + name_len, search_val, table->index_key_size); + return lox_append_wal_entry(db, + LOX_WAL_ENGINE_REL, + LOX_WAL_OP_DEL, + payload, + (uint16_t)(1u + name_len + table->index_key_size)); +} + +lox_err_t lox_persist_rel_table_create(lox_t *db, const lox_schema_t *schema) { + lox_core_t *core = lox_core(db); + const lox_schema_impl_t *impl; + uint8_t payload[512]; + uint32_t i; + uint16_t off = 0u; + size_t name_len; + + if (!lox_storage_ready(core) || core->storage_loading || core->wal_replaying) { + return LOX_OK; + } + if (!core->wal_enabled) { + return lox_storage_flush(db); + } + if (schema == NULL) { + return LOX_ERR_INVALID; + } + + impl = (const lox_schema_impl_t *)&schema->_opaque[0]; + if (!impl->sealed) { + return LOX_ERR_INVALID; + } + + name_len = strlen(impl->name); + if (name_len >= LOX_REL_TABLE_NAME_LEN) { + return LOX_ERR_INVALID; + } + if (1u + name_len + 2u + 4u + 4u > sizeof(payload)) { + return LOX_ERR_STORAGE; + } + + payload[off++] = (uint8_t)name_len; + memcpy(payload + off, impl->name, name_len); + off = (uint16_t)(off + name_len); + lox_put_u16(payload + off, impl->schema_version); + off = (uint16_t)(off + 2u); + lox_put_u32(payload + off, impl->max_rows); + off = (uint16_t)(off + 4u); + lox_put_u32(payload + off, impl->col_count); + off = (uint16_t)(off + 4u); + + for (i = 0u; i < impl->col_count; ++i) { + size_t col_name_len = strlen(impl->cols[i].name); + if (col_name_len >= LOX_REL_COL_NAME_LEN) { + return LOX_ERR_SCHEMA; + } + if ((size_t)off + 1u + col_name_len + 1u + 1u + 4u > sizeof(payload)) { + return LOX_ERR_STORAGE; + } + payload[off++] = (uint8_t)col_name_len; + memcpy(payload + off, impl->cols[i].name, col_name_len); + off = (uint16_t)(off + col_name_len); + payload[off++] = (uint8_t)impl->cols[i].type; + payload[off++] = (uint8_t)(impl->cols[i].is_index ? 1u : 0u); + lox_put_u32(payload + off, (uint32_t)impl->cols[i].size); + off = (uint16_t)(off + 4u); + } + + return lox_append_wal_entry(db, + LOX_WAL_ENGINE_REL, + LOX_WAL_OP_REL_TABLE_CREATE, + payload, + off); +} + +lox_err_t lox_persist_rel_clear(lox_t *db, const lox_table_t *table) { + lox_core_t *core = lox_core(db); + uint8_t payload[LOX_REL_TABLE_NAME_LEN]; + size_t name_len; + + if (!lox_storage_ready(core) || core->storage_loading || core->wal_replaying) { + return LOX_OK; + } + if (!core->wal_enabled) { + return lox_storage_flush(db); + } + if (table == NULL) { + return LOX_ERR_INVALID; + } + + name_len = strlen(table->name); + if (name_len + 1u > sizeof(payload)) { + return LOX_ERR_INVALID; + } + payload[0] = (uint8_t)name_len; + memcpy(payload + 1u, table->name, name_len); + return lox_append_wal_entry(db, + LOX_WAL_ENGINE_REL, + LOX_WAL_OP_CLEAR, + payload, + (uint16_t)(1u + name_len)); +} diff --git a/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/src/loxdb.c b/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/src/loxdb.c new file mode 100644 index 0000000..b31e62b --- /dev/null +++ b/bench/loxdb_esp32_s3_bench_head/lox_esp32_s3_bench/src/loxdb.c @@ -0,0 +1,1033 @@ +// SPDX-License-Identifier: MIT +#include "lox_internal.h" +#include "lox_lock.h" + +#include "lox_arena.h" + +#include +#include + +LOX_STATIC_ASSERT(core_ram_pct_sum, (LOX_RAM_KV_PCT + LOX_RAM_TS_PCT + LOX_RAM_REL_PCT) == 100u); +LOX_STATIC_ASSERT(core_ram_kb_min, LOX_RAM_KB >= 8u); +LOX_STATIC_ASSERT(core_kv_max_keys_min, LOX_KV_MAX_KEYS >= 1u); +LOX_STATIC_ASSERT(core_kv_key_max_len_min, LOX_KV_KEY_MAX_LEN >= 4u); +LOX_STATIC_ASSERT(core_kv_val_max_len_min, LOX_KV_VAL_MAX_LEN >= 1u); +LOX_STATIC_ASSERT(core_ts_max_streams_min, LOX_TS_MAX_STREAMS >= 1u); +LOX_STATIC_ASSERT(core_rel_max_tables_min, LOX_REL_MAX_TABLES >= 1u); +LOX_STATIC_ASSERT(core_rel_max_cols_min, LOX_REL_MAX_COLS >= 1u); + +static size_t lox_bytes_from_kb(uint32_t ram_kb) { + return (size_t)ram_kb * 1024u; +} + +static size_t lox_slice_bytes(size_t total, uint32_t pct) { + return (total * (size_t)pct) / 100u; +} + +static uint8_t *lox_align_ptr(uint8_t *ptr, size_t align) { + uintptr_t p = (uintptr_t)ptr; + uintptr_t a = (p + (uintptr_t)(align - 1u)) & ~((uintptr_t)align - 1u); + return (uint8_t *)a; +} + +const char *lox_err_to_string(lox_err_t err) { + switch (err) { + case LOX_OK: + return "LOX_OK"; + case LOX_ERR_INVALID: + return "LOX_ERR_INVALID"; + case LOX_ERR_NO_MEM: + return "LOX_ERR_NO_MEM"; + case LOX_ERR_FULL: + return "LOX_ERR_FULL"; + case LOX_ERR_NOT_FOUND: + return "LOX_ERR_NOT_FOUND"; + case LOX_ERR_EXPIRED: + return "LOX_ERR_EXPIRED"; + case LOX_ERR_STORAGE: + return "LOX_ERR_STORAGE"; + case LOX_ERR_CORRUPT: + return "LOX_ERR_CORRUPT"; + case LOX_ERR_SEALED: + return "LOX_ERR_SEALED"; + case LOX_ERR_EXISTS: + return "LOX_ERR_EXISTS"; + case LOX_ERR_DISABLED: + return "LOX_ERR_DISABLED"; + case LOX_ERR_OVERFLOW: + return "LOX_ERR_OVERFLOW"; + case LOX_ERR_SCHEMA: + return "LOX_ERR_SCHEMA"; + case LOX_ERR_TXN_ACTIVE: + return "LOX_ERR_TXN_ACTIVE"; + case LOX_ERR_MODIFIED: + return "LOX_ERR_MODIFIED"; + default: + return "LOX_ERR_UNKNOWN"; + } +} + +lox_core_t *lox_core(lox_t *db) { + return (lox_core_t *)&db->_opaque[0]; +} + +const lox_core_t *lox_core_const(const lox_t *db) { + return (const lox_core_t *)&db->_opaque[0]; +} + +static lox_err_t lox_validate_handle(const lox_t *db) { + if (db == NULL) { + return LOX_ERR_INVALID; + } + + if (lox_core_const(db)->magic != LOX_MAGIC) { + return LOX_ERR_INVALID; + } + + return LOX_OK; +} + +static uint8_t lox_fill_pct_u32(uint32_t used, uint32_t total) { + if (total == 0u) { + return 0u; + } + return (uint8_t)((used * 100u) / total); +} + +static uint32_t lox_kv_tombstone_count(const lox_core_t *core) { + uint32_t i; + uint32_t tombstones = 0u; + for (i = 0u; i < core->kv.bucket_count; ++i) { + if (core->kv.buckets[i].state == 2u) { + tombstones++; + } + } + return tombstones; +} + +static uint32_t lox_kv_live_value_bytes_local(const lox_core_t *core) { + return core->kv.live_value_bytes; +} + +static const lox_kv_bucket_t *lox_kv_find_bucket_const(const lox_core_t *core, const char *key) { + uint32_t i; + for (i = 0u; i < core->kv.bucket_count; ++i) { + const lox_kv_bucket_t *bucket = &core->kv.buckets[i]; + if (bucket->state == 1u && strncmp(bucket->key, key, LOX_KV_KEY_MAX_LEN) == 0) { + return bucket; + } + } + return NULL; +} + +static const lox_ts_stream_t *lox_ts_find_const(const lox_core_t *core, const char *name) { + uint32_t i; + for (i = 0u; i < LOX_TS_MAX_STREAMS; ++i) { + const lox_ts_stream_t *stream = &core->ts.streams[i]; + if (stream->registered && strcmp(stream->name, name) == 0) { + return stream; + } + } + return NULL; +} + +static const lox_table_t *lox_rel_find_table_const(const lox_core_t *core, const char *name) { + uint32_t i; + for (i = 0u; i < LOX_REL_MAX_TABLES; ++i) { + const lox_table_t *table = &core->rel.tables[i]; + if (table->registered && strcmp(table->name, name) == 0) { + return table; + } + } + return NULL; +} + +static uint32_t lox_wal_entry_size_for_payload(uint32_t payload_len) { + return 16u + ((payload_len + 3u) & ~3u); +} + +static void lox_fill_wal_admission(const lox_core_t *core, + uint32_t required_wal_bytes, + uint8_t *out_would_compact, + uint32_t *out_wal_free) { + uint32_t wal_free = 0u; + uint8_t would_compact = 0u; + if (core->wal_enabled && core->layout.wal_size > core->wal_used) { + wal_free = core->layout.wal_size - core->wal_used; + if (required_wal_bytes > wal_free) { + would_compact = 1u; + } else if (core->wal_compact_auto != 0u && core->layout.wal_size > 32u) { + uint32_t threshold = (core->wal_compact_threshold_pct != 0u) ? core->wal_compact_threshold_pct : 75u; + uint32_t total = core->layout.wal_size - 32u; + uint32_t used_after = ((core->wal_used > 32u) ? (core->wal_used - 32u) : 0u) + required_wal_bytes; + uint32_t fill = (total == 0u) ? 0u : ((used_after * 100u) / total); + if (fill >= threshold) { + would_compact = 1u; + } + } + } + *out_would_compact = would_compact; + *out_wal_free = wal_free; +} + +lox_err_t lox_init(lox_t *db, const lox_cfg_t *cfg) { + lox_core_t *core; + uint8_t *cursor; + uint32_t ram_kb; + uint8_t kv_pct; + uint8_t ts_pct; + uint8_t rel_pct; + bool custom_split; + size_t total_bytes; + size_t kv_bytes; + size_t ts_bytes; + lox_err_t err; + + if (db == NULL || cfg == NULL) { + return LOX_ERR_INVALID; + } + + memset(db, 0, sizeof(*db)); + core = lox_core(db); + + ram_kb = cfg->ram_kb != 0u ? cfg->ram_kb : LOX_RAM_KB; + custom_split = (cfg->kv_pct != 0u) || (cfg->ts_pct != 0u) || (cfg->rel_pct != 0u); + if (custom_split) { + if (cfg->kv_pct == 0u || cfg->ts_pct == 0u || cfg->rel_pct == 0u) { + return LOX_ERR_INVALID; + } + kv_pct = cfg->kv_pct; + ts_pct = cfg->ts_pct; + rel_pct = cfg->rel_pct; + } else { + kv_pct = (uint8_t)LOX_RAM_KV_PCT; + ts_pct = (uint8_t)LOX_RAM_TS_PCT; + rel_pct = (uint8_t)LOX_RAM_REL_PCT; + } + if ((uint32_t)kv_pct + (uint32_t)ts_pct + (uint32_t)rel_pct != 100u) { + return LOX_ERR_INVALID; + } + if (cfg->wal_compact_auto != 0u && + (cfg->wal_compact_threshold_pct == 0u || cfg->wal_compact_threshold_pct > 100u)) { + return LOX_ERR_INVALID; + } + if (cfg->wal_sync_mode > LOX_WAL_SYNC_FLUSH_ONLY) { + return LOX_ERR_INVALID; + } + total_bytes = lox_bytes_from_kb(ram_kb); + + core->heap = (uint8_t *)malloc(total_bytes); + if (core->heap == NULL) { + LOX_LOG("ERROR", + "malloc(%u) failed for RAM budget", + (unsigned)(ram_kb * 1024u)); + memset(db, 0, sizeof(*db)); + return LOX_ERR_NO_MEM; + } + + memset(core->heap, 0, total_bytes); + core->magic = LOX_MAGIC; + core->heap_size = total_bytes; + core->storage = cfg->storage; + core->now = cfg->now; + core->lock = cfg->lock; + core->unlock = cfg->unlock; + core->lock_destroy = cfg->lock_destroy; + if (cfg->lock_create != NULL) { + core->lock_handle = cfg->lock_create(); + } + core->wal_compact_auto = cfg->wal_compact_auto; + core->wal_compact_threshold_pct = cfg->wal_compact_threshold_pct; + core->wal_sync_mode = cfg->wal_sync_mode; + core->on_migrate = cfg->on_migrate; + core->last_runtime_error = LOX_OK; + core->last_recovery_status = LOX_OK; + core->wal_enabled = (cfg->storage != NULL) && (LOX_ENABLE_WAL != 0); + lox_arena_init(&core->arena, core->heap, total_bytes); + + kv_bytes = lox_slice_bytes(total_bytes, kv_pct); + ts_bytes = lox_slice_bytes(total_bytes, ts_pct); + + cursor = core->heap; + lox_arena_init(&core->kv_arena, cursor, kv_bytes); + cursor += kv_bytes; + { + uint8_t *heap_end = core->heap + total_bytes; + uint8_t *ts_base = lox_align_ptr(cursor, sizeof(uint32_t)); + uint8_t *ts_end; + uint8_t *rel_base; + + if (ts_base > heap_end) { + free(core->heap); + memset(db, 0, sizeof(*db)); + return LOX_ERR_NO_MEM; + } + if ((size_t)(heap_end - ts_base) < ts_bytes) { + free(core->heap); + memset(db, 0, sizeof(*db)); + return LOX_ERR_NO_MEM; + } + ts_end = ts_base + ts_bytes; + rel_base = lox_align_ptr(ts_end, sizeof(void *)); + if (rel_base > heap_end) { + free(core->heap); + memset(db, 0, sizeof(*db)); + return LOX_ERR_NO_MEM; + } + + lox_arena_init(&core->ts_arena, ts_base, ts_bytes); + lox_arena_init(&core->rel_arena, rel_base, (size_t)(heap_end - rel_base)); + } + + err = lox_kv_init(db); + if (err != LOX_OK) { + free(core->heap); + memset(db, 0, sizeof(*db)); + return err; + } + +#if LOX_ENABLE_TS + err = lox_ts_init(db); + if (err != LOX_OK) { + free(core->heap); + memset(db, 0, sizeof(*db)); + return err; + } +#endif + + err = lox_storage_bootstrap(db); + if (err != LOX_OK) { + core->last_runtime_error = err; + if (err == LOX_ERR_STORAGE && cfg->storage != NULL) { + LOX_LOG("ERROR", + "Storage capacity %u too small, need %u bytes", + (unsigned)cfg->storage->capacity, + (unsigned)core->layout.total_size); + } + free(core->heap); + memset(db, 0, sizeof(*db)); + return err; + } + + core->live_bytes = lox_kv_live_bytes(db); + return LOX_OK; +} + +lox_err_t lox_flush(lox_t *db) { + lox_core_t *core; + lox_err_t status; + + if (db == NULL) { + return LOX_ERR_INVALID; + } + LOX_LOCK(db); + status = lox_validate_handle(db); + if (status != LOX_OK) { + LOX_UNLOCK(db); + return status; + } + + core = lox_core(db); + status = lox_storage_flush(db); + lox_record_error(core, status); + LOX_UNLOCK(db); + return status; +} + +lox_err_t lox_deinit(lox_t *db) { + lox_core_t *core; + uint8_t *heap; + void (*lock_destroy)(void *hdl); + void *lock_handle; + lox_err_t status; + + if (db == NULL) { + return LOX_ERR_INVALID; + } + + status = lox_validate_handle(db); + if (status != LOX_OK) { + return status; + } + + LOX_LOCK(db); + core = lox_core(db); + if (core->magic != LOX_MAGIC) { + LOX_UNLOCK(db); + return LOX_ERR_INVALID; + } + status = lox_storage_flush(db); + heap = core->heap; + lock_destroy = core->lock_destroy; + lock_handle = core->lock_handle; + core->magic = 0u; + LOX_UNLOCK(db); + + if (lock_destroy != NULL) { + lock_destroy(lock_handle); + } + lox_record_error(core, status); + free(heap); + memset(db, 0, sizeof(*db)); + return status; +} + +lox_err_t lox_stats(const lox_t *db, lox_stats_t *out) { + return lox_inspect((lox_t *)db, out); +} + +lox_err_t lox_inspect(lox_t *db, lox_stats_t *out) { + const lox_core_t *core; + uint32_t ts_capacity_total = 0u; + uint32_t ts_samples_total = 0u; + uint32_t rel_rows_total = 0u; + uint32_t wal_bytes_used = 0u; + uint32_t wal_bytes_total = 0u; + uint32_t i; + lox_err_t status; + + if (out == NULL) { + return LOX_ERR_INVALID; + } + + status = lox_validate_handle(db); + if (status != LOX_OK) { + return status; + } + + LOX_LOCK(db); + core = lox_core_const(db); + if (core->magic != LOX_MAGIC) { + LOX_UNLOCK(db); + return LOX_ERR_INVALID; + } + + memset(out, 0, sizeof(*out)); + + out->kv_entries_used = core->kv.entry_count; + out->kv_entries_max = (LOX_KV_MAX_KEYS > LOX_TXN_STAGE_KEYS) ? (LOX_KV_MAX_KEYS - LOX_TXN_STAGE_KEYS) : 0u; + out->kv_fill_pct = lox_fill_pct_u32(out->kv_entries_used, out->kv_entries_max); + out->kv_collision_count = core->kv.collision_count; + out->kv_eviction_count = core->kv.eviction_count; + + out->ts_streams_registered = core->ts.registered_streams; + for (i = 0u; i < LOX_TS_MAX_STREAMS; ++i) { + ts_samples_total += core->ts.streams[i].count; + ts_capacity_total += core->ts.streams[i].capacity; + } + out->ts_samples_total = ts_samples_total; + out->ts_fill_pct = lox_fill_pct_u32(ts_samples_total, ts_capacity_total); + + if (core->wal_enabled && core->layout.wal_size > 32u) { + wal_bytes_total = core->layout.wal_size - 32u; + wal_bytes_used = (core->wal_used > 32u) ? (core->wal_used - 32u) : 0u; + } + out->wal_bytes_total = wal_bytes_total; + out->wal_bytes_used = wal_bytes_used; + out->wal_fill_pct = lox_fill_pct_u32(wal_bytes_used, wal_bytes_total); + + out->rel_tables_count = core->rel.registered_tables; + for (i = 0u; i < LOX_REL_MAX_TABLES; ++i) { + if (core->rel.tables[i].registered) { + rel_rows_total += core->rel.tables[i].live_count; + } + } + out->rel_rows_total = rel_rows_total; + + LOX_UNLOCK(db); + return LOX_OK; +} + +lox_err_t lox_get_db_stats(lox_t *db, lox_db_stats_t *out) { + const lox_core_t *core; + uint32_t wal_bytes_used = 0u; + uint32_t wal_bytes_total = 0u; + lox_err_t status; + + if (out == NULL) { + return LOX_ERR_INVALID; + } + + status = lox_validate_handle(db); + if (status != LOX_OK) { + return status; + } + + LOX_LOCK(db); + core = lox_core_const(db); + if (core->magic != LOX_MAGIC) { + LOX_UNLOCK(db); + return LOX_ERR_INVALID; + } + + memset(out, 0, sizeof(*out)); + if (core->wal_enabled && core->layout.wal_size > 32u) { + wal_bytes_total = core->layout.wal_size - 32u; + wal_bytes_used = (core->wal_used > 32u) ? (core->wal_used - 32u) : 0u; + } + out->effective_capacity_bytes = (core->storage != NULL) ? core->storage->capacity : 0u; + out->wal_bytes_total = wal_bytes_total; + out->wal_bytes_used = wal_bytes_used; + out->wal_fill_pct = lox_fill_pct_u32(wal_bytes_used, wal_bytes_total); + out->compact_count = core->compact_count; + out->reopen_count = core->reopen_count; + out->recovery_count = core->recovery_count; + out->last_runtime_error = core->last_runtime_error; + out->last_recovery_status = core->last_recovery_status; + out->active_generation = core->layout.active_generation; + out->active_bank = core->layout.active_bank; + + LOX_UNLOCK(db); + return LOX_OK; +} + +lox_err_t lox_get_kv_stats(lox_t *db, lox_kv_stats_t *out) { + const lox_core_t *core; + lox_err_t status; + uint32_t entry_limit; + + if (out == NULL) { + return LOX_ERR_INVALID; + } + + status = lox_validate_handle(db); + if (status != LOX_OK) { + return status; + } + + LOX_LOCK(db); + core = lox_core_const(db); + if (core->magic != LOX_MAGIC) { + LOX_UNLOCK(db); + return LOX_ERR_INVALID; + } + + entry_limit = (LOX_KV_MAX_KEYS > LOX_TXN_STAGE_KEYS) ? (LOX_KV_MAX_KEYS - LOX_TXN_STAGE_KEYS) : 0u; + memset(out, 0, sizeof(*out)); + out->live_keys = core->kv.entry_count; + out->collisions = core->kv.collision_count; + out->evictions = core->kv.eviction_count; + out->tombstones = lox_kv_tombstone_count(core); + out->value_bytes_used = core->kv.value_used; + out->fill_pct = lox_fill_pct_u32(out->live_keys, entry_limit); + + LOX_UNLOCK(db); + return LOX_OK; +} + +lox_err_t lox_get_ts_stats(lox_t *db, lox_ts_stats_t *out) { + const lox_core_t *core; + lox_err_t status; + uint32_t ts_capacity_total = 0u; + uint32_t ts_samples_total = 0u; + uint32_t i; + + if (out == NULL) { + return LOX_ERR_INVALID; + } + + status = lox_validate_handle(db); + if (status != LOX_OK) { + return status; + } + + LOX_LOCK(db); + core = lox_core_const(db); + if (core->magic != LOX_MAGIC) { + LOX_UNLOCK(db); + return LOX_ERR_INVALID; + } + + memset(out, 0, sizeof(*out)); + out->stream_count = core->ts.registered_streams; + for (i = 0u; i < LOX_TS_MAX_STREAMS; ++i) { + ts_samples_total += core->ts.streams[i].count; + ts_capacity_total += core->ts.streams[i].capacity; + } + out->retained_samples = ts_samples_total; + out->dropped_samples = core->ts_dropped_samples; + out->fill_pct = lox_fill_pct_u32(ts_samples_total, ts_capacity_total); + + LOX_UNLOCK(db); + return LOX_OK; +} + +lox_err_t lox_get_rel_stats(lox_t *db, lox_rel_stats_t *out) { + const lox_core_t *core; + lox_err_t status; + uint32_t i; + uint32_t rows_live = 0u; + uint32_t rows_capacity = 0u; + uint32_t indexed_tables = 0u; + uint32_t index_entries = 0u; + + if (out == NULL) { + return LOX_ERR_INVALID; + } + + status = lox_validate_handle(db); + if (status != LOX_OK) { + return status; + } + + LOX_LOCK(db); + core = lox_core_const(db); + if (core->magic != LOX_MAGIC) { + LOX_UNLOCK(db); + return LOX_ERR_INVALID; + } + + memset(out, 0, sizeof(*out)); + out->table_count = core->rel.registered_tables; + for (i = 0u; i < LOX_REL_MAX_TABLES; ++i) { + const lox_table_t *table = &core->rel.tables[i]; + if (!table->registered) { + continue; + } + rows_live += table->live_count; + rows_capacity += table->max_rows; + if (table->index_col != UINT32_MAX) { + indexed_tables++; + index_entries += table->index_count; + } + } + out->rows_live = rows_live; + out->rows_free = (rows_capacity > rows_live) ? (rows_capacity - rows_live) : 0u; + out->indexed_tables = indexed_tables; + out->index_entries = index_entries; + + LOX_UNLOCK(db); + return LOX_OK; +} + +lox_err_t lox_get_effective_capacity(lox_t *db, lox_effective_capacity_t *out) { + const lox_core_t *core; + lox_err_t status; + uint32_t ts_retained = 0u; + uint32_t ts_total = 0u; + uint32_t i; + uint32_t entry_limit; + uint32_t kv_free_now; + uint32_t wal_total = 0u; + uint32_t wal_used = 0u; + uint32_t wal_free = 0u; + uint32_t threshold_pct = 0u; + + if (out == NULL) { + return LOX_ERR_INVALID; + } + + status = lox_validate_handle(db); + if (status != LOX_OK) { + return status; + } + + LOX_LOCK(db); + core = lox_core_const(db); + if (core->magic != LOX_MAGIC) { + LOX_UNLOCK(db); + return LOX_ERR_INVALID; + } + + memset(out, 0, sizeof(*out)); + entry_limit = (LOX_KV_MAX_KEYS > LOX_TXN_STAGE_KEYS) ? (LOX_KV_MAX_KEYS - LOX_TXN_STAGE_KEYS) : 0u; + out->kv_entries_usable = entry_limit; + out->kv_entries_free = (entry_limit > core->kv.entry_count) ? (entry_limit - core->kv.entry_count) : 0u; + out->kv_value_bytes_usable = core->kv.value_capacity; + kv_free_now = (core->kv.value_capacity > core->kv.value_used) ? (core->kv.value_capacity - core->kv.value_used) : 0u; + out->kv_value_bytes_free_now = kv_free_now; + + for (i = 0u; i < LOX_TS_MAX_STREAMS; ++i) { + ts_retained += core->ts.streams[i].count; + ts_total += core->ts.streams[i].capacity; + } + out->ts_samples_usable = ts_total; + out->ts_samples_retained = ts_retained; + out->ts_samples_free = (ts_total > ts_retained) ? (ts_total - ts_retained) : 0u; + + if (core->wal_enabled && core->layout.wal_size > 32u) { + wal_total = core->layout.wal_size - 32u; + wal_used = (core->wal_used > 32u) ? (core->wal_used - 32u) : 0u; + wal_free = (wal_total > wal_used) ? (wal_total - wal_used) : 0u; + } + out->wal_budget_total = wal_total; + out->wal_budget_used = wal_used; + out->wal_budget_free = wal_free; + threshold_pct = (core->wal_compact_threshold_pct != 0u) ? core->wal_compact_threshold_pct : 75u; + out->compact_threshold_pct = threshold_pct; + out->wal_safety_reserved = (wal_total * threshold_pct) / 100u; + + if (core->storage == NULL) { + out->limiting_flags |= LOX_CAP_LIMIT_STORAGE_DISABLED; + } + if (out->kv_entries_free == 0u) { + out->limiting_flags |= LOX_CAP_LIMIT_KV_ENTRIES; + } + if (out->kv_value_bytes_free_now == 0u) { + out->limiting_flags |= LOX_CAP_LIMIT_KV_VALUE_BYTES; + } + if (out->ts_samples_free == 0u) { + out->limiting_flags |= LOX_CAP_LIMIT_TS_SAMPLES; + } + if (out->wal_budget_total != 0u && out->wal_budget_free == 0u) { + out->limiting_flags |= LOX_CAP_LIMIT_WAL_BUDGET; + } + + LOX_UNLOCK(db); + return LOX_OK; +} + +lox_err_t lox_get_pressure(lox_t *db, lox_pressure_t *out) { + const lox_core_t *core; + lox_err_t status; + uint32_t ts_total = 0u; + uint32_t ts_retained = 0u; + uint32_t rel_rows_live = 0u; + uint32_t rel_rows_capacity = 0u; + uint32_t wal_total = 0u; + uint32_t wal_used = 0u; + uint32_t threshold_pct = 0u; + uint32_t i; + uint32_t max_risk; + + if (out == NULL) { + return LOX_ERR_INVALID; + } + + status = lox_validate_handle(db); + if (status != LOX_OK) { + return status; + } + + LOX_LOCK(db); + core = lox_core_const(db); + if (core->magic != LOX_MAGIC) { + LOX_UNLOCK(db); + return LOX_ERR_INVALID; + } + + memset(out, 0, sizeof(*out)); + out->kv_fill_pct = lox_fill_pct_u32(core->kv.entry_count, + (LOX_KV_MAX_KEYS > LOX_TXN_STAGE_KEYS) + ? (LOX_KV_MAX_KEYS - LOX_TXN_STAGE_KEYS) + : 0u); + + for (i = 0u; i < LOX_TS_MAX_STREAMS; ++i) { + ts_total += core->ts.streams[i].capacity; + ts_retained += core->ts.streams[i].count; + } + out->ts_fill_pct = lox_fill_pct_u32(ts_retained, ts_total); + + for (i = 0u; i < LOX_REL_MAX_TABLES; ++i) { + const lox_table_t *table = &core->rel.tables[i]; + if (!table->registered) { + continue; + } + rel_rows_live += table->live_count; + rel_rows_capacity += table->max_rows; + } + out->rel_fill_pct = lox_fill_pct_u32(rel_rows_live, rel_rows_capacity); + + if (core->wal_enabled && core->layout.wal_size > 32u) { + wal_total = core->layout.wal_size - 32u; + wal_used = (core->wal_used > 32u) ? (core->wal_used - 32u) : 0u; + } + out->wal_fill_pct = lox_fill_pct_u32(wal_used, wal_total); + + threshold_pct = (core->wal_compact_threshold_pct != 0u) ? core->wal_compact_threshold_pct : 75u; + if (wal_total == 0u) { + out->compact_pressure_pct = 0u; + } else if (threshold_pct == 0u) { + out->compact_pressure_pct = out->wal_fill_pct; + } else { + uint32_t pressure = (uint32_t)out->wal_fill_pct * 100u / threshold_pct; + out->compact_pressure_pct = (uint8_t)((pressure > 100u) ? 100u : pressure); + } + + out->risk_flags = LOX_CAP_LIMIT_NONE; + if (core->storage == NULL) { + out->risk_flags |= LOX_CAP_LIMIT_STORAGE_DISABLED; + } + if (out->kv_fill_pct >= 100u) { + out->risk_flags |= LOX_CAP_LIMIT_KV_ENTRIES; + } + if (out->ts_fill_pct >= 100u) { + out->risk_flags |= LOX_CAP_LIMIT_TS_SAMPLES; + } + if (out->wal_fill_pct >= 100u) { + out->risk_flags |= LOX_CAP_LIMIT_WAL_BUDGET; + } + + max_risk = out->kv_fill_pct; + if (out->ts_fill_pct > max_risk) { + max_risk = out->ts_fill_pct; + } + if (out->rel_fill_pct > max_risk) { + max_risk = out->rel_fill_pct; + } + if (out->wal_fill_pct > max_risk) { + max_risk = out->wal_fill_pct; + } + if (out->compact_pressure_pct > max_risk) { + max_risk = out->compact_pressure_pct; + } + out->near_full_risk_pct = (uint8_t)max_risk; + + LOX_UNLOCK(db); + return LOX_OK; +} + +lox_err_t lox_admit_kv_set(lox_t *db, const char *key, size_t val_len, lox_admission_t *out) { + if (out == NULL || key == NULL || key[0] == '\0') { + return LOX_ERR_INVALID; + } + memset(out, 0, sizeof(*out)); + +#if !LOX_ENABLE_KV + (void)db; + (void)val_len; + out->status = LOX_ERR_DISABLED; + return LOX_ERR_DISABLED; +#else + const lox_core_t *core; + lox_err_t status; + uint32_t required = 0u; + uint32_t available = 0u; + uint32_t compact_available = 0u; + uint32_t entry_limit; + uint8_t would_compact = 0u; + uint32_t wal_free = 0u; + uint32_t payload_len = 0u; + uint32_t wal_bytes = 0u; + const lox_kv_bucket_t *existing; + if (val_len > LOX_KV_VAL_MAX_LEN || strlen(key) >= LOX_KV_KEY_MAX_LEN) { + out->status = LOX_ERR_INVALID; + return LOX_ERR_INVALID; + } + status = lox_validate_handle(db); + if (status != LOX_OK) { + out->status = status; + return status; + } + + LOX_LOCK(db); + core = lox_core_const(db); + if (core->magic != LOX_MAGIC) { + LOX_UNLOCK(db); + out->status = LOX_ERR_INVALID; + return LOX_ERR_INVALID; + } + + existing = lox_kv_find_bucket_const(core, key); + if (existing != NULL) { + required = (val_len > existing->val_len) ? (uint32_t)(val_len - existing->val_len) : 0u; + } else { + required = (uint32_t)val_len; + entry_limit = (LOX_KV_MAX_KEYS > LOX_TXN_STAGE_KEYS) ? (LOX_KV_MAX_KEYS - LOX_TXN_STAGE_KEYS) : 0u; + if (core->kv.entry_count >= entry_limit) { +#if LOX_KV_OVERFLOW_POLICY == LOX_KV_POLICY_REJECT + out->status = LOX_ERR_FULL; + out->deterministic_budget_ok = 0u; + LOX_UNLOCK(db); + return LOX_OK; +#else + out->would_degrade = 1u; + out->deterministic_budget_ok = 0u; +#endif + } + } + + available = (core->kv.value_capacity > core->kv.value_used) ? (core->kv.value_capacity - core->kv.value_used) : 0u; + compact_available = core->kv.value_capacity - lox_kv_live_value_bytes_local(core); + out->required_bytes = required; + out->available_bytes = available; + + if (required > available) { + if (required <= compact_available) { + out->would_compact = 1u; + } else { + out->status = LOX_ERR_NO_MEM; + out->deterministic_budget_ok = 0u; + LOX_UNLOCK(db); + return LOX_OK; + } + } + + if (core->wal_enabled) { + payload_len = (uint32_t)(1u + strlen(key) + 4u + val_len + 4u); + wal_bytes = lox_wal_entry_size_for_payload(payload_len); + out->required_wal_bytes = wal_bytes; + lox_fill_wal_admission(core, wal_bytes, &would_compact, &wal_free); + out->wal_bytes_free = wal_free; + if (would_compact != 0u) { + out->would_compact = 1u; + } + } + + if (out->deterministic_budget_ok == 0u && out->would_degrade == 0u) { + out->deterministic_budget_ok = 1u; + } + out->status = LOX_OK; + LOX_UNLOCK(db); + return LOX_OK; +#endif +} + +lox_err_t lox_admit_ts_insert(lox_t *db, const char *stream_name, size_t sample_len, lox_admission_t *out) { + if (out == NULL || stream_name == NULL || stream_name[0] == '\0') { + return LOX_ERR_INVALID; + } + memset(out, 0, sizeof(*out)); + +#if !LOX_ENABLE_TS + (void)db; + (void)sample_len; + out->status = LOX_ERR_DISABLED; + return LOX_ERR_DISABLED; +#else + const lox_core_t *core; + const lox_ts_stream_t *stream; + lox_err_t status; + uint32_t expected_len = 0u; + uint8_t would_compact = 0u; + uint32_t wal_free = 0u; + uint32_t wal_bytes = 0u; + uint32_t payload_len = 0u; + status = lox_validate_handle(db); + if (status != LOX_OK) { + out->status = status; + return status; + } + + LOX_LOCK(db); + core = lox_core_const(db); + if (core->magic != LOX_MAGIC) { + LOX_UNLOCK(db); + out->status = LOX_ERR_INVALID; + return LOX_ERR_INVALID; + } + + stream = lox_ts_find_const(core, stream_name); + if (stream == NULL) { + out->status = LOX_ERR_NOT_FOUND; + LOX_UNLOCK(db); + return LOX_OK; + } + expected_len = (stream->type == LOX_TS_RAW) ? (uint32_t)stream->raw_size : 4u; + if (sample_len != expected_len) { + out->status = LOX_ERR_INVALID; + LOX_UNLOCK(db); + return LOX_OK; + } + + out->required_bytes = 1u; + out->available_bytes = (stream->capacity > stream->count) ? (stream->capacity - stream->count) : 0u; + if (stream->count >= stream->capacity) { +#if LOX_TS_OVERFLOW_POLICY == LOX_TS_POLICY_REJECT + out->status = LOX_ERR_FULL; + out->deterministic_budget_ok = 0u; + LOX_UNLOCK(db); + return LOX_OK; +#else + out->would_degrade = 1u; + out->deterministic_budget_ok = 0u; +#endif + } + + if (core->wal_enabled) { + payload_len = (uint32_t)(1u + strlen(stream_name) + 9u + sample_len); + wal_bytes = lox_wal_entry_size_for_payload(payload_len); + out->required_wal_bytes = wal_bytes; + lox_fill_wal_admission(core, wal_bytes, &would_compact, &wal_free); + out->wal_bytes_free = wal_free; + if (would_compact != 0u) { + out->would_compact = 1u; + } + } + + if (out->deterministic_budget_ok == 0u && out->would_degrade == 0u) { + out->deterministic_budget_ok = 1u; + } + out->status = LOX_OK; + LOX_UNLOCK(db); + return LOX_OK; +#endif +} + +lox_err_t lox_admit_rel_insert(lox_t *db, const char *table_name, size_t row_len, lox_admission_t *out) { + if (out == NULL || table_name == NULL || table_name[0] == '\0') { + return LOX_ERR_INVALID; + } + memset(out, 0, sizeof(*out)); + +#if !LOX_ENABLE_REL + (void)db; + (void)row_len; + out->status = LOX_ERR_DISABLED; + return LOX_ERR_DISABLED; +#else + const lox_core_t *core; + const lox_table_t *table; + lox_err_t status; + uint8_t would_compact = 0u; + uint32_t wal_free = 0u; + uint32_t wal_bytes = 0u; + uint32_t payload_len = 0u; + status = lox_validate_handle(db); + if (status != LOX_OK) { + out->status = status; + return status; + } + + LOX_LOCK(db); + core = lox_core_const(db); + if (core->magic != LOX_MAGIC) { + LOX_UNLOCK(db); + out->status = LOX_ERR_INVALID; + return LOX_ERR_INVALID; + } + + table = lox_rel_find_table_const(core, table_name); + if (table == NULL) { + out->status = LOX_ERR_NOT_FOUND; + LOX_UNLOCK(db); + return LOX_OK; + } + if (row_len != table->row_size) { + out->status = LOX_ERR_INVALID; + LOX_UNLOCK(db); + return LOX_OK; + } + + out->required_bytes = 1u; + out->available_bytes = (table->max_rows > table->live_count) ? (table->max_rows - table->live_count) : 0u; + if (table->live_count >= table->max_rows) { + out->status = LOX_ERR_FULL; + out->deterministic_budget_ok = 0u; + LOX_UNLOCK(db); + return LOX_OK; + } + + if (core->wal_enabled) { + payload_len = (uint32_t)(1u + strlen(table_name) + 4u + row_len); + wal_bytes = lox_wal_entry_size_for_payload(payload_len); + out->required_wal_bytes = wal_bytes; + lox_fill_wal_admission(core, wal_bytes, &would_compact, &wal_free); + out->wal_bytes_free = wal_free; + if (would_compact != 0u) { + out->would_compact = 1u; + } + } + + if (out->would_compact != 0u || out->would_degrade != 0u) { + out->deterministic_budget_ok = 0u; + } else if (out->status == LOX_OK && + out->deterministic_budget_ok == 0u && + out->would_degrade == 0u) { + out->deterministic_budget_ok = 1u; + } + out->status = LOX_OK; + LOX_UNLOCK(db); + return LOX_OK; +#endif +} diff --git a/bench/loxdb_esp32_s3_bench_head/run_bench.ps1 b/bench/loxdb_esp32_s3_bench_head/run_bench.ps1 index 1d46d95..d82bfda 100644 --- a/bench/loxdb_esp32_s3_bench_head/run_bench.ps1 +++ b/bench/loxdb_esp32_s3_bench_head/run_bench.ps1 @@ -86,6 +86,7 @@ $serial.RtsEnable = $false $fullLog = "" $ok = $false +$prompts = @("loxdb-bench>", "microdb-bench>") $commands = @() foreach ($part in ($CommandScript -split ';')) { $cmd = $part.Trim() @@ -103,10 +104,11 @@ try { $serial.WriteLine("") $buf = "" - $ready = Read-UntilPattern -Serial $serial -Pattern "loxdb-bench>" -TimeoutSec $OpenTimeoutSec -Buffer ([ref]$buf) + $matched = "" + $ready = Read-UntilAnyPattern -Serial $serial -Patterns $prompts -TimeoutSec $OpenTimeoutSec -Buffer ([ref]$buf) -MatchedPattern ([ref]$matched) $fullLog += $buf if (-not $ready) { - throw "Prompt 'loxdb-bench>' not detected on $Port within $OpenTimeoutSec s." + throw "Prompt 'loxdb-bench>' or 'microdb-bench>' not detected on $Port within $OpenTimeoutSec s." } foreach ($cmd in $commands) { @@ -114,24 +116,27 @@ try { $buf = "" if ($cmd -eq "run" -or $cmd -eq "run_det" -or $cmd -eq "run_det_paced") { $matched = "" - $runDone = Read-UntilAnyPattern -Serial $serial -Patterns @("=== loxdb ESP32-S3 benchmark end ===", "loxdb-bench>") -TimeoutSec $RunTimeoutSec -Buffer ([ref]$buf) -MatchedPattern ([ref]$matched) + $runDone = Read-UntilAnyPattern -Serial $serial -Patterns (@("=== loxdb ESP32-S3 benchmark end ===", "=== microdb ESP32-S3 benchmark end ===") + $prompts) -TimeoutSec $RunTimeoutSec -Buffer ([ref]$buf) -MatchedPattern ([ref]$matched) $fullLog += $buf if (-not $runDone) { throw "Benchmark command '$cmd' did not finish within $RunTimeoutSec s." } - if ($matched -eq "loxdb-bench>" -and $buf -notmatch [regex]::Escape("=== loxdb ESP32-S3 benchmark end ===")) { + if (($prompts -contains $matched) -and $buf -notmatch [regex]::Escape("=== loxdb ESP32-S3 benchmark end ===") -and $buf -notmatch [regex]::Escape("=== microdb ESP32-S3 benchmark end ===")) { throw "Benchmark command '$cmd' returned to prompt without benchmark end marker." } - if ($buf -notmatch [regex]::Escape("loxdb-bench>")) { + $promptRegex = ($prompts | ForEach-Object { [regex]::Escape($_) }) -join "|" + if ($buf -notmatch $promptRegex) { $buf = "" - $promptBack = Read-UntilPattern -Serial $serial -Pattern "loxdb-bench>" -TimeoutSec 20 -Buffer ([ref]$buf) + $promptBackMatched = "" + $promptBack = Read-UntilAnyPattern -Serial $serial -Patterns $prompts -TimeoutSec 20 -Buffer ([ref]$buf) -MatchedPattern ([ref]$promptBackMatched) $fullLog += $buf if (-not $promptBack) { throw "Prompt not returned after '$cmd'." } } } else { - $okBack = Read-UntilPattern -Serial $serial -Pattern "loxdb-bench>" -TimeoutSec 30 -Buffer ([ref]$buf) + $okBackMatched = "" + $okBack = Read-UntilAnyPattern -Serial $serial -Patterns $prompts -TimeoutSec 30 -Buffer ([ref]$buf) -MatchedPattern ([ref]$okBackMatched) $fullLog += $buf if (-not $okBack) { throw "Command '$cmd' did not return to prompt." diff --git a/bench/loxdb_esp32_s3_sd_stress_bench/bench_admission.h b/bench/loxdb_esp32_s3_sd_stress_bench/bench_admission.h new file mode 100644 index 0000000..281041e --- /dev/null +++ b/bench/loxdb_esp32_s3_sd_stress_bench/bench_admission.h @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: MIT +#ifndef LOXDB_SD_STRESS_BENCH_ADMISSION_H +#define LOXDB_SD_STRESS_BENCH_ADMISSION_H + +#include + +typedef struct bench_admission_profile_t bench_admission_profile_t; + +struct bench_admission_profile_t { + const char *name; + uint16_t ram_kb; + uint8_t kv_pct; + uint8_t ts_pct; + uint8_t rel_pct; + uint8_t wal_compact_threshold_pct; +}; + +#endif /* LOXDB_SD_STRESS_BENCH_ADMISSION_H */ diff --git a/bench/loxdb_esp32_s3_sd_stress_bench/loxdb_esp32_s3_sd_stress_bench.ino b/bench/loxdb_esp32_s3_sd_stress_bench/loxdb_esp32_s3_sd_stress_bench.ino index 70023d5..5ef9afd 100644 --- a/bench/loxdb_esp32_s3_sd_stress_bench/loxdb_esp32_s3_sd_stress_bench.ino +++ b/bench/loxdb_esp32_s3_sd_stress_bench/loxdb_esp32_s3_sd_stress_bench.ino @@ -4,6 +4,8 @@ #include #include +#include "bench_admission.h" + /* Increase core engine limits for large SD stress profile. */ #ifndef LOX_KV_MAX_KEYS #define LOX_KV_MAX_KEYS 4096 @@ -94,6 +96,11 @@ static bool g_sd_ready = false; static bool g_db_ready = false; static bool g_running = true; static bool g_verify_enabled = true; +static uint16_t g_admitted_ram_kb = 0u; +static uint8_t g_admitted_kv_pct = 0u; +static uint8_t g_admitted_ts_pct = 0u; +static uint8_t g_admitted_rel_pct = 0u; +static uint8_t g_admitted_wal_th_pct = 0u; static uint8_t *g_erase_buf = NULL; static uint32_t g_ops = 0u; @@ -149,6 +156,77 @@ static uint32_t rng_next(void) { return s; } +static void startup_fail(const char *what, const char *hint) { + Serial.printf("[FATAL] %s\n", what ? what : "startup failed"); + if (hint && hint[0] != '\0') Serial.printf("[HINT] %s\n", hint); + Serial.println("[HINT] use 'stats' (if running) or 'resetdb' after fixing the issue"); + g_db_ready = false; + g_running = false; +} + +static void init_cleanup() { + (void)lox_deinit(&g_db); + uint32_t i; + for (i = 0u; i < kRelTableCount; ++i) g_rel_tables[i] = NULL; +} + +static bool register_ts_streams(lox_err_t *out_rc) { + uint32_t i; + for (i = 0u; i < kTsStreamCount; ++i) { + lox_err_t rc = lox_ts_register(&g_db, kTsStreams[i], LOX_TS_U32, 0u); + if (!(rc == LOX_OK || rc == LOX_ERR_EXISTS)) { + Serial.printf("[ERR] lox_ts_register(%s) rc=%d (%s)\n", kTsStreams[i], (int)rc, lox_err_to_string(rc)); + if (out_rc) *out_rc = rc; + return false; + } + g_ts_seq[i] = 0u; + } + if (out_rc) *out_rc = LOX_OK; + return true; +} + +static bool post_init_setup(lox_err_t *out_rc) { + lox_err_t rc = LOX_OK; + if (!register_ts_streams(&rc)) { + if (out_rc) *out_rc = rc; + return false; + } + if (!setup_rel()) { + Serial.println("[ERR] setup_rel failed (try smaller profile or resetdb)"); + if (out_rc) *out_rc = LOX_ERR_NO_MEM; + return false; + } + if (out_rc) *out_rc = LOX_OK; + return true; +} + +static bool preflight_profile(const bench_admission_profile_t *p, char *reason, size_t reason_cap) { + if (!p) return false; + if ((uint32_t)p->kv_pct + (uint32_t)p->ts_pct + (uint32_t)p->rel_pct != 100u) { + if (reason && reason_cap) snprintf(reason, reason_cap, "split must sum to 100 (got %u/%u/%u)", + (unsigned)p->kv_pct, (unsigned)p->ts_pct, (unsigned)p->rel_pct); + return false; + } + if (p->ram_kb == 0u) { + if (reason && reason_cap) snprintf(reason, reason_cap, "ram_kb must be > 0"); + return false; + } +#if defined(ARDUINO_ARCH_ESP32) + size_t free_int = heap_caps_get_free_size(MALLOC_CAP_INTERNAL | MALLOC_CAP_8BIT); + size_t free_psram = heap_caps_get_free_size(MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); + size_t free_total = free_int + free_psram; + size_t need = (size_t)p->ram_kb * 1024u; + size_t headroom = 128u * 1024u; + if (free_total < (need + headroom)) { + if (reason && reason_cap) snprintf(reason, reason_cap, "not enough heap/psram (need~%luKB + headroom, free=%luKB)", + (unsigned long)(need / 1024u), (unsigned long)(free_total / 1024u)); + return false; + } +#endif + if (reason && reason_cap) reason[0] = '\0'; + return true; +} + static const char *mode_name(stress_mode_t m) { switch (m) { case MODE_KV: return "kv"; @@ -626,6 +704,7 @@ static bool init_db() { lox_err_t rc; memset(&cfg, 0, sizeof(cfg)); memset(&g_storage, 0, sizeof(g_storage)); + init_cleanup(); g_storage.read = st_read; g_storage.write = st_write; @@ -637,52 +716,104 @@ static bool init_db() { g_storage.ctx = NULL; cfg.storage = &g_storage; - cfg.ram_kb = 8192u; - cfg.kv_pct = 45u; - cfg.ts_pct = 20u; - cfg.rel_pct = 35u; - cfg.wal_compact_auto = 1u; - cfg.wal_compact_threshold_pct = 75u; - cfg.wal_sync_mode = LOX_WAL_SYNC_FLUSH_ONLY; - - rc = lox_init(&g_db, &cfg); - if (rc == LOX_ERR_CORRUPT || rc == LOX_ERR_EXISTS || rc == LOX_ERR_SCHEMA) { - Serial.printf("[WARN] lox_init rc=%d (%s), recreating storage file\n", (int)rc, lox_err_to_string(rc)); - (void)lox_deinit(&g_db); - if (g_store) g_store.close(); - (void)SD_MMC.remove(kStoragePath); - if (!open_storage_file()) { - Serial.println("[ERR] recreate storage file failed"); - return false; + + /* Admission ladder: try bigger configs first, fallback deterministically. */ + static const bench_admission_profile_t kLadderStress[] = { + {"stress-A", 8192u, 45u, 20u, 35u, 75u}, + {"stress-B", 4096u, 40u, 30u, 30u, 70u}, + {"stress-C", 2048u, 34u, 33u, 33u, 65u}, + }; + static const bench_admission_profile_t kLadderSoak[] = { + {"soak-A", 4096u, 40u, 30u, 30u, 70u}, + {"soak-B", 2048u, 34u, 33u, 33u, 65u}, + }; + static const bench_admission_profile_t kLadderSmoke[] = { + {"smoke-A", 2048u, 34u, 33u, 33u, 65u}, + {"smoke-B", 1024u, 34u, 33u, 33u, 60u}, + }; + + const bench_admission_profile_t *ladder = NULL; + size_t ladder_n = 0u; + if (g_profile == PROFILE_STRESS) { + ladder = kLadderStress; + ladder_n = sizeof(kLadderStress) / sizeof(kLadderStress[0]); + } else if (g_profile == PROFILE_SMOKE) { + ladder = kLadderSmoke; + ladder_n = sizeof(kLadderSmoke) / sizeof(kLadderSmoke[0]); + } else { + ladder = kLadderSoak; + ladder_n = sizeof(kLadderSoak) / sizeof(kLadderSoak[0]); + } + + char reason[128]; + for (size_t i = 0u; i < ladder_n; ++i) { + const bench_admission_profile_t *p = &ladder[i]; + if (!preflight_profile(p, reason, sizeof(reason))) { + Serial.printf("[WARN] preflight reject profile=%s: %s\n", p->name, reason); + continue; } + + cfg.ram_kb = p->ram_kb; + cfg.kv_pct = p->kv_pct; + cfg.ts_pct = p->ts_pct; + cfg.rel_pct = p->rel_pct; + cfg.wal_compact_auto = 1u; + cfg.wal_compact_threshold_pct = p->wal_compact_threshold_pct; + cfg.wal_sync_mode = LOX_WAL_SYNC_FLUSH_ONLY; + rc = lox_init(&g_db, &cfg); - } - if (rc != LOX_OK) { - Serial.printf("[ERR] lox_init rc=%d cap=%lu erase=%lu write=%lu ram_kb=%u split=%u/%u/%u wal_th=%u\n", - (int)rc, - (unsigned long)g_storage.capacity, - (unsigned long)g_storage.erase_size, - (unsigned long)g_storage.write_size, - (unsigned)cfg.ram_kb, - (unsigned)cfg.kv_pct, - (unsigned)cfg.ts_pct, - (unsigned)cfg.rel_pct, - (unsigned)cfg.wal_compact_threshold_pct); - return false; - } - { - uint32_t i; - for (i = 0u; i < kTsStreamCount; ++i) { - rc = lox_ts_register(&g_db, kTsStreams[i], LOX_TS_U32, 0u); - if (!(rc == LOX_OK || rc == LOX_ERR_EXISTS)) { - Serial.printf("[ERR] lox_ts_register(%s) rc=%d\n", kTsStreams[i], (int)rc); + if (rc == LOX_ERR_CORRUPT || rc == LOX_ERR_EXISTS || rc == LOX_ERR_SCHEMA) { + Serial.printf("[WARN] lox_init profile=%s rc=%d (%s), recreating storage file\n", + p->name, (int)rc, lox_err_to_string(rc)); + init_cleanup(); + if (g_store) g_store.close(); + (void)SD_MMC.remove(kStoragePath); + if (!open_storage_file()) { + Serial.println("[ERR] recreate storage file failed"); return false; } - g_ts_seq[i] = 0u; + rc = lox_init(&g_db, &cfg); + } + + if (rc == LOX_OK) { + lox_err_t post_rc = LOX_OK; + if (post_init_setup(&post_rc)) { + g_admitted_ram_kb = p->ram_kb; + g_admitted_kv_pct = p->kv_pct; + g_admitted_ts_pct = p->ts_pct; + g_admitted_rel_pct = p->rel_pct; + g_admitted_wal_th_pct = p->wal_compact_threshold_pct; + Serial.printf("[OK] admitted profile=%s ram_kb=%u split=%u/%u/%u wal_th=%u\n", + p->name, + (unsigned)p->ram_kb, + (unsigned)p->kv_pct, (unsigned)p->ts_pct, (unsigned)p->rel_pct, + (unsigned)p->wal_compact_threshold_pct); + break; + } + + Serial.printf("[WARN] post-init reject profile=%s rc=%d (%s)\n", + p->name, (int)post_rc, lox_err_to_string(post_rc)); + init_cleanup(); + if (post_rc == LOX_ERR_NO_MEM || post_rc == LOX_ERR_FULL || post_rc == LOX_ERR_STORAGE) { + continue; + } + rc = post_rc; + break; + } + + Serial.printf("[WARN] lox_init reject profile=%s rc=%d (%s)\n", p->name, (int)rc, lox_err_to_string(rc)); + if (!(rc == LOX_ERR_NO_MEM || rc == LOX_ERR_FULL || rc == LOX_ERR_STORAGE)) { + break; } } - if (!setup_rel()) { - Serial.println("[ERR] setup_rel failed"); + + if (rc != LOX_OK) { + Serial.printf("[ERR] lox_init failed rc=%d (%s) cap=%lu erase=%lu write=%lu\n", + (int)rc, lox_err_to_string(rc), + (unsigned long)g_storage.capacity, + (unsigned long)g_storage.erase_size, + (unsigned long)g_storage.write_size); + Serial.println("[HINT] try: ensure PSRAM enabled, use profile smoke/soak, or reduce storage image size"); return false; } return true; @@ -799,6 +930,7 @@ static void print_usage() { Serial.println("Commands:"); Serial.println(" run | pause | resume"); Serial.println(" profile smoke|soak|stress"); + Serial.println(" reinit"); Serial.println(" verify on|off"); Serial.println(" mode all|kv|ts|rel"); Serial.println(" clear kv|ts|rel|all"); @@ -812,6 +944,7 @@ static void set_profile_from_text(const String &arg) { else if (arg == "soak") apply_profile(PROFILE_SOAK); else if (arg == "stress") apply_profile(PROFILE_STRESS); else Serial.println("[ERR] profile must be smoke|soak|stress"); + Serial.println("[INFO] profile affects admission; run 'reinit' (or resetdb/formatdb) to re-admit"); } static void set_verify_from_text(const String &arg) { @@ -946,6 +1079,19 @@ static void format_db() { reset_db(); } +static void reinit_db() { + if (g_store) g_store.flush(); + g_running = false; + g_db_ready = false; + init_cleanup(); + if (!init_db()) { + Serial.println("[ERR] reinit failed; try profile smoke|soak|stress, then reinit, or resetdb"); + return; + } + g_db_ready = true; + Serial.println("[OK] reinit complete"); +} + void setup() { Serial.begin(115200); delay(1200); @@ -958,31 +1104,42 @@ void setup() { g_erase_buf = (uint8_t *)malloc(kEraseSize); #endif if (!g_erase_buf) { - Serial.println("[FATAL] no erase buffer"); + startup_fail("no erase buffer", "enable PSRAM or reduce erase_size buffer"); return; } memset(g_erase_buf, 0xFF, kEraseSize); if (!open_storage_file()) { - Serial.println("[FATAL] SD storage file open failed"); + startup_fail("SD storage file open failed", "check SD wiring, card inserted, and that card is readable (FAT) via SD_MMC"); return; } if (kFreshStartOnBoot) { if (!recreate_storage_file()) { - Serial.println("[FATAL] fresh-start recreate failed"); + startup_fail("fresh-start recreate failed", "try another SD card, check write-protect, or lower storage image size"); return; } Serial.println("[OK] fresh-start storage image created"); } + + /* Apply default bench mix before admission (affects ladder choice via g_profile). */ + apply_profile(PROFILE_SOAK); + if (!init_db()) { - Serial.println("[FATAL] lox_init failed"); + startup_fail("lox_init failed", "try 'profile smoke' (smaller RAM), ensure PSRAM is enabled, or run 'formatdb'"); return; } g_db_ready = true; - apply_profile(PROFILE_SOAK); Serial.println("[OK] loxdb SD stress bench ready"); Serial.printf("SD pins CLK=%d CMD=%d D0=%d D3=%d\n", SDMMC_PIN_CLK, SDMMC_PIN_CMD, SDMMC_PIN_D0, SDMMC_PIN_D3); Serial.printf("LCD pins SCLK=%d MOSI=%d CS=%d DC=%d RST=%d\n", LCD_PIN_SCLK, LCD_PIN_MOSI, LCD_PIN_CS, LCD_PIN_DC, LCD_PIN_RST); + if (g_admitted_ram_kb) { + Serial.printf("ADMISSION ram_kb=%u split=%u/%u/%u wal_th=%u\n", + (unsigned)g_admitted_ram_kb, + (unsigned)g_admitted_kv_pct, + (unsigned)g_admitted_ts_pct, + (unsigned)g_admitted_rel_pct, + (unsigned)g_admitted_wal_th_pct); + } print_usage(); } @@ -994,14 +1151,22 @@ void loop() { char c = (char)Serial.read(); if (c == '\r') continue; if (c == '\n') { - if (cmd == "run" || cmd == "resume") g_running = true; - else if (cmd == "pause") g_running = false; - else if (cmd.startsWith("profile ")) set_profile_from_text(cmd.substring(8)); + if (cmd.startsWith("profile ")) set_profile_from_text(cmd.substring(8)); else if (cmd.startsWith("verify ")) set_verify_from_text(cmd.substring(7)); else if (cmd.startsWith("mode ")) set_mode_from_text(cmd.substring(5)); - else if (cmd.startsWith("clear ")) clear_engine(cmd.substring(6)); else if (cmd == "slist") cmd_slist(); else if (cmd.startsWith("swipe ")) cmd_swipe(cmd.substring(6)); + else if (cmd == "resetdb") reset_db(); + else if (cmd == "formatdb") format_db(); + else if (cmd == "reinit") reinit_db(); + else if (!g_db_ready) { + if (cmd.length() > 0) { + Serial.println("[ERR] database not ready yet; try: profile smoke|soak|stress, reinit, resetdb, formatdb"); + print_usage(); + } + } else if (cmd == "run" || cmd == "resume") g_running = true; + else if (cmd == "pause") g_running = false; + else if (cmd.startsWith("clear ")) clear_engine(cmd.substring(6)); else if (cmd == "compact") { uint32_t t0 = millis(); (void)lox_compact(&g_db); @@ -1009,8 +1174,6 @@ void loop() { g_last_compact_ms = millis() - t0; } else if (cmd == "stats") show_stats(); - else if (cmd == "resetdb") reset_db(); - else if (cmd == "formatdb") format_db(); else if (cmd.length() > 0) print_usage(); cmd = ""; } else { diff --git a/cmake/loxdbConfig.cmake.in b/cmake/loxdbConfig.cmake.in new file mode 100644 index 0000000..9d51f4f --- /dev/null +++ b/cmake/loxdbConfig.cmake.in @@ -0,0 +1,4 @@ +@PACKAGE_INIT@ + +include("${CMAKE_CURRENT_LIST_DIR}/loxdbTargets.cmake") + diff --git a/docs/BENCHMARKS.md b/docs/BENCHMARKS.md new file mode 100644 index 0000000..dfe201b --- /dev/null +++ b/docs/BENCHMARKS.md @@ -0,0 +1,119 @@ +# Benchmarks (ESP32-S3 N16R8) + +This page is the publication home for **measured** benchmark results from the verified ESP32-S3 N16R8 setup. + +It is intentionally a template first: fill it only with real measured numbers from the existing local benchmark runs. + + + +## Test platform + +- Platform: ESP32-S3 N16R8 (16MB NOR flash, 8MB PSRAM) +- ESP-IDF / Arduino core: + - Arduino-ESP32 core `3.3.8` (FQBN `esp32:esp32:esp32s3:...`) +- CPU frequency: + - `240 MHz` +- Flash mode / frequency: + - `QIO @ 80 MHz` (flash size `16MB`) +- Storage backend used: + - In-RAM flash-like storage HAL (see `bench/loxdb_esp32_s3_bench_head/README.md`) + +## Methodology + +- Iterations per measurement: + - +- Latency reporting: + - p50 / p95 / max (microseconds) +- Outliers: + - +- Warmup / cold vs steady: + - + + +## Results - KV engine (deterministic profile) + +| Operation | p50 (us) | p95 (us) | max (us) | throughput (ops/s) | Notes | +|---|---:|---:|---:|---:|---| +| `kv_put` | 26 | 27 | 69 | 38117.9 | `esp32_deterministic_20260511_101754_1a1c569_com19.log` | +| `kv_get` | 9 | 9 | 24 | 113609.5 | `esp32_deterministic_20260511_101754_1a1c569_com19.log` | +| `kv_del` | 22 | 26 | 108 | 42077.6 | `esp32_deterministic_20260511_101754_1a1c569_com19.log` | + +WAL impact (KV): +- `wal_kv_put` p50/p95/max: 32/33/42 us (`esp32_deterministic_20260511_101754_1a1c569_com19.log`) + +## Results - TS engine (deterministic profile) + +| Stream type | insert rate (samples/s) | query p50 (us) | query p95 (us) | Notes | +|---|---:|---:|---:|---| +| `F32` | 52538.0 | 337 | 337 | `esp32_deterministic_20260511_101754_1a1c569_com19.log` (retained=384) | +| `I32` | TBD | TBD | TBD | | +| `U32` | TBD | TBD | TBD | | +| `RAW` | TBD | TBD | TBD | | + +## Results - REL engine (deterministic profile) + +| Rows (N) | insert p50 (us) | find_by_index p50 (us) | Notes | +|---:|---:|---:|---| +| 240 | 25 | 10 | `esp32_deterministic_20260511_101754_1a1c569_com19.log` | + +## WAL / maintenance (deterministic profile) + +| Operation | total (ms) | Notes | +|---|---:|---| +| `compact` | 8.783 | `esp32_deterministic_20260511_101754_1a1c569_com19.log` | +| `reopen` | 12.346 | `esp32_deterministic_20260511_101754_1a1c569_com19.log` | + +## Throughput reference - balanced profile + +| Operation | throughput (ops/s) | Notes | +|---|---:|---| +| `kv_put` | 38025.1 | `esp32_balanced_20260511_101754_1a1c569_com19.log` | +| `kv_get` | 112471.7 | `esp32_balanced_20260511_101754_1a1c569_com19.log` | +| `kv_del` | 42524.0 | `esp32_balanced_20260511_101754_1a1c569_com19.log` | +| `ts_insert` | 32030.4 | `esp32_balanced_20260511_101754_1a1c569_com19.log` | +| `rel_insert` | 13264.4 | `esp32_balanced_20260511_101754_1a1c569_com19.log` | + + +## Stress profile reference + +| Metric | Value | Notes | +|---|---:|---| +| `kv_put` throughput (ops/s) | 38007.7 | `esp32_stress_20260511_102425_1a1c569_com19.log` | +| `kv_get` throughput (ops/s) | 111762.1 | `esp32_stress_20260511_102425_1a1c569_com19.log` | +| `kv_del` throughput (ops/s) | 42958.6 | `esp32_stress_20260511_102425_1a1c569_com19.log` | +| `ts_insert` throughput (samples/s) | 29407.4 | `esp32_stress_20260511_102425_1a1c569_com19.log` (retained=1792) | +| `rel_insert` throughput (rows/s) | 4153.4 | `esp32_stress_20260511_102425_1a1c569_com19.log` (N=1200) | +| `wal_kv_put` throughput (ops/s) | 23837.9 | `esp32_stress_20260511_102425_1a1c569_com19.log` | +| `compact` total (ms) | 22.798 | `esp32_stress_20260511_102425_1a1c569_com19.log` | +| `reopen` total (ms) | 277.816 | `esp32_stress_20260511_102425_1a1c569_com19.log` | + + +## Reproducibility + +Benchmark runner(s) in this repository: + +- `bench/loxdb_esp32_s3_bench_head/` +- `bench/loxdb_esp32_s3_bench_base/` + +Steps to reproduce: + +1. Build and flash the bench sketch for ESP32-S3 N16R8. +2. Run the terminal-driven commands described in the bench README. +3. Copy measured outputs into the tables above (only real numbers; no estimates). + +Optional automation (logs + doc update): + +- `./scripts/run_esp32_bench_and_update_docs.ps1 -Port COM19` + +## Run notes + +- Latest merge-prep verdict: `docs/results/bench_verdict_20260511.md` +- First SD stress run artifacts (2026-05-11): + - `docs/results/esp32_sd_stress_20260511_113124_4d1fd65_com19.md` + - `docs/results/esp32_sd_stress_20260511_113124_4d1fd65_com19.log` + - `docs/results/esp32_sd_stress_20260511_113124_4d1fd65_com19.csv` + +## Related benches + +- SD endurance / pressure stress test: `docs/SD_STRESS_BENCH.md` + diff --git a/docs/DOCS_MAP.md b/docs/DOCS_MAP.md deleted file mode 100644 index 3605baf..0000000 --- a/docs/DOCS_MAP.md +++ /dev/null @@ -1,69 +0,0 @@ -# Documentation Map - -This page is the central navigation entry for `loxdb` docs. - -## Start Here - -- `README.md` (project overview) -- `docs/GETTING_STARTED_5_MIN.md` (quick setup) -- `docs/GETTING_STARTED_DEV_10_MIN.md` (developer onboarding in 10 minutes) -- `docs/PROGRAMMER_MANUAL.md` (full API + architecture) -- `docs/LIMITS_AND_FAILURES.md` (hard limits, invariants, fail behavior) -- `docs/STARTUP_DECISION_FLOW.md` (deterministic init/recovery flow) -- `docs/TROUBLESHOOTING.md` (symptom -> cause -> action) -- `docs/GOLDEN_PROFILES.md` (known-good hardware baselines) - -## Product and Contract Docs - -- `docs/PRODUCT_POSITIONING.md` -- `docs/PRODUCT_BRIEF.md` -- `docs/PROFILE_GUARANTEES.md` -- `docs/FAIL_CODE_CONTRACT.md` -- `docs/FOOTPRINT_MIN_CONTRACT.md` -- `docs/OFFLINE_VERIFIER.md` -- `docs/CORE_INVARIANTS.md` -- `docs/WCET_ANALYSIS.md` -- `docs/SAFETY_READINESS.md` - -## Integration Docs - -- `docs/BACKEND_INTEGRATION_GUIDE.md` -- `docs/PORT_AUTHORING_GUIDE.md` -- `docs/FS_BLOCK_ADAPTER_CONTRACT.md` -- `docs/THREAD_SAFETY.md` - -## Testing and Verification - -- `docs/MEGA_TEST_CHECKLIST_STATUS.md` -- `docs/IMPLEMENTATION_STATUS_VERIFIED.md` -- `docs/MANAGED_STRESS_BASELINES.md` -- `docs/REL_CORRUPTION_CORPUS.md` -- `docs/results/` (historical outputs and verdict artifacts) - -## Release and Repository Ops - -- `CHANGELOG.md` -- `RELEASE_LOG.md` -- `docs/release-notes.md` -- `docs/RELEASE_CHECKLIST.md` -- `docs/RELEASE_TAG_TEMPLATE.md` -- `docs/repository-topics.md` - -## Licensing and Governance - -- `LICENSE` -- `docs/FREE_EDITION_LICENSING.md` -- `CONTRIBUTING.md` -- `SECURITY.md` -- `CODE_OF_CONDUCT.md` -- `SUPPORT.md` - -## Change Workflow - -- `docs/CHANGE_CYCLE_CHECKLIST.md` (per-change implementation/test/docs gate) - -## Cross-Repo Synchronization - -- docs/DOCS_SYNC_PLAN.md (core/pro documentation sync workflow) - - diff --git a/docs/EDITIONS.md b/docs/EDITIONS.md new file mode 100644 index 0000000..29f4dd0 --- /dev/null +++ b/docs/EDITIONS.md @@ -0,0 +1,53 @@ +# loxdb editions + +This repository is the **core** embedded database engine. A planned commercial edition (`loxdb_pro`) is intended to add higher-level operational tooling and workflows **on top of** the core, without duplicating core responsibilities. + +This document is a distillation of: +- `ROOT_SPEC_CORE_VS_PRO.md` +- `docs/LOXDB_PRO_BACKLOG.md` + +## loxdb (this repository) — MIT licensed OSS core + +`loxdb` core is responsible for: + +- deterministic engine behavior (KV / TS / REL) +- strict storage contract and durability semantics (WAL + recovery) +- predictable memory behavior and stable return-code contract (`lox_err_t`) +- a portable storage HAL (`lox_storage_t` callbacks) +- correctness evidence (tests, sanitizer lanes, static analysis) +- small, engine-adjacent optional helpers (for example read-only image verification) + +Non-goals for core: + +- operational control planes, fleet workflows, and “operator UX” +- governance/policy orchestration layers (quotas, policy packs, admission bundles) +- security-at-rest orchestration and tamper-response workflows +- certification packaging pipelines and compliance “kits” +- multi-image media catalogs and filesystem inventory lifecycle in the core public API + +Stability intent: + +- The MIT-licensed core API surface (`lox_*` symbols and current public headers) is intended to remain stable across releases. +- Features shipped in the OSS core are not intended to be removed or relicensed away from MIT in future versions. + +## loxdb_pro — planned commercial edition (separate repository) + +`loxdb_pro` is planned as a separate product/repository, shipping **additional modules** that compose core primitives into operational workflows. + +Target scope (examples, distilled from the PRO backlog): + +- multi-database image management on shared media (SD/eMMC/FS catalogs) + - discovery/scanning, classification (valid/corrupt/non-loxdb), fingerprinting + - lifecycle operations (create/use/rename/clone/delete) with safety rails (dry-run) + - optional manifest/catalog for fast startup and drift reconciliation +- extended observability and operational tooling + - structured diagnostics around scan/open/manage operations + - operator-friendly commands/UX (`db list`, `db info`, `db verify`, etc.) +- commercial support and integration assistance + +Boundary rules (non-duplication): + +- Core exposes narrow, stable primitives; PRO composes them into workflows. +- Core remains policy-neutral and crypto-agnostic; PRO may implement policy/security orchestration. +- PRO must map (not redefine) core error semantics; no separate competing error taxonomy for core behavior. + diff --git a/docs/PROFILES.md b/docs/PROFILES.md new file mode 100644 index 0000000..5cbaf35 --- /dev/null +++ b/docs/PROFILES.md @@ -0,0 +1,71 @@ +# Profiles (compile-time) and footprint-min notes + +This document summarizes the supported compile-time “core profiles” and the smallest **durable** configuration (`LOX_PROFILE_FOOTPRINT_MIN`), without mixing in benchmark numbers. + +Source of truth remains: +- public API and limits: `include/lox.h` +- build variants and test targets: `CMakeLists.txt` +- behavioral evidence: `tests/` + +## Supported core profiles + +Enable **at most one** of: + +- `LOX_PROFILE_CORE_MIN` +- `LOX_PROFILE_CORE_WAL` +- `LOX_PROFILE_CORE_PERF` +- `LOX_PROFILE_CORE_HIMEM` +- `LOX_PROFILE_FOOTPRINT_MIN` + +If none is set, `LOX_PROFILE_CORE_WAL` is selected by default. + +## Engine availability (build-time) + +- KV: available when `LOX_ENABLE_KV=1` +- TS: available when `LOX_ENABLE_TS=1` +- REL: available when `LOX_ENABLE_REL=1` +- WAL/recovery path: available when `LOX_ENABLE_WAL=1` and a storage backend is provided + +## Durable storage contract (current releases) + +Validated at `lox_init()` / open path: + +- `erase_size > 0` +- `write_size == 1` + +If violated, initialization fails with `LOX_ERR_INVALID`. + +## `LOX_PROFILE_FOOTPRINT_MIN` (smallest durable profile) + +Intended behavior: + +- KV enabled +- TS disabled +- REL disabled +- WAL enabled (power-fail/recovery path remains active) + +Important separation: + +- `FOOTPRINT_MIN` is the smallest supported **durable** profile. +- `lox_tiny` is a separate “smallest size” variant (KV-only, WAL-off) and has weaker power-loss durability semantics than WAL-enabled profiles. + +## Footprint-min baseline test intent + +The canonical footprint sanity is `test_footprint_min_baseline` and focuses on: + +1. `init/open` (persistent POSIX storage) +2. a minimal KV set/get +3. `close/deinit` +4. `reopen` +5. KV get (persistence/recovery) + +No benchmark workload and no extra features. + +## Size gates and linkage audit (CI) + +The footprint-min size-gate tests are intended to fail CI if the minimal durable profile exceeds section budgets (Release) or links forbidden objects. + +See: +- `CMakeLists.txt` (size-gate test definitions) +- `tests/` (baseline + gate helpers) + diff --git a/docs/PROGRAMMER_MANUAL.md b/docs/PROGRAMMER_MANUAL.md index c29f12f..5601ce2 100644 --- a/docs/PROGRAMMER_MANUAL.md +++ b/docs/PROGRAMMER_MANUAL.md @@ -84,6 +84,30 @@ Depending on your platform/path: - CMake (main build and variants) - CTest (test execution) +## 3.4 Error codes (public API) + +Public APIs return `lox_err_t` (see `include/lox.h`). Convert to a stable symbolic name via: + +- `lox_err_to_string(lox_err_t)` + +Common codes (high-level intent): + +- `LOX_OK`: success (some recovery paths are “success with recovery performed”) +- `LOX_ERR_INVALID`: invalid argument/handle; storage contract violation at init/open (`erase_size == 0`, `write_size != 1`) +- `LOX_ERR_NO_MEM`: RAM allocation/budget failure during init/profile setup +- `LOX_ERR_FULL`: bounded container capacity reached (for example table max rows) +- `LOX_ERR_NOT_FOUND`: missing key/stream/row +- `LOX_ERR_EXPIRED`: KV value exists but TTL expired +- `LOX_ERR_STORAGE`: backend I/O failure (`read/write/erase/sync`) +- `LOX_ERR_CORRUPT`: unrecoverable persisted corruption detected in strict decode paths +- `LOX_ERR_DISABLED`: feature disabled at compile time +- `LOX_ERR_OVERFLOW`: caller buffer too small +- `LOX_ERR_SCHEMA`: schema mismatch / unsupported migration path +- `LOX_ERR_TXN_ACTIVE`: conflicting transaction state + +Recovery note: +- WAL tail truncation and WAL header reset scenarios are designed to be *recoverable*; callers should treat success as “committed state preserved” rather than “no anomaly happened”. Use the offline verifier in QA gates when needed. + ## 4. Data model and engine semantics ## 4.1 KV engine diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..2a74b4a --- /dev/null +++ b/docs/README.md @@ -0,0 +1,27 @@ +# Documentation index + +Start here: + +- Getting started: `GETTING_STARTED_5_MIN.md` +- Programmer manual: `PROGRAMMER_MANUAL.md` +- Backend integration: `BACKEND_INTEGRATION_GUIDE.md` +- Port authoring (ESP32 reference): `PORT_AUTHORING_GUIDE.md` +- Schema migration: `SCHEMA_MIGRATION_GUIDE.md` + +Other technical notes: + +- Profiles: `PROFILES.md` +- Safety notes: `SAFETY_NOTES.md` +- Offline verifier: `OFFLINE_VERIFIER.md` +- WCET analysis: `WCET_ANALYSIS.md` +- Benchmarks (results publication): `BENCHMARKS.md` + +Internal/process documents live in `docs/internal/`. + +## Distribution (planned) + +Publishing is maintainer-driven and not automated yet: + +- PlatformIO Registry (`library.json`) +- Arduino Library Manager (`library.properties`) +- CMake install + `find_package(loxdb)` via installed config files (see `CMakeLists.txt` and `cmake/loxdbConfig.cmake.in`) diff --git a/docs/SAFETY_NOTES.md b/docs/SAFETY_NOTES.md new file mode 100644 index 0000000..a5222d9 --- /dev/null +++ b/docs/SAFETY_NOTES.md @@ -0,0 +1,37 @@ +# Safety notes (non-certified) + +`loxdb` is **not** a certified safety/security library. This document is an honest summary of what the repository *does* and *does not* provide today, so integrators can make appropriate project-level decisions. + +## What is tested + +Evidence that exists in this repository: + +- Deterministic return-code contract (`lox_err_t`) validated by tests (see `tests/`). +- Durability and recovery behavior validated by WAL/recovery tests. +- Multi-profile/configuration coverage via build-time/profile matrices (see `CMakeLists.txt`). +- Sanitizer lane on Linux (ASan/UBSan) in CI. +- Static analysis via cppcheck in CI (non-blocking lanes today). +- Read-only offline verifier (`lox_verify`) for persisted images (see `docs/OFFLINE_VERIFIER.md`). + +## What is *not* claimed + +No claims are made about: + +- compliance with any specific safety/security standard +- MISRA compliance status +- tool qualification packages +- a complete safety case / hazard analysis / threat model + +If you need those, you must establish them at the product/program level and treat this library as a component within that process. + +## Integration expectations for safety-critical use + +If you consider deploying in a safety- or mission-critical context, typical minimum expectations include: + +1. Run the full test suite on your production toolchain and flags. +2. Validate the storage HAL contract on your real media (`erase_size`, `write_size`, `read/write/erase/sync`). +3. Provide real locking when concurrency is possible (`LOX_THREAD_SAFE=1` + lock hooks). +4. Execute power-loss testing around WAL replay and compaction boundaries on your target hardware. +5. Pin and document configuration (profile, RAM/storage budgets, split percentages, retention policies). +6. Add your own fault-injection, stress, and long-duration validation gates appropriate to the product. + diff --git a/docs/SD_STRESS_BENCH.md b/docs/SD_STRESS_BENCH.md new file mode 100644 index 0000000..e03b8e7 --- /dev/null +++ b/docs/SD_STRESS_BENCH.md @@ -0,0 +1,41 @@ +# SD stress bench (ESP32-S3 N16R8, SD_MMC) + +This benchmark is a **long-running real-hardware stress test** that uses SD_MMC storage (persistent file) and continuously writes mixed `KV`/`TS`/`REL` workload while reporting live utilization. + +It is complementary to `docs/BENCHMARKS.md`: + +- `docs/BENCHMARKS.md` focuses on short, reproducible latency/throughput micro-bench runs. +- SD stress bench focuses on endurance, pressure behavior, compaction, and long-run stability on real SD media. + +## Hardware / wiring + +See `bench/loxdb_esp32_s3_sd_stress_bench/README.md`. + +## How to run (automated logging) + +1. Flash the sketch: + - `bench/loxdb_esp32_s3_sd_stress_bench/loxdb_esp32_s3_sd_stress_bench.ino` +2. Run the logger: + + - `./scripts/run_sd_stress_bench.ps1 -Port COM19 -DurationSec 600 -Profile soak -Mode all -Verify on -ResetDb` + - If you omit `-ResetDb` / `-FormatDb`, the logger will run `reinit` to re-admit the selected profile without wiping the SD image. + +If you see a message like "Detected terminal bench firmware (loxdb-bench>)", it means the board is running the other bench sketch (HEAD/BASE terminal bench). Re-flash the SD stress sketch and retry. + +Artifacts are written to `docs/results/`: + +- raw serial log: `esp32_sd_stress___com19.log` +- pressure CSV: `esp32_sd_stress___com19.csv` (parsed from `[PRESSURE]` lines) +- short run note: `esp32_sd_stress___com19.md` + +## What to look at + +From the raw log: + +- `[PRESSURE] kv/ts/rel/wal/risk ops=...` (pressure trend over time) +- `[STATS] kv_entries / ts_samples / rel_rows / wal_bytes` (capacity + growth) +- `[BENCH] ... compact= last_compact_ms= ok/fail=` (compaction + verification health) + +From the CSV: + +- plot `ops` over time vs `risk_pct`, `wal_pct` to see sustained ingest and near-full behavior. diff --git a/docs/TEST_SUITE_SIZE.md b/docs/TEST_SUITE_SIZE.md new file mode 100644 index 0000000..53b14a4 --- /dev/null +++ b/docs/TEST_SUITE_SIZE.md @@ -0,0 +1,29 @@ +# Test suite size (measured) + +The repo uses CMake/CTest, but the most meaningful “test case” unit is the in-repo microtest harness (`tests/microtest.h`) executed via `MDB_RUN_TEST(...)`. + +Use this exact phrasing in user-facing docs: + +> **504 microtest cases across 48 test files (+1 C++ wrapper test), organized into ~78 CTest entries including RAM-budget sweep matrices.** + +## Breakdown + +- **Microtest cases (504):** exact count of `MDB_RUN_TEST(` call sites across `tests/*.c`. +- **Test files (48 + 1):** + - `48` × `tests/test_*.c` + - `+1` × `tests/test_*.cpp` (`tests/test_cpp_wrapper.cpp`) +- **CTest entries (~78):** + - Root `CMakeLists.txt` contains `72` textual `add_test` tokens. + - Two `foreach(RAM_KB 128 256 512 1024)` matrices generate `4 + 4` additional configured CTest entries: + - `integration_${RAM_KB}kb` + - `limits_${RAM_KB}kb` + +## Re-measuring (quick) + +- Microtest cases: count occurrences of `MDB_RUN_TEST(` under `tests/`. +- Effective configured CTest list for a preset: configure + `ctest -N` (e.g. `cmake --preset ci-debug-linux` then `ctest --preset ci-debug-linux -N`). + +Notes: +- The microtest-case count is stable and meaningful. +- The effective CTest entry count may vary with optional/conditional targets enabled at configure time. + diff --git a/docs/CHANGE_CYCLE_CHECKLIST.md b/docs/internal/CHANGE_CYCLE_CHECKLIST.md similarity index 96% rename from docs/CHANGE_CYCLE_CHECKLIST.md rename to docs/internal/CHANGE_CYCLE_CHECKLIST.md index 3043743..f0b43f0 100644 --- a/docs/CHANGE_CYCLE_CHECKLIST.md +++ b/docs/internal/CHANGE_CYCLE_CHECKLIST.md @@ -24,7 +24,7 @@ Use this checklist for every change batch. ## 5) Documentation Sync - [ ] Update canonical core docs (`LIMITS_AND_FAILURES`, `STARTUP_DECISION_FLOW`, etc.). -- [ ] Update `docs/DOCS_MAP.md` if new docs were added. +- [ ] Update `docs/README.md` if new docs were added. - [ ] Verify cross-repo links per `docs/DOCS_SYNC_PLAN.md`. ## 6) Release Hygiene diff --git a/docs/DOCS_SYNC_PLAN.md b/docs/internal/DOCS_SYNC_PLAN.md similarity index 100% rename from docs/DOCS_SYNC_PLAN.md rename to docs/internal/DOCS_SYNC_PLAN.md diff --git a/docs/FAIL_CODE_CONTRACT.md b/docs/internal/FAIL_CODE_CONTRACT.md similarity index 100% rename from docs/FAIL_CODE_CONTRACT.md rename to docs/internal/FAIL_CODE_CONTRACT.md diff --git a/docs/FOOTPRINT_MIN_CONTRACT.md b/docs/internal/FOOTPRINT_MIN_CONTRACT.md similarity index 100% rename from docs/FOOTPRINT_MIN_CONTRACT.md rename to docs/internal/FOOTPRINT_MIN_CONTRACT.md diff --git a/docs/LOXDB_PRO_BACKLOG.md b/docs/internal/LOXDB_PRO_BACKLOG.md similarity index 100% rename from docs/LOXDB_PRO_BACKLOG.md rename to docs/internal/LOXDB_PRO_BACKLOG.md diff --git a/docs/internal/MAINTAINER_TODO.md b/docs/internal/MAINTAINER_TODO.md new file mode 100644 index 0000000..bbef47c --- /dev/null +++ b/docs/internal/MAINTAINER_TODO.md @@ -0,0 +1,55 @@ +# Maintainer TODO (repo settings + follow-ups) + +This file lists actions that require repository admin access or infrastructure decisions and therefore cannot be done from a docs-only PR. + +## GitHub repository settings checklist + +1. **Description (About)** + - Suggested text: + - Predictable-memory embedded database for microcontrollers. KV, time-series, and relational engines with WAL recovery. C99, zero dependencies, verified on ESP32-S3. + +2. **Topics** + - Suggested topics: + - `embedded-database`, `wal`, `flash-storage`, `microcontroller`, `esp32`, `c99` + - (optional) `littlefs-alternative` + +3. **Wiki** + - Disable GitHub Wiki in repo settings. + - Keep `wiki/` (in-repo Markdown) as the single source of truth for wiki content. + - Rationale: avoids double maintenance and drift. + +4. **Discussions** + - Keep enabled. + - Add a short pinned “Welcome / How to ask for help” post pointing to: + - the minimal repro expectations + - target platform details (MCU, storage backend, erase/write sizes) + +5. **Releases** + - For `v1.4.0`, ensure GitHub Release notes are complete and not duplicated in repo-root process docs. + +6. **Social preview image** + - Create a 1280×640 social preview image: project name + one-line technical tagline (no marketing contract language). + +## Documentation surface policy + +- Keep `README.md` short and technical. +- Keep “process” and “status artifact” docs under `docs/internal/`. +- Keep `docs/` for technical docs users actually need to integrate and ship. +- Keep `docs/results/` as a working directory for tooling outputs, but avoid linking to it from top-level docs. + +## CI roadmap (future; infrastructure decisions required) + +Not implemented in this PR: + +1. **Coverage lane** + - Add a dedicated CMake preset `ci-coverage-linux` (`-O0 -g --coverage`) and a CI job that runs *only* that preset, then uploads coverage (Codecov/Coveralls). + - Do not mix coverage flags into sanitizer lanes. + - Optional (badge): enable Codecov for the repo, then add `codecov/codecov-action` upload step and a Codecov badge to `README.md`. + +2. **Hardware-in-the-loop (HIL) or emulator lane** + - Decide between: + - ESP32-S3 hardware-in-the-loop runner (self-hosted or farm), or + - a QEMU-based lane (if/when ESP32-S3 support is feasible), or + - a hybrid approach (QEMU smoke + periodic HIL). + - Define the gate: smoke-only, nightly, or release-only. + - Determine how artifacts (serial logs, crash dumps, perf outputs) are captured and retained. diff --git a/docs/PRODUCT_BRIEF.md b/docs/internal/PRODUCT_BRIEF.md similarity index 100% rename from docs/PRODUCT_BRIEF.md rename to docs/internal/PRODUCT_BRIEF.md diff --git a/docs/PRODUCT_POSITIONING.md b/docs/internal/PRODUCT_POSITIONING.md similarity index 100% rename from docs/PRODUCT_POSITIONING.md rename to docs/internal/PRODUCT_POSITIONING.md diff --git a/docs/PROFESSIONAL_READINESS.md b/docs/internal/PROFESSIONAL_READINESS.md similarity index 100% rename from docs/PROFESSIONAL_READINESS.md rename to docs/internal/PROFESSIONAL_READINESS.md diff --git a/docs/PROFILE_GUARANTEES.md b/docs/internal/PROFILE_GUARANTEES.md similarity index 100% rename from docs/PROFILE_GUARANTEES.md rename to docs/internal/PROFILE_GUARANTEES.md diff --git a/docs/RELEASE_CHECKLIST.md b/docs/internal/RELEASE_CHECKLIST.md similarity index 100% rename from docs/RELEASE_CHECKLIST.md rename to docs/internal/RELEASE_CHECKLIST.md diff --git a/RELEASE_LOG.md b/docs/internal/RELEASE_LOG.md similarity index 97% rename from RELEASE_LOG.md rename to docs/internal/RELEASE_LOG.md index c9c43fb..b27b398 100644 --- a/RELEASE_LOG.md +++ b/docs/internal/RELEASE_LOG.md @@ -1,7 +1,7 @@ # Release Log This file tracks release-level outcomes and notable delivery notes. -For detailed code-level change history, see [CHANGELOG.md](CHANGELOG.md). +For detailed code-level change history, see [CHANGELOG.md](../../CHANGELOG.md). ## Unreleased diff --git a/docs/RELEASE_TAG_TEMPLATE.md b/docs/internal/RELEASE_TAG_TEMPLATE.md similarity index 100% rename from docs/RELEASE_TAG_TEMPLATE.md rename to docs/internal/RELEASE_TAG_TEMPLATE.md diff --git a/ROOT_SPEC_CORE_VS_PRO.md b/docs/internal/ROOT_SPEC_CORE_VS_PRO.md similarity index 100% rename from ROOT_SPEC_CORE_VS_PRO.md rename to docs/internal/ROOT_SPEC_CORE_VS_PRO.md diff --git a/docs/SAFETY_READINESS.md b/docs/internal/SAFETY_READINESS.md similarity index 100% rename from docs/SAFETY_READINESS.md rename to docs/internal/SAFETY_READINESS.md diff --git a/TODO.md b/docs/internal/TODO.md similarity index 100% rename from TODO.md rename to docs/internal/TODO.md diff --git a/docs/release-notes.md b/docs/internal/release-notes.md similarity index 100% rename from docs/release-notes.md rename to docs/internal/release-notes.md diff --git a/docs/results/bench_verdict_20260511.md b/docs/results/bench_verdict_20260511.md new file mode 100644 index 0000000..c0cafee --- /dev/null +++ b/docs/results/bench_verdict_20260511.md @@ -0,0 +1,57 @@ +# Bench verdict (ESP32-S3 N16R8) — 2026-05-11 + +This note summarizes the ESP32-S3 N16R8 benchmark runs executed on `COM19` on **2026-05-11** for merge readiness. + +Repo state: + +- repo commit: `1a1c569` +- Arduino-ESP32 core: `3.3.8` +- FQBN: `esp32:esp32:esp32s3:CDCOnBoot=cdc,FlashSize=16M,PSRAM=opi` +- CPU: `240 MHz` +- Flash: `QIO @ 80 MHz` + +## Artifacts + +HEAD bench (from `bench/loxdb_esp32_s3_bench_head/`): + +- deterministic: `docs/results/esp32_deterministic_20260511_101754_1a1c569_com19.log` +- balanced: `docs/results/esp32_balanced_20260511_101754_1a1c569_com19.log` +- stress: `docs/results/esp32_stress_20260511_102425_1a1c569_com19.log` + +BASE bench (from `bench/loxdb_esp32_s3_bench_base/`): + +- deterministic: `docs/results/esp32_deterministic_20260511_104504_1a1c569_com19.log` +- balanced: `docs/results/esp32_balanced_20260511_104504_1a1c569_com19.log` +- stress: `docs/results/esp32_stress_20260511_104504_1a1c569_com19.log` + +## Findings + +- HEAD is substantially faster than BASE for write-heavy KV + WAL paths on this setup (deterministic + balanced). +- `kv_get` is roughly neutral between HEAD and BASE. +- Stress runs show tail spikes in both, but BASE stress shows much larger extremes in KV/WAL and much smaller `wal_total` in its effective config (`8160B` vs `32736B` in HEAD stress). + +### Deterministic profile (p50 / ops/s) + +| Metric | HEAD | BASE | Notes | +|---|---:|---:|---| +| `kv_put` p50 (us) | 26 | 64 | ~2.46× faster on HEAD | +| `kv_del` p50 (us) | 22 | 97 | ~4.41× faster on HEAD | +| `wal_kv_put` p50 (us) | 32 | 70 | ~2.19× faster on HEAD | +| `kv_get` p50 (us) | 9 | 8 | neutral | +| `kv_put` ops/s | 38117.9 | 15523.9 | ~2.46× higher on HEAD | +| `kv_del` ops/s | 42077.6 | 10114.3 | ~4.16× higher on HEAD | + +### Balanced profile (throughput ops/s) + +| Metric | HEAD | BASE | Notes | +|---|---:|---:|---| +| `kv_put` ops/s | 38025.1 | 15450.7 | ~2.46× higher on HEAD | +| `kv_del` ops/s | 42524.0 | 10082.2 | ~4.22× higher on HEAD | +| `ts_insert` ops/s | 32030.4 | 30923.9 | ~1.04× higher on HEAD | +| `rel_insert` ops/s | 13264.4 | 12395.3 | ~1.07× higher on HEAD | + +### Stress profile notes + +- HEAD stress: KV ops capped to capacity (`kv_capacity=248`) and TS drops samples once the TS arena fills (`retained=1792 dropped=608` in the run). +- BASE stress: KV shows large max spikes (`kv_put max=7439us`, `kv_del max=7737us`), and WAL shows a large max spike (`wal_kv_put max=15866us`); also `wal_total` effective size is much smaller (`8160B`). + diff --git a/docs/results/esp32_balanced_20260511_101754_1a1c569_com19.log b/docs/results/esp32_balanced_20260511_101754_1a1c569_com19.log new file mode 100644 index 0000000..1904c57 --- /dev/null +++ b/docs/results/esp32_balanced_20260511_101754_1a1c569_com19.log @@ -0,0 +1,58 @@ +loxdb-bench> [PROFILE] switched to balanced paced=OFF +[OK] DB ready (wipe=1, profile=balanced) +loxdb-bench> +=== loxdb ESP32-S3 benchmark start (profile=balanced) === +[EFFECTIVE] kv_capacity=248 (target=320) wal_total=32736B +[KV] capped ops to capacity: 248 +[BENCH] kv_put total=6.522 ms avg=26.298 us p50=26 p95=27 min=24 max=58 max_op~0 xmax/p50=2.2 spk>1ms=0@0 spk>5ms=0@0 ops/s=38025.1 MB/s=0.145 ops=248 samp=248 heap_d=0 +[SLO] kv_put OK (max=58<=15000, spk>5ms=0<=12) +[PHASE] kv_put cold_ops=64 cold_avg=26.312 us steady_ops=184 steady_avg=26.293 us +[BENCH] kv_get total=2.205 ms avg=8.891 us p50=9 p95=9 min=6 max=23 max_op~11 xmax/p50=2.6 spk>1ms=0@0 spk>5ms=0@0 ops/s=112471.7 MB/s=0.429 ops=248 samp=248 heap_d=0 +[SLO] kv_get OK (max=23<=15000, spk>5ms=0<=12) +[PHASE] kv_get cold_ops=64 cold_avg=8.781 us steady_ops=184 steady_avg=8.929 us +[BENCH] kv_del total=5.832 ms avg=23.516 us p50=22 p95=26 min=20 max=118 max_op~124 xmax/p50=5.4 spk>1ms=0@0 spk>5ms=0@0 ops/s=42524.0 MB/s=0.000 ops=248 samp=248 heap_d=0 +[SLO] kv_del OK (max=118<=15000, spk>5ms=0<=12) +[PHASE] kv_del cold_ops=64 cold_avg=22.344 us steady_ops=184 steady_avg=23.924 us +[CHECK] KV benchmark: PASS +[BENCH] ts_insert total=19.981 ms avg=31.220 us p50=19 p95=19 min=18 max=7797 max_op~494 xmax/p50=410.4 spk>1ms=1@494 spk>5ms=1@494 ops/s=32030.4 MB/s=0.122 ops=640 samp=640 heap_d=0 +[SLO] ts_insert OK (max=7797<=15000, spk>5ms=1<=12) +[PHASE] ts_insert cold_ops=64 cold_avg=19.062 us steady_ops=576 steady_avg=32.571 us +[TS] target=640 retained=640 dropped=0 +[BENCH] ts_query_buf total=0.542 ms avg=542.000 us p50=542 p95=542 min=542 max=542 max_op~0 xmax/p50=1.0 spk>1ms=0@0 spk>5ms=0@0 ops/s=1845.0 MB/s=2.252 ops=1 samp=1 heap_d=0 +[CHECK] TS benchmark: PASS +[BENCH] rel_insert total=37.695 ms avg=75.390 us p50=29 p95=200 min=22 max=219 max_op~260 xmax/p50=7.6 spk>1ms=0@0 spk>5ms=0@0 ops/s=13264.4 MB/s=0.101 ops=500 samp=500 heap_d=0 +[SLO] rel_insert OK (max=219<=15000, spk>5ms=0<=12) +[PHASE] rel_insert cold_ops=64 cold_avg=23.719 us steady_ops=436 steady_avg=82.975 us +[BENCH] rel_find(index) total=0.013 ms avg=13.000 us p50=13 p95=13 min=13 max=13 max_op~0 xmax/p50=1.0 spk>1ms=0@0 spk>5ms=0@0 ops/s=76923.1 MB/s=0.587 ops=1 samp=1 heap_d=0 +[REL] rows_expected=500 rows_actual=500 +[CHECK] REL benchmark: PASS +[WAL] baseline before warmup: used=0 total=32736 fill=0% +[BENCH] wal_kv_put total=13.072 ms avg=34.042 us p50=34 p95=38 min=32 max=50 max_op~0 xmax/p50=1.5 spk>1ms=0@0 spk>5ms=0@0 ops/s=29375.8 MB/s=0.896 ops=384 samp=77 heap_d=0 +[SLO] wal_kv_put OK (max=50<=15000, spk>5ms=0<=12) +[WAL] warmup target_fill=75% reached=75% peak=75% ops_done=384/9600 steady_ops=320 (min=128) +[PHASE] wal_kv_put cold_ops=64 cold_avg=34.234 us steady_ops=320 steady_avg=34.003 us +[WAL] before compact: used=24624 total=32736 fill=75% +[BENCH] compact total=12.255 ms avg=12255.000 us p50=12255 p95=12255 min=12255 max=12255 max_op~0 xmax/p50=1.0 spk>1ms=1@0 spk>5ms=1@0 ops/s=81.6 MB/s=0.000 ops=1 samp=1 heap_d=0 +[WAL] after compact: used=0 total=32736 fill=0% +[CHECK] WAL compact: PASS +[BENCH] reopen total=43.514 ms avg=43514.000 us p50=43514 p95=43514 min=43514 max=43514 max_op~0 xmax/p50=1.0 spk>1ms=1@0 spk>5ms=1@0 ops/s=23.0 MB/s=0.000 ops=1 samp=1 heap_d=0 +[CHECK] Reopen integrity: PASS +[MIGRATE] calls=1 old=1 new=2 +[CHECK] Migration callback: PASS +[CHECK] TXN commit/rollback: PASS +[STATS] kv=202/248 (81%) coll=0 evict=0 +[STATS] ts_streams=1 ts_samples=640 ts_fill=39% +[STATS] wal=96/32736 (0%) rel_tables=2 rel_rows=500 +=== loxdb ESP32-S3 benchmark end === +loxdb-bench> [METRICS] count=10 +[METRIC] kv_put total=6.522ms avg=26.298us p50=26 p95=27 max=58@0 xmax/p50=2.2 spk>1ms=0@0 spk>5ms=0@0 ops/s=38025.1 MB/s=0.145 heap_d=0 +[METRIC] kv_get total=2.205ms avg=8.891us p50=9 p95=9 max=23@11 xmax/p50=2.6 spk>1ms=0@0 spk>5ms=0@0 ops/s=112471.7 MB/s=0.429 heap_d=0 +[METRIC] kv_del total=5.832ms avg=23.516us p50=22 p95=26 max=118@124 xmax/p50=5.4 spk>1ms=0@0 spk>5ms=0@0 ops/s=42524.0 MB/s=0.000 heap_d=0 +[METRIC] ts_insert total=19.981ms avg=31.220us p50=19 p95=19 max=7797@494 xmax/p50=410.4 spk>1ms=1@494 spk>5ms=1@494 ops/s=32030.4 MB/s=0.122 heap_d=0 +[METRIC] ts_query_buf total=0.542ms avg=542.000us p50=542 p95=542 max=542@0 xmax/p50=1.0 spk>1ms=0@0 spk>5ms=0@0 ops/s=1845.0 MB/s=2.252 heap_d=0 +[METRIC] rel_insert total=37.695ms avg=75.390us p50=29 p95=200 max=219@260 xmax/p50=7.6 spk>1ms=0@0 spk>5ms=0@0 ops/s=13264.4 MB/s=0.101 heap_d=0 +[METRIC] rel_find(index) total=0.013ms avg=13.000us p50=13 p95=13 max=13@0 xmax/p50=1.0 spk>1ms=0@0 spk>5ms=0@0 ops/s=76923.1 MB/s=0.587 heap_d=0 +[METRIC] wal_kv_put total=13.072ms avg=34.042us p50=34 p95=38 max=50@0 xmax/p50=1.5 spk>1ms=0@0 spk>5ms=0@0 ops/s=29375.8 MB/s=0.896 heap_d=0 +[METRIC] compact total=12.255ms avg=12255.000us p50=12255 p95=12255 max=12255@0 xmax/p50=1.0 spk>1ms=1@0 spk>5ms=1@0 ops/s=81.6 MB/s=0.000 heap_d=0 +[METRIC] reopen total=43.514ms avg=43514.000us p50=43514 p95=43514 max=43514@0 xmax/p50=1.0 spk>1ms=1@0 spk>5ms=1@0 ops/s=23.0 MB/s=0.000 heap_d=0 +loxdb-bench> \ No newline at end of file diff --git a/docs/results/esp32_balanced_20260511_104504_1a1c569_com19.log b/docs/results/esp32_balanced_20260511_104504_1a1c569_com19.log new file mode 100644 index 0000000..630d1c1 --- /dev/null +++ b/docs/results/esp32_balanced_20260511_104504_1a1c569_com19.log @@ -0,0 +1,57 @@ +loxdb-bench> [PROFILE] switched to balanced paced=OFF +[OK] DB ready (wipe=1, profile=balanced) +loxdb-bench> +=== loxdb ESP32-S3 benchmark start (profile=balanced) === +[EFFECTIVE] kv_capacity=376 (target=320) wal_total=32736B +[BENCH] kv_put total=20.711 ms avg=64.722 us p50=65 p95=68 min=61 max=111 max_op~0 xmax/p50=1.7 spk>1ms=0@0 spk>5ms=0@0 ops/s=15450.7 MB/s=0.059 ops=320 samp=320 heap_d=0 +[SLO] kv_put OK (max=111<=15000, spk>5ms=0<=12) +[PHASE] kv_put cold_ops=64 cold_avg=64.062 us steady_ops=256 steady_avg=64.887 us +[BENCH] kv_get total=2.858 ms avg=8.931 us p50=9 p95=10 min=6 max=24 max_op~15 xmax/p50=2.7 spk>1ms=0@0 spk>5ms=0@0 ops/s=111966.4 MB/s=0.427 ops=320 samp=320 heap_d=0 +[SLO] kv_get OK (max=24<=15000, spk>5ms=0<=12) +[PHASE] kv_get cold_ops=64 cold_avg=8.938 us steady_ops=256 steady_avg=8.930 us +[BENCH] kv_del total=31.739 ms avg=99.184 us p50=97 p95=103 min=96 max=197 max_op~160 xmax/p50=2.0 spk>1ms=0@0 spk>5ms=0@0 ops/s=10082.2 MB/s=0.000 ops=320 samp=320 heap_d=0 +[SLO] kv_del OK (max=197<=15000, spk>5ms=0<=12) +[PHASE] kv_del cold_ops=64 cold_avg=99.234 us steady_ops=256 steady_avg=99.172 us +[CHECK] KV benchmark: PASS +[BENCH] ts_insert total=20.696 ms avg=32.338 us p50=20 p95=21 min=20 max=7598 max_op~374 xmax/p50=379.9 spk>1ms=1@374 spk>5ms=1@374 ops/s=30923.9 MB/s=0.118 ops=640 samp=640 heap_d=0 +[SLO] ts_insert OK (max=7598<=15000, spk>5ms=1<=12) +[PHASE] ts_insert cold_ops=64 cold_avg=21.375 us steady_ops=576 steady_avg=33.556 us +[TS] target=640 retained=640 dropped=0 +[BENCH] ts_query_buf total=0.152 ms avg=152.000 us p50=152 p95=152 min=152 max=152 max_op~0 xmax/p50=1.0 spk>1ms=0@0 spk>5ms=0@0 ops/s=6578.9 MB/s=8.031 ops=1 samp=1 heap_d=0 +[CHECK] TS benchmark: PASS +[BENCH] rel_insert total=40.338 ms avg=80.676 us p50=32 p95=206 min=23 max=226 max_op~258 xmax/p50=7.1 spk>1ms=0@0 spk>5ms=0@0 ops/s=12395.3 MB/s=0.095 ops=500 samp=500 heap_d=0 +[SLO] rel_insert OK (max=226<=15000, spk>5ms=0<=12) +[PHASE] rel_insert cold_ops=64 cold_avg=25.453 us steady_ops=436 steady_avg=88.782 us +[BENCH] rel_find(index) total=0.026 ms avg=26.000 us p50=26 p95=26 min=26 max=26 max_op~0 xmax/p50=1.0 spk>1ms=0@0 spk>5ms=0@0 ops/s=38461.5 MB/s=0.293 ops=1 samp=1 heap_d=0 +[REL] rows_expected=500 rows_actual=500 +[CHECK] REL benchmark: PASS +[WAL] baseline before warmup: used=0 total=32736 fill=0% +[BENCH] wal_kv_put total=27.707 ms avg=72.154 us p50=72 p95=76 min=69 max=83 max_op~0 xmax/p50=1.2 spk>1ms=0@0 spk>5ms=0@0 ops/s=13859.3 MB/s=0.423 ops=384 samp=77 heap_d=0 +[SLO] wal_kv_put OK (max=83<=15000, spk>5ms=0<=12) +[WAL] warmup target_fill=75% reached=75% peak=75% ops_done=384/9600 steady_ops=320 (min=128) +[PHASE] wal_kv_put cold_ops=64 cold_avg=71.266 us steady_ops=320 steady_avg=72.331 us +[WAL] before compact: used=24624 total=32736 fill=75% +[BENCH] compact total=12.614 ms avg=12614.000 us p50=12614 p95=12614 min=12614 max=12614 max_op~0 xmax/p50=1.0 spk>1ms=1@0 spk>5ms=1@0 ops/s=79.3 MB/s=0.000 ops=1 samp=1 heap_d=0 +[WAL] after compact: used=0 total=32736 fill=0% +[CHECK] WAL compact: PASS +[BENCH] reopen total=53.102 ms avg=53102.000 us p50=53102 p95=53102 min=53102 max=53102 max_op~0 xmax/p50=1.0 spk>1ms=1@0 spk>5ms=1@0 ops/s=18.8 MB/s=0.000 ops=1 samp=1 heap_d=0 +[CHECK] Reopen integrity: PASS +[MIGRATE] calls=1 old=1 new=2 +[CHECK] Migration callback: PASS +[CHECK] TXN commit/rollback: PASS +[STATS] kv=202/376 (53%) coll=43 evict=0 +[STATS] ts_streams=1 ts_samples=640 ts_fill=12% +[STATS] wal=96/32736 (0%) rel_tables=2 rel_rows=500 +=== loxdb ESP32-S3 benchmark end === +loxdb-bench> [METRICS] count=10 +[METRIC] kv_put total=20.711ms avg=64.722us p50=65 p95=68 max=111@0 xmax/p50=1.7 spk>1ms=0@0 spk>5ms=0@0 ops/s=15450.7 MB/s=0.059 heap_d=0 +[METRIC] kv_get total=2.858ms avg=8.931us p50=9 p95=10 max=24@15 xmax/p50=2.7 spk>1ms=0@0 spk>5ms=0@0 ops/s=111966.4 MB/s=0.427 heap_d=0 +[METRIC] kv_del total=31.739ms avg=99.184us p50=97 p95=103 max=197@160 xmax/p50=2.0 spk>1ms=0@0 spk>5ms=0@0 ops/s=10082.2 MB/s=0.000 heap_d=0 +[METRIC] ts_insert total=20.696ms avg=32.338us p50=20 p95=21 max=7598@374 xmax/p50=379.9 spk>1ms=1@374 spk>5ms=1@374 ops/s=30923.9 MB/s=0.118 heap_d=0 +[METRIC] ts_query_buf total=0.152ms avg=152.000us p50=152 p95=152 max=152@0 xmax/p50=1.0 spk>1ms=0@0 spk>5ms=0@0 ops/s=6578.9 MB/s=8.031 heap_d=0 +[METRIC] rel_insert total=40.338ms avg=80.676us p50=32 p95=206 max=226@258 xmax/p50=7.1 spk>1ms=0@0 spk>5ms=0@0 ops/s=12395.3 MB/s=0.095 heap_d=0 +[METRIC] rel_find(index) total=0.026ms avg=26.000us p50=26 p95=26 max=26@0 xmax/p50=1.0 spk>1ms=0@0 spk>5ms=0@0 ops/s=38461.5 MB/s=0.293 heap_d=0 +[METRIC] wal_kv_put total=27.707ms avg=72.154us p50=72 p95=76 max=83@0 xmax/p50=1.2 spk>1ms=0@0 spk>5ms=0@0 ops/s=13859.3 MB/s=0.423 heap_d=0 +[METRIC] compact total=12.614ms avg=12614.000us p50=12614 p95=12614 max=12614@0 xmax/p50=1.0 spk>1ms=1@0 spk>5ms=1@0 ops/s=79.3 MB/s=0.000 heap_d=0 +[METRIC] reopen total=53.102ms avg=53102.000us p50=53102 p95=53102 max=53102@0 xmax/p50=1.0 spk>1ms=1@0 spk>5ms=1@0 ops/s=18.8 MB/s=0.000 heap_d=0 +loxdb-bench> \ No newline at end of file diff --git a/docs/results/esp32_deterministic_20260511_101754_1a1c569_com19.log b/docs/results/esp32_deterministic_20260511_101754_1a1c569_com19.log new file mode 100644 index 0000000..bf50fe3 --- /dev/null +++ b/docs/results/esp32_deterministic_20260511_101754_1a1c569_com19.log @@ -0,0 +1,59 @@ +loxdb-bench> [OK] DB ready (wipe=1, profile=deterministic) +loxdb-bench> [NOTE] run_det validates deterministic profile latency, not all profiles/workloads. +[PACED] mode=OFF +[OK] DB ready (wipe=1, profile=deterministic) + +=== loxdb ESP32-S3 benchmark start (profile=deterministic) === +[EFFECTIVE] kv_capacity=248 (target=192) wal_total=32736B +[BENCH] kv_put total=5.037 ms avg=26.234 us p50=26 p95=27 min=24 max=69 max_op~0 xmax/p50=2.7 spk>1ms=0@0 spk>5ms=0@0 ops/s=38117.9 MB/s=0.145 ops=192 samp=192 heap_d=0 +[SLO] kv_put OK (max=69<=5000, spk>5ms=0<=0) +[PHASE] kv_put cold_ops=64 cold_avg=26.266 us steady_ops=128 steady_avg=26.219 us +[BENCH] kv_get total=1.690 ms avg=8.802 us p50=9 p95=9 min=6 max=24 max_op~15 xmax/p50=2.7 spk>1ms=0@0 spk>5ms=0@0 ops/s=113609.5 MB/s=0.433 ops=192 samp=192 heap_d=0 +[SLO] kv_get OK (max=24<=5000, spk>5ms=0<=0) +[PHASE] kv_get cold_ops=64 cold_avg=8.750 us steady_ops=128 steady_avg=8.828 us +[BENCH] kv_del total=4.563 ms avg=23.766 us p50=22 p95=26 min=20 max=108 max_op~96 xmax/p50=4.9 spk>1ms=0@0 spk>5ms=0@0 ops/s=42077.6 MB/s=0.000 ops=192 samp=192 heap_d=0 +[SLO] kv_del OK (max=108<=5000, spk>5ms=0<=0) +[PHASE] kv_del cold_ops=64 cold_avg=22.188 us steady_ops=128 steady_avg=24.555 us +[CHECK] KV benchmark: PASS +[BENCH] ts_insert total=7.309 ms avg=19.034 us p50=19 p95=19 min=18 max=35 max_op~0 xmax/p50=1.8 spk>1ms=0@0 spk>5ms=0@0 ops/s=52538.0 MB/s=0.200 ops=384 samp=384 heap_d=0 +[SLO] ts_insert OK (max=35<=5000, spk>5ms=0<=0) +[PHASE] ts_insert cold_ops=64 cold_avg=19.203 us steady_ops=320 steady_avg=19.000 us +[TS] target=384 retained=384 dropped=0 +[BENCH] ts_query_buf total=0.337 ms avg=337.000 us p50=337 p95=337 min=337 max=337 max_op~0 xmax/p50=1.0 spk>1ms=0@0 spk>5ms=0@0 ops/s=2967.4 MB/s=3.622 ops=1 samp=1 heap_d=0 +[CHECK] TS benchmark: PASS +[BENCH] rel_insert total=6.020 ms avg=25.083 us p50=25 p95=27 min=22 max=50 max_op~0 xmax/p50=2.0 spk>1ms=0@0 spk>5ms=0@0 ops/s=39867.1 MB/s=0.304 ops=240 samp=240 heap_d=0 +[SLO] rel_insert OK (max=50<=5000, spk>5ms=0<=0) +[PHASE] rel_insert cold_ops=64 cold_avg=23.938 us steady_ops=176 steady_avg=25.500 us +[BENCH] rel_find(index) total=0.010 ms avg=10.000 us p50=10 p95=10 min=10 max=10 max_op~0 xmax/p50=1.0 spk>1ms=0@0 spk>5ms=0@0 ops/s=100000.0 MB/s=0.763 ops=1 samp=1 heap_d=0 +[REL] rows_expected=240 rows_actual=240 +[CHECK] REL benchmark: PASS +[WAL] baseline before warmup: used=0 total=32736 fill=0% +[BENCH] wal_kv_put total=14.157 ms avg=31.600 us p50=32 p95=33 min=29 max=42 max_op~0 xmax/p50=1.3 spk>1ms=0@0 spk>5ms=0@0 ops/s=31645.1 MB/s=0.724 ops=448 samp=150 heap_d=0 +[SLO] wal_kv_put OK (max=42<=5000, spk>5ms=0<=0) +[WAL] warmup target_fill=70% reached=76% peak=76% ops_done=448/5600 steady_ops=384 (min=256) +[PHASE] wal_kv_put cold_ops=64 cold_avg=31.594 us steady_ops=384 steady_avg=31.602 us +[WAL] before compact: used=25136 total=32736 fill=76% +[BENCH] compact total=8.783 ms avg=8783.000 us p50=8783 p95=8783 min=8783 max=8783 max_op~0 xmax/p50=1.0 spk>1ms=1@0 spk>5ms=1@0 ops/s=113.9 MB/s=0.000 ops=1 samp=1 heap_d=0 +[WAL] after compact: used=0 total=32736 fill=0% +[CHECK] WAL compact: PASS +[BENCH] reopen total=12.346 ms avg=12346.000 us p50=12346 p95=12346 min=12346 max=12346 max_op~0 xmax/p50=1.0 spk>1ms=1@0 spk>5ms=1@0 ops/s=81.0 MB/s=0.000 ops=1 samp=1 heap_d=0 +[CHECK] Reopen integrity: PASS +[MIGRATE] calls=1 old=1 new=2 +[CHECK] Migration callback: PASS +[CHECK] TXN commit/rollback: PASS +[STATS] kv=142/248 (57%) coll=0 evict=0 +[STATS] ts_streams=1 ts_samples=384 ts_fill=30% +[STATS] wal=96/32736 (0%) rel_tables=2 rel_rows=240 +=== loxdb ESP32-S3 benchmark end === +loxdb-bench> [METRICS] count=10 +[METRIC] kv_put total=5.037ms avg=26.234us p50=26 p95=27 max=69@0 xmax/p50=2.7 spk>1ms=0@0 spk>5ms=0@0 ops/s=38117.9 MB/s=0.145 heap_d=0 +[METRIC] kv_get total=1.690ms avg=8.802us p50=9 p95=9 max=24@15 xmax/p50=2.7 spk>1ms=0@0 spk>5ms=0@0 ops/s=113609.5 MB/s=0.433 heap_d=0 +[METRIC] kv_del total=4.563ms avg=23.766us p50=22 p95=26 max=108@96 xmax/p50=4.9 spk>1ms=0@0 spk>5ms=0@0 ops/s=42077.6 MB/s=0.000 heap_d=0 +[METRIC] ts_insert total=7.309ms avg=19.034us p50=19 p95=19 max=35@0 xmax/p50=1.8 spk>1ms=0@0 spk>5ms=0@0 ops/s=52538.0 MB/s=0.200 heap_d=0 +[METRIC] ts_query_buf total=0.337ms avg=337.000us p50=337 p95=337 max=337@0 xmax/p50=1.0 spk>1ms=0@0 spk>5ms=0@0 ops/s=2967.4 MB/s=3.622 heap_d=0 +[METRIC] rel_insert total=6.020ms avg=25.083us p50=25 p95=27 max=50@0 xmax/p50=2.0 spk>1ms=0@0 spk>5ms=0@0 ops/s=39867.1 MB/s=0.304 heap_d=0 +[METRIC] rel_find(index) total=0.010ms avg=10.000us p50=10 p95=10 max=10@0 xmax/p50=1.0 spk>1ms=0@0 spk>5ms=0@0 ops/s=100000.0 MB/s=0.763 heap_d=0 +[METRIC] wal_kv_put total=14.157ms avg=31.600us p50=32 p95=33 max=42@0 xmax/p50=1.3 spk>1ms=0@0 spk>5ms=0@0 ops/s=31645.1 MB/s=0.724 heap_d=0 +[METRIC] compact total=8.783ms avg=8783.000us p50=8783 p95=8783 max=8783@0 xmax/p50=1.0 spk>1ms=1@0 spk>5ms=1@0 ops/s=113.9 MB/s=0.000 heap_d=0 +[METRIC] reopen total=12.346ms avg=12346.000us p50=12346 p95=12346 max=12346@0 xmax/p50=1.0 spk>1ms=1@0 spk>5ms=1@0 ops/s=81.0 MB/s=0.000 heap_d=0 +loxdb-bench> \ No newline at end of file diff --git a/docs/results/esp32_deterministic_20260511_104504_1a1c569_com19.log b/docs/results/esp32_deterministic_20260511_104504_1a1c569_com19.log new file mode 100644 index 0000000..652cb7c --- /dev/null +++ b/docs/results/esp32_deterministic_20260511_104504_1a1c569_com19.log @@ -0,0 +1,102 @@ +erministic profile + paced ON + run + note: run_det validates deterministic profile latency, not all profiles/workloads + paced - print paced mode + + resetdb - wipe storage + reopen DB + reopen - reopen DB without wipe +loxdb-bench> ESP-ROM:esp32s3-20210327 +Build:Mar 27 2021 +rst:0x15 (USB_UART_CHIP_RESET),boot:0x8 (SPI_FAST_FLASH_BOOT) +Saved PC:0x4037cb32 +SPIWP:0xee +mode:DIO, clock div:1 +load:0x3fce2820,len:0x10cc +load:0x403c8700,len:0xc2c +load:0x403cb700,len:0x30c0 +entry 0x403c88b8 + +loxdb ESP32-S3 terminal bench is ready. +Tests do NOT run automatically at power-on. +[CONFIG] profile=balanced storage=512KB ram=256KB split=40/40/20 wal_thr=75% +[CONFIG] target_kv=320 target_ts=640 target_rel=500 wal_ops=1200 wal_key=200 wal_val=32 +[CONFIG] paced=OFF pace_every=0 pace_us=0 flush_every=0 +[EFFECTIVE] kv_capacity=376 (target=320) wal_total=32736B +Commands: + help - show commands + run - run full benchmark suite (fresh DB) + kv/ts/rel/wal - run single benchmark stage + reopenchk - run reopen latency + integrity check + migrate - run schema migration check + txn - run txn check + stats - print inspect snapshot + metrics - print last captured metrics + config - print active config + profiles - list profiles + profile - show active profile + profile - switch profile and reopen DB (wipe) + run_det - deterministic profile + paced OFF + run (recommended) + run_det_paced - deterministic profile + paced ON + run + note: run_det validates deterministic profile latency, not all profiles/workloads + paced - print paced mode + paced on|off - toggle paced mode + resetdb - wipe storage + reopen DB + reopen - reopen DB without wipe +loxdb-bench> loxdb-bench> [OK] DB ready (wipe=1, profile=balanced) +loxdb-bench> [NOTE] run_det validates deterministic profile latency, not all profiles/workloads. +[PACED] mode=OFF +[OK] DB ready (wipe=1, profile=deterministic) + +=== loxdb ESP32-S3 benchmark start (profile=deterministic) === +[EFFECTIVE] kv_capacity=376 (target=192) wal_total=32736B +[BENCH] kv_put total=12.368 ms avg=64.417 us p50=64 p95=68 min=61 max=131 max_op~0 xmax/p50=2.0 spk>1ms=0@0 spk>5ms=0@0 ops/s=15523.9 MB/s=0.059 ops=192 samp=192 heap_d=0 +[SLO] kv_put OK (max=131<=5000, spk>5ms=0<=0) +[PHASE] kv_put cold_ops=64 cold_avg=64.406 us steady_ops=128 steady_avg=64.422 us +[BENCH] kv_get total=1.692 ms avg=8.812 us p50=8 p95=10 min=6 max=31 max_op~0 xmax/p50=3.9 spk>1ms=0@0 spk>5ms=0@0 ops/s=113475.2 MB/s=0.433 ops=192 samp=192 heap_d=0 +[SLO] kv_get OK (max=31<=5000, spk>5ms=0<=0) +[PHASE] kv_get cold_ops=64 cold_avg=9.047 us steady_ops=128 steady_avg=8.695 us +[BENCH] kv_del total=18.983 ms avg=98.870 us p50=97 p95=102 min=95 max=169 max_op~96 xmax/p50=1.7 spk>1ms=0@0 spk>5ms=0@0 ops/s=10114.3 MB/s=0.000 ops=192 samp=192 heap_d=0 +[SLO] kv_del OK (max=169<=5000, spk>5ms=0<=0) +[PHASE] kv_del cold_ops=64 cold_avg=97.531 us steady_ops=128 steady_avg=99.539 us +[CHECK] KV benchmark: PASS +[BENCH] ts_insert total=7.860 ms avg=20.469 us p50=20 p95=21 min=20 max=39 max_op~0 xmax/p50=2.0 spk>1ms=0@0 spk>5ms=0@0 ops/s=48855.0 MB/s=0.186 ops=384 samp=384 heap_d=0 +[SLO] ts_insert OK (max=39<=5000, spk>5ms=0<=0) +[PHASE] ts_insert cold_ops=64 cold_avg=20.672 us steady_ops=320 steady_avg=20.428 us +[TS] target=384 retained=384 dropped=0 +[BENCH] ts_query_buf total=0.093 ms avg=93.000 us p50=93 p95=93 min=93 max=93 max_op~0 xmax/p50=1.0 spk>1ms=0@0 spk>5ms=0@0 ops/s=10752.7 MB/s=13.126 ops=1 samp=1 heap_d=0 +[CHECK] TS benchmark: PASS +[BENCH] rel_insert total=6.848 ms avg=28.533 us p50=28 p95=32 min=23 max=56 max_op~0 xmax/p50=2.0 spk>1ms=0@0 spk>5ms=0@0 ops/s=35046.7 MB/s=0.267 ops=240 samp=240 heap_d=0 +[SLO] rel_insert OK (max=56<=5000, spk>5ms=0<=0) +[PHASE] rel_insert cold_ops=64 cold_avg=25.984 us steady_ops=176 steady_avg=29.460 us +[BENCH] rel_find(index) total=0.014 ms avg=14.000 us p50=14 p95=14 min=14 max=14 max_op~0 xmax/p50=1.0 spk>1ms=0@0 spk>5ms=0@0 ops/s=71428.6 MB/s=0.545 ops=1 samp=1 heap_d=0 +[REL] rows_expected=240 rows_actual=240 +[CHECK] REL benchmark: PASS +[WAL] baseline before warmup: used=0 total=32736 fill=0% +[BENCH] wal_kv_put total=31.260 ms avg=69.777 us p50=70 p95=74 min=67 max=79 max_op~0 xmax/p50=1.1 spk>1ms=0@0 spk>5ms=0@0 ops/s=14331.4 MB/s=0.328 ops=448 samp=150 heap_d=0 +[SLO] wal_kv_put OK (max=79<=5000, spk>5ms=0<=0) +[WAL] warmup target_fill=70% reached=76% peak=76% ops_done=448/5600 steady_ops=384 (min=256) +[PHASE] wal_kv_put cold_ops=64 cold_avg=68.906 us steady_ops=384 steady_avg=69.922 us +[WAL] before compact: used=25136 total=32736 fill=76% +[BENCH] compact total=9.134 ms avg=9134.000 us p50=9134 p95=9134 min=9134 max=9134 max_op~0 xmax/p50=1.0 spk>1ms=1@0 spk>5ms=1@0 ops/s=109.5 MB/s=0.000 ops=1 samp=1 heap_d=0 +[WAL] after compact: used=0 total=32736 fill=0% +[CHECK] WAL compact: PASS +[BENCH] reopen total=18.217 ms avg=18217.000 us p50=18217 p95=18217 min=18217 max=18217 max_op~0 xmax/p50=1.0 spk>1ms=1@0 spk>5ms=1@0 ops/s=54.9 MB/s=0.000 ops=1 samp=1 heap_d=0 +[CHECK] Reopen integrity: PASS +[MIGRATE] calls=1 old=1 new=2 +[CHECK] Migration callback: PASS +[CHECK] TXN commit/rollback: PASS +[STATS] kv=142/376 (37%) coll=11 evict=0 +[STATS] ts_streams=1 ts_samples=384 ts_fill=9% +[STATS] wal=96/32736 (0%) rel_tables=2 rel_rows=240 +=== loxdb ESP32-S3 benchmark end === +loxdb-bench> [METRICS] count=10 +[METRIC] kv_put total=12.368ms avg=64.417us p50=64 p95=68 max=131@0 xmax/p50=2.0 spk>1ms=0@0 spk>5ms=0@0 ops/s=15523.9 MB/s=0.059 heap_d=0 +[METRIC] kv_get total=1.692ms avg=8.812us p50=8 p95=10 max=31@0 xmax/p50=3.9 spk>1ms=0@0 spk>5ms=0@0 ops/s=113475.2 MB/s=0.433 heap_d=0 +[METRIC] kv_del total=18.983ms avg=98.870us p50=97 p95=102 max=169@96 xmax/p50=1.7 spk>1ms=0@0 spk>5ms=0@0 ops/s=10114.3 MB/s=0.000 heap_d=0 +[METRIC] ts_insert total=7.860ms avg=20.469us p50=20 p95=21 max=39@0 xmax/p50=2.0 spk>1ms=0@0 spk>5ms=0@0 ops/s=48855.0 MB/s=0.186 heap_d=0 +[METRIC] ts_query_buf total=0.093ms avg=93.000us p50=93 p95=93 max=93@0 xmax/p50=1.0 spk>1ms=0@0 spk>5ms=0@0 ops/s=10752.7 MB/s=13.126 heap_d=0 +[METRIC] rel_insert total=6.848ms avg=28.533us p50=28 p95=32 max=56@0 xmax/p50=2.0 spk>1ms=0@0 spk>5ms=0@0 ops/s=35046.7 MB/s=0.267 heap_d=0 +[METRIC] rel_find(index) total=0.014ms avg=14.000us p50=14 p95=14 max=14@0 xmax/p50=1.0 spk>1ms=0@0 spk>5ms=0@0 ops/s=71428.6 MB/s=0.545 heap_d=0 +[METRIC] wal_kv_put total=31.260ms avg=69.777us p50=70 p95=74 max=79@0 xmax/p50=1.1 spk>1ms=0@0 spk>5ms=0@0 ops/s=14331.4 MB/s=0.328 heap_d=0 +[METRIC] compact total=9.134ms avg=9134.000us p50=9134 p95=9134 max=9134@0 xmax/p50=1.0 spk>1ms=1@0 spk>5ms=1@0 ops/s=109.5 MB/s=0.000 heap_d=0 +[METRIC] reopen total=18.217ms avg=18217.000us p50=18217 p95=18217 max=18217@0 xmax/p50=1.0 spk>1ms=1@0 spk>5ms=1@0 ops/s=54.9 MB/s=0.000 heap_d=0 +loxdb-bench> \ No newline at end of file diff --git a/docs/results/esp32_sd_stress_20260511_113124_4d1fd65_com19.csv b/docs/results/esp32_sd_stress_20260511_113124_4d1fd65_com19.csv new file mode 100644 index 0000000..3066200 --- /dev/null +++ b/docs/results/esp32_sd_stress_20260511_113124_4d1fd65_com19.csv @@ -0,0 +1,111 @@ +"ts_iso","kv_pct","ts_pct","rel_pct","wal_pct","risk_pct","ops" +"2026-05-11T11:34:17.1007361+02:00","0","0","0","1","1","0" +"2026-05-11T11:34:17.1916160+02:00","0","0","0","1","1","1" +"2026-05-11T11:34:18.1221558+02:00","31","0","0","48","68","320" +"2026-05-11T11:34:24.3852974+02:00","47","0","1","0","47","472" +"2026-05-11T11:34:25.4578193+02:00","84","0","1","46","84","796" +"2026-05-11T11:34:31.4605603+02:00","100","0","2","0","100","963" +"2026-05-11T11:34:32.4584008+02:00","100","0","3","38","100","1232" +"2026-05-11T11:34:38.9312777+02:00","100","0","3","0","100","1450" +"2026-05-11T11:34:39.9296284+02:00","100","0","4","45","100","1769" +"2026-05-11T11:34:46.4463000+02:00","100","0","4","0","100","1943" +"2026-05-11T11:34:47.4497694+02:00","100","0","5","45","100","2258" +"2026-05-11T11:34:53.6354477+02:00","100","0","6","0","100","2425" +"2026-05-11T11:34:54.6393554+02:00","100","0","6","45","100","2742" +"2026-05-11T11:35:00.8517059+02:00","100","0","7","0","100","2917" +"2026-05-11T11:35:01.9120777+02:00","100","0","7","45","100","3236" +"2026-05-11T11:35:08.3232939+02:00","100","0","8","0","100","3407" +"2026-05-11T11:35:09.3237509+02:00","100","0","9","44","100","3714" +"2026-05-11T11:35:15.7143365+02:00","100","0","9","0","100","3893" +"2026-05-11T11:35:16.7125361+02:00","100","0","10","44","100","4207" +"2026-05-11T11:35:23.2741430+02:00","100","0","10","0","100","4385" +"2026-05-11T11:35:24.2732025+02:00","100","1","11","44","100","4697" +"2026-05-11T11:35:30.8124674+02:00","100","1","11","0","100","4873" +"2026-05-11T11:35:31.8137601+02:00","100","1","12","45","100","5188" +"2026-05-11T11:35:38.7405365+02:00","100","1","13","0","100","5358" +"2026-05-11T11:35:39.7524284+02:00","100","1","13","45","100","5678" +"2026-05-11T11:35:46.3973216+02:00","100","1","14","0","100","5847" +"2026-05-11T11:35:47.4031635+02:00","100","1","15","44","100","6156" +"2026-05-11T11:35:54.0233874+02:00","100","1","15","0","100","6334" +"2026-05-11T11:35:55.0858051+02:00","100","1","16","44","100","6647" +"2026-05-11T11:36:01.5357409+02:00","100","1","16","0","100","6828" +"2026-05-11T11:36:02.5255830+02:00","100","1","17","45","100","7141" +"2026-05-11T11:36:09.2987833+02:00","100","1","17","0","100","7314" +"2026-05-11T11:36:10.2986032+02:00","100","1","18","44","100","7627" +"2026-05-11T11:36:17.0465392+02:00","100","1","18","0","100","7807" +"2026-05-11T11:36:18.0445123+02:00","100","1","19","43","100","8111" +"2026-05-11T11:36:24.9046777+02:00","100","1","20","0","100","8294" +"2026-05-11T11:36:25.9015331+02:00","100","1","20","41","100","8587" +"2026-05-11T11:36:32.8446007+02:00","100","1","21","0","100","8785" +"2026-05-11T11:36:33.8417792+02:00","100","1","22","44","100","9096" +"2026-05-11T11:36:40.9738151+02:00","100","2","22","0","100","9276" +"2026-05-11T11:36:41.9888285+02:00","100","2","23","45","100","9590" +"2026-05-11T11:36:48.6947036+02:00","100","2","23","0","100","9766" +"2026-05-11T11:36:49.7064080+02:00","100","2","24","24","100","9936" +"2026-05-11T11:36:50.6945187+02:00","100","2","24","67","100","10236" +"2026-05-11T11:36:57.1311456+02:00","100","2","24","0","100","10254" +"2026-05-11T11:36:58.1579062+02:00","100","2","25","42","100","10547" +"2026-05-11T11:37:05.3373156+02:00","100","2","26","0","100","10743" +"2026-05-11T11:37:06.3125911+02:00","100","2","26","43","100","11049" +"2026-05-11T11:37:13.5693998+02:00","100","2","27","0","100","11236" +"2026-05-11T11:37:14.5797452+02:00","100","2","27","43","100","11538" +"2026-05-11T11:37:21.6391803+02:00","100","2","28","0","100","11725" +"2026-05-11T11:37:22.6322696+02:00","100","2","28","41","100","12015" +"2026-05-11T11:37:29.8155530+02:00","100","2","29","0","100","12214" +"2026-05-11T11:37:30.8120467+02:00","100","2","30","42","100","12508" +"2026-05-11T11:37:38.1347466+02:00","100","2","30","0","100","12700" +"2026-05-11T11:37:39.1327149+02:00","100","2","31","43","100","12998" +"2026-05-11T11:37:40.1309245+02:00","100","2","31","65","100","13155" +"2026-05-11T11:37:46.6519899+02:00","100","2","31","0","100","13185" +"2026-05-11T11:37:47.6455697+02:00","100","2","32","27","100","13379" +"2026-05-11T11:37:55.1294968+02:00","100","2","33","0","100","13674" +"2026-05-11T11:37:56.0966520+02:00","100","3","33","41","100","13965" +"2026-05-11T11:38:03.7749516+02:00","100","3","34","0","100","14162" +"2026-05-11T11:38:04.7878115+02:00","100","3","34","40","100","14446" +"2026-05-11T11:38:12.3865155+02:00","100","3","35","0","100","14650" +"2026-05-11T11:38:13.3861726+02:00","100","3","36","43","100","14948" +"2026-05-11T11:38:20.6874501+02:00","100","3","36","0","100","15136" +"2026-05-11T11:38:21.6827847+02:00","100","3","37","43","100","15443" +"2026-05-11T11:38:29.2619877+02:00","100","3","37","0","100","15628" +"2026-05-11T11:38:30.2602750+02:00","100","3","38","41","100","15917" +"2026-05-11T11:38:37.7681718+02:00","100","3","39","0","100","16118" +"2026-05-11T11:38:38.8249943+02:00","100","3","39","39","100","16390" +"2026-05-11T11:38:46.4446674+02:00","100","3","40","0","100","16601" +"2026-05-11T11:38:47.4474844+02:00","100","3","41","40","100","16887" +"2026-05-11T11:38:55.3376249+02:00","100","3","41","0","100","17088" +"2026-05-11T11:38:56.3348929+02:00","100","3","42","31","100","17303" +"2026-05-11T11:38:57.3377917+02:00","100","3","42","69","100","17576" +"2026-05-11T11:39:04.4607295+02:00","100","3","42","0","100","17579" +"2026-05-11T11:39:05.4704444+02:00","100","3","43","41","100","17872" +"2026-05-11T11:39:13.2552885+02:00","100","3","43","0","100","18068" +"2026-05-11T11:39:14.2814129+02:00","100","3","44","39","100","18341" +"2026-05-11T11:39:22.2049127+02:00","100","4","45","0","100","18557" +"2026-05-11T11:39:23.1556691+02:00","100","4","45","39","100","18835" +"2026-05-11T11:39:31.0276814+02:00","100","4","46","0","100","19048" +"2026-05-11T11:39:32.0223200+02:00","100","4","47","42","100","19342" +"2026-05-11T11:39:39.9377633+02:00","100","4","47","0","100","19534" +"2026-05-11T11:39:40.9343688+02:00","100","4","48","41","100","19825" +"2026-05-11T11:39:49.0684183+02:00","100","4","48","0","100","20024" +"2026-05-11T11:39:50.0839280+02:00","100","4","49","39","100","20297" +"2026-05-11T11:39:51.0649331+02:00","100","4","49","66","100","20483" +"2026-05-11T11:39:58.1934146+02:00","100","4","49","0","100","20512" +"2026-05-11T11:39:59.3277734+02:00","100","4","50","34","100","20751" +"2026-05-11T11:40:07.6386523+02:00","100","4","51","0","100","20999" +"2026-05-11T11:40:08.6303674+02:00","100","4","51","41","100","21286" +"2026-05-11T11:40:16.9046649+02:00","100","4","52","0","100","21485" +"2026-05-11T11:40:17.9027065+02:00","100","4","52","42","100","21782" +"2026-05-11T11:40:25.9789303+02:00","100","4","53","0","100","21975" +"2026-05-11T11:40:26.9751936+02:00","100","4","54","39","100","22245" +"2026-05-11T11:40:35.1686773+02:00","100","4","54","0","100","22461" +"2026-05-11T11:40:36.1631985+02:00","100","4","55","37","100","22724" +"2026-05-11T11:40:44.4542421+02:00","100","4","55","0","100","22952" +"2026-05-11T11:40:45.4435860+02:00","100","5","56","40","100","23236" +"2026-05-11T11:40:46.4556708+02:00","100","5","56","65","100","23411" +"2026-05-11T11:40:53.8458771+02:00","100","5","56","0","100","23441" +"2026-05-11T11:40:55.0955954+02:00","100","5","57","41","100","23731" +"2026-05-11T11:41:03.5895254+02:00","100","5","58","0","100","23930" +"2026-05-11T11:41:04.5874455+02:00","100","5","58","39","100","24202" +"2026-05-11T11:41:12.9778638+02:00","100","5","59","0","100","24417" +"2026-05-11T11:41:13.9766880+02:00","100","5","59","36","100","24675" +"2026-05-11T11:41:22.4577521+02:00","100","5","60","0","100","24910" +"2026-05-11T11:41:23.4865328+02:00","100","5","61","40","100","25193" diff --git a/docs/results/esp32_sd_stress_20260511_113124_4d1fd65_com19.log b/docs/results/esp32_sd_stress_20260511_113124_4d1fd65_com19.log new file mode 100644 index 0000000..2642eed --- /dev/null +++ b/docs/results/esp32_sd_stress_20260511_113124_4d1fd65_com19.log @@ -0,0 +1,358 @@ +ESP-ROM:esp32s3-20210327 +[OK] SD mounted card=7580MB +[OK] profile=soak mix=35/35/30 +[OK] admitted profile=soak-A ram_kb=4096 split=40/30/30 wal_th=70 +[OK] loxdb SD stress bench ready +SD pins CLK=17 CMD=18 D0=16 D3=47 +LCD pins SCLK=10 MOSI=11 CS=12 DC=13 RST=14 +ADMISSION ram_kb=4096 split=40/30/30 wal_th=70 +Commands: + run | pause | resume + profile smoke|soak|stress + reinit + verify on|off + mode all|kv|ts|rel + clear kv|ts|rel|all + slist + swipe confirm | swipe all confirm + compact | stats | resetdb | formatdb +[PRESSURE] kv=100 ts=2 rel=19 wal=1 risk=100 ops=1 +[STATS] kv=248/248 ts_samples=3622 rel_rows=3148 wal=332/32736 +[BENCH] profile=soak mode=all verify=on ok=1 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[OK] profile=stress mix=25/35/40 +[INFO] profile affects admission; run 'reinit' (or resetdb/formatdb) to re-admit +[WARN] lox_init reject profile=stress-A rc=-2 (LOX_ERR_NO_MEM) +[OK] admitted profile=stress-B ram_kb=4096 split=40/30/30 wal_th=70 +[OK] db reset complete +[OK] mode=all +[OK] verify=on +[PRESSURE] kv=0 ts=0 rel=0 wal=1 risk=1 ops=0 +[STATS] kv=0/248 ts_samples=0 rel_rows=0 wal=580/32736 +[BENCH] profile=stress mode=all verify=on ok=0 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=0 ts=0 rel=0 wal=1 risk=1 ops=1 +[STATS] kv=1/248 ts_samples=0 rel_rows=0 wal=620/32736 +[BENCH] profile=stress mode=all verify=on ok=1 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=31 ts=0 rel=0 wal=48 risk=68 ops=320 +[STATS] kv=78/248 ts_samples=105 rel_rows=137 wal=15724/32736 +[BENCH] profile=stress mode=all verify=on ok=3 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=47 ts=0 rel=1 wal=0 risk=47 ops=472 +[STATS] kv=117/248 ts_samples=151 rel_rows=204 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=5 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=84 ts=0 rel=1 wal=46 risk=84 ops=796 +[STATS] kv=209/248 ts_samples=263 rel_rows=324 wal=15088/32736 +[BENCH] profile=stress mode=all verify=on ok=9 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=0 rel=2 wal=0 risk=100 ops=963 +[STATS] kv=248/248 ts_samples=319 rel_rows=392 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=10 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=0 rel=3 wal=38 risk=100 ops=1232 +[STATS] kv=248/248 ts_samples=406 rel_rows=498 wal=12592/32736 +[BENCH] profile=stress mode=all verify=on ok=12 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=0 rel=3 wal=0 risk=100 ops=1450 +[STATS] kv=248/248 ts_samples=475 rel_rows=593 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=14 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=0 rel=4 wal=45 risk=100 ops=1769 +[STATS] kv=248/248 ts_samples=596 rel_rows=709 wal=14868/32736 +[BENCH] profile=stress mode=all verify=on ok=17 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=0 rel=4 wal=0 risk=100 ops=1943 +[STATS] kv=248/248 ts_samples=650 rel_rows=775 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=20 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=0 rel=5 wal=45 risk=100 ops=2258 +[STATS] kv=248/248 ts_samples=749 rel_rows=918 wal=15004/32736 +[BENCH] profile=stress mode=all verify=on ok=23 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=0 rel=6 wal=0 risk=100 ops=2425 +[STATS] kv=248/248 ts_samples=808 rel_rows=990 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=24 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=0 rel=6 wal=45 risk=100 ops=2742 +[STATS] kv=248/248 ts_samples=922 rel_rows=1107 wal=14768/32736 +[BENCH] profile=stress mode=all verify=on ok=28 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=0 rel=7 wal=0 risk=100 ops=2917 +[STATS] kv=248/248 ts_samples=977 rel_rows=1173 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=31 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=0 rel=7 wal=45 risk=100 ops=3236 +[STATS] kv=248/248 ts_samples=1080 rel_rows=1306 wal=15028/32736 +[BENCH] profile=stress mode=all verify=on ok=35 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=0 rel=8 wal=0 risk=100 ops=3407 +[STATS] kv=248/248 ts_samples=1144 rel_rows=1365 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=36 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=0 rel=9 wal=44 risk=100 ops=3714 +[STATS] kv=248/248 ts_samples=1242 rel_rows=1491 wal=14424/32736 +[BENCH] profile=stress mode=all verify=on ok=39 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=0 rel=9 wal=0 risk=100 ops=3893 +[STATS] kv=248/248 ts_samples=1301 rel_rows=1569 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=40 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=0 rel=10 wal=44 risk=100 ops=4207 +[STATS] kv=248/248 ts_samples=1420 rel_rows=1681 wal=14604/32736 +[BENCH] profile=stress mode=all verify=on ok=44 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=0 rel=10 wal=0 risk=100 ops=4385 +[STATS] kv=248/248 ts_samples=1483 rel_rows=1749 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=46 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=1 rel=11 wal=44 risk=100 ops=4697 +[STATS] kv=248/248 ts_samples=1604 rel_rows=1869 wal=14644/32736 +[BENCH] profile=stress mode=all verify=on ok=48 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=1 rel=11 wal=0 risk=100 ops=4873 +[STATS] kv=248/248 ts_samples=1664 rel_rows=1942 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=50 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=1 rel=12 wal=45 risk=100 ops=5188 +[STATS] kv=248/248 ts_samples=1763 rel_rows=2082 wal=14956/32736 +[BENCH] profile=stress mode=all verify=on ok=53 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=1 rel=13 wal=0 risk=100 ops=5358 +[STATS] kv=248/248 ts_samples=1819 rel_rows=2150 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=55 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=1 rel=13 wal=45 risk=100 ops=5678 +[STATS] kv=248/248 ts_samples=1928 rel_rows=2267 wal=14868/32736 +[BENCH] profile=stress mode=all verify=on ok=60 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=1 rel=14 wal=0 risk=100 ops=5847 +[STATS] kv=248/248 ts_samples=1989 rel_rows=2341 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=61 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=1 rel=15 wal=44 risk=100 ops=6156 +[STATS] kv=248/248 ts_samples=2090 rel_rows=2468 wal=14536/32736 +[BENCH] profile=stress mode=all verify=on ok=64 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=1 rel=15 wal=0 risk=100 ops=6334 +[STATS] kv=248/248 ts_samples=2147 rel_rows=2543 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=65 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=1 rel=16 wal=44 risk=100 ops=6647 +[STATS] kv=248/248 ts_samples=2260 rel_rows=2657 wal=14580/32736 +[BENCH] profile=stress mode=all verify=on ok=68 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=1 rel=16 wal=0 risk=100 ops=6828 +[STATS] kv=248/248 ts_samples=2329 rel_rows=2720 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=70 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=1 rel=17 wal=45 risk=100 ops=7141 +[STATS] kv=248/248 ts_samples=2439 rel_rows=2848 wal=14752/32736 +[BENCH] profile=stress mode=all verify=on ok=74 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=1 rel=17 wal=0 risk=100 ops=7314 +[STATS] kv=248/248 ts_samples=2487 rel_rows=2924 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=76 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=1 rel=18 wal=44 risk=100 ops=7627 +[STATS] kv=248/248 ts_samples=2593 rel_rows=3041 wal=14588/32736 +[BENCH] profile=stress mode=all verify=on ok=79 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=1 rel=18 wal=0 risk=100 ops=7807 +[STATS] kv=248/248 ts_samples=2657 rel_rows=3107 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=80 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=1 rel=19 wal=43 risk=100 ops=8111 +[STATS] kv=248/248 ts_samples=2770 rel_rows=3228 wal=14312/32736 +[BENCH] profile=stress mode=all verify=on ok=83 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=1 rel=20 wal=0 risk=100 ops=8294 +[STATS] kv=248/248 ts_samples=2837 rel_rows=3304 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=86 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=1 rel=20 wal=41 risk=100 ops=8587 +[STATS] kv=248/248 ts_samples=2946 rel_rows=3406 wal=13584/32736 +[BENCH] profile=stress mode=all verify=on ok=91 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=1 rel=21 wal=0 risk=100 ops=8785 +[STATS] kv=248/248 ts_samples=3013 rel_rows=3488 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=92 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=1 rel=22 wal=44 risk=100 ops=9096 +[STATS] kv=248/248 ts_samples=3120 rel_rows=3607 wal=14528/32736 +[BENCH] profile=stress mode=all verify=on ok=95 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=2 rel=22 wal=0 risk=100 ops=9276 +[STATS] kv=248/248 ts_samples=3184 rel_rows=3674 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=97 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=2 rel=23 wal=45 risk=100 ops=9590 +[STATS] kv=248/248 ts_samples=3279 rel_rows=3804 wal=14760/32736 +[BENCH] profile=stress mode=all verify=on ok=100 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=2 rel=23 wal=0 risk=100 ops=9766 +[STATS] kv=248/248 ts_samples=3339 rel_rows=3868 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=102 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=2 rel=24 wal=24 risk=100 ops=9936 +[STATS] kv=248/248 ts_samples=3398 rel_rows=3935 wal=7968/32736 +[BENCH] profile=stress mode=all verify=on ok=103 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=2 rel=24 wal=67 risk=100 ops=10236 +[STATS] kv=248/248 ts_samples=3495 rel_rows=4059 wal=22092/32736 +[BENCH] profile=stress mode=all verify=on ok=106 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=2 rel=24 wal=0 risk=100 ops=10254 +[STATS] kv=248/248 ts_samples=3501 rel_rows=4067 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=107 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=2 rel=25 wal=42 risk=100 ops=10547 +[STATS] kv=248/248 ts_samples=3599 rel_rows=4184 wal=13756/32736 +[BENCH] profile=stress mode=all verify=on ok=110 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=2 rel=26 wal=0 risk=100 ops=10743 +[STATS] kv=248/248 ts_samples=3666 rel_rows=4261 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=111 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=2 rel=26 wal=43 risk=100 ops=11049 +[STATS] kv=248/248 ts_samples=3775 rel_rows=4381 wal=14356/32736 +[BENCH] profile=stress mode=all verify=on ok=115 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=2 rel=27 wal=0 risk=100 ops=11236 +[STATS] kv=248/248 ts_samples=3847 rel_rows=4439 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=117 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=2 rel=27 wal=43 risk=100 ops=11538 +[STATS] kv=248/248 ts_samples=3941 rel_rows=4560 wal=14156/32736 +[BENCH] profile=stress mode=all verify=on ok=121 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=2 rel=28 wal=0 risk=100 ops=11725 +[STATS] kv=248/248 ts_samples=4007 rel_rows=4633 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=122 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=2 rel=28 wal=41 risk=100 ops=12015 +[STATS] kv=248/248 ts_samples=4113 rel_rows=4749 wal=13648/32736 +[BENCH] profile=stress mode=all verify=on ok=124 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=2 rel=29 wal=0 risk=100 ops=12214 +[STATS] kv=248/248 ts_samples=4194 rel_rows=4822 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=126 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=2 rel=30 wal=42 risk=100 ops=12508 +[STATS] kv=248/248 ts_samples=4294 rel_rows=4940 wal=13812/32736 +[BENCH] profile=stress mode=all verify=on ok=130 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=2 rel=30 wal=0 risk=100 ops=12700 +[STATS] kv=248/248 ts_samples=4354 rel_rows=5025 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=132 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=2 rel=31 wal=43 risk=100 ops=12998 +[STATS] kv=248/248 ts_samples=4448 rel_rows=5161 wal=14200/32736 +[BENCH] profile=stress mode=all verify=on ok=134 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=2 rel=31 wal=65 risk=100 ops=13155 +[STATS] kv=248/248 ts_samples=4503 rel_rows=5223 wal=21556/32736 +[BENCH] profile=stress mode=all verify=on ok=136 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=2 rel=31 wal=0 risk=100 ops=13185 +[STATS] kv=248/248 ts_samples=4517 rel_rows=5232 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=137 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=2 rel=32 wal=27 risk=100 ops=13379 +[STATS] kv=248/248 ts_samples=4591 rel_rows=5309 wal=9140/32736 +[BENCH] profile=stress mode=all verify=on ok=138 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=2 rel=33 wal=0 risk=100 ops=13674 +[STATS] kv=248/248 ts_samples=4686 rel_rows=5425 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=140 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=3 rel=33 wal=41 risk=100 ops=13965 +[STATS] kv=248/248 ts_samples=4781 rel_rows=5539 wal=13604/32736 +[BENCH] profile=stress mode=all verify=on ok=143 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=3 rel=34 wal=0 risk=100 ops=14162 +[STATS] kv=248/248 ts_samples=4847 rel_rows=5622 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=146 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=3 rel=34 wal=40 risk=100 ops=14446 +[STATS] kv=248/248 ts_samples=4949 rel_rows=5729 wal=13272/32736 +[BENCH] profile=stress mode=all verify=on ok=149 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=3 rel=35 wal=0 risk=100 ops=14650 +[STATS] kv=248/248 ts_samples=5017 rel_rows=5819 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=152 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=3 rel=36 wal=43 risk=100 ops=14948 +[STATS] kv=248/248 ts_samples=5116 rel_rows=5948 wal=14128/32736 +[BENCH] profile=stress mode=all verify=on ok=156 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=3 rel=36 wal=0 risk=100 ops=15136 +[STATS] kv=248/248 ts_samples=5183 rel_rows=6020 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=159 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=3 rel=37 wal=43 risk=100 ops=15443 +[STATS] kv=248/248 ts_samples=5287 rel_rows=6139 wal=14356/32736 +[BENCH] profile=stress mode=all verify=on ok=163 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=3 rel=37 wal=0 risk=100 ops=15628 +[STATS] kv=248/248 ts_samples=5358 rel_rows=6204 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=165 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=3 rel=38 wal=41 risk=100 ops=15917 +[STATS] kv=248/248 ts_samples=5457 rel_rows=6315 wal=13504/32736 +[BENCH] profile=stress mode=all verify=on ok=167 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=3 rel=39 wal=0 risk=100 ops=16118 +[STATS] kv=248/248 ts_samples=5524 rel_rows=6395 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=168 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=3 rel=39 wal=39 risk=100 ops=16390 +[STATS] kv=248/248 ts_samples=5611 rel_rows=6508 wal=12816/32736 +[BENCH] profile=stress mode=all verify=on ok=171 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=3 rel=40 wal=0 risk=100 ops=16601 +[STATS] kv=248/248 ts_samples=5676 rel_rows=6609 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=173 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=3 rel=41 wal=40 risk=100 ops=16887 +[STATS] kv=248/248 ts_samples=5783 rel_rows=6720 wal=13416/32736 +[BENCH] profile=stress mode=all verify=on ok=175 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=3 rel=41 wal=0 risk=100 ops=17088 +[STATS] kv=248/248 ts_samples=5841 rel_rows=6811 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=178 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=3 rel=42 wal=31 risk=100 ops=17303 +[STATS] kv=248/248 ts_samples=5898 rel_rows=6905 wal=10156/32736 +[BENCH] profile=stress mode=all verify=on ok=181 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=3 rel=42 wal=69 risk=100 ops=17576 +[STATS] kv=248/248 ts_samples=5992 rel_rows=6999 wal=22756/32736 +[BENCH] profile=stress mode=all verify=on ok=184 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=3 rel=42 wal=0 risk=100 ops=17579 +[STATS] kv=248/248 ts_samples=5993 rel_rows=7001 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=184 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=3 rel=43 wal=41 risk=100 ops=17872 +[STATS] kv=248/248 ts_samples=6097 rel_rows=7114 wal=13712/32736 +[BENCH] profile=stress mode=all verify=on ok=187 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=3 rel=43 wal=0 risk=100 ops=18068 +[STATS] kv=248/248 ts_samples=6162 rel_rows=7193 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=189 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=3 rel=44 wal=39 risk=100 ops=18341 +[STATS] kv=248/248 ts_samples=6242 rel_rows=7311 wal=12880/32736 +[BENCH] profile=stress mode=all verify=on ok=193 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=4 rel=45 wal=0 risk=100 ops=18557 +[STATS] kv=248/248 ts_samples=6317 rel_rows=7390 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=195 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=4 rel=45 wal=39 risk=100 ops=18835 +[STATS] kv=248/248 ts_samples=6413 rel_rows=7495 wal=12968/32736 +[BENCH] profile=stress mode=all verify=on ok=198 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=4 rel=46 wal=0 risk=100 ops=19048 +[STATS] kv=248/248 ts_samples=6489 rel_rows=7577 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=199 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=4 rel=47 wal=42 risk=100 ops=19342 +[STATS] kv=248/248 ts_samples=6585 rel_rows=7703 wal=13896/32736 +[BENCH] profile=stress mode=all verify=on ok=204 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=4 rel=47 wal=0 risk=100 ops=19534 +[STATS] kv=248/248 ts_samples=6649 rel_rows=7781 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=205 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=4 rel=48 wal=41 risk=100 ops=19825 +[STATS] kv=248/248 ts_samples=6751 rel_rows=7896 wal=13652/32736 +[BENCH] profile=stress mode=all verify=on ok=207 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=4 rel=48 wal=0 risk=100 ops=20024 +[STATS] kv=248/248 ts_samples=6823 rel_rows=7970 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=210 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=4 rel=49 wal=39 risk=100 ops=20297 +[STATS] kv=248/248 ts_samples=6930 rel_rows=8083 wal=12924/32736 +[BENCH] profile=stress mode=all verify=on ok=213 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=4 rel=49 wal=66 risk=100 ops=20483 +[STATS] kv=248/248 ts_samples=7002 rel_rows=8153 wal=21644/32736 +[BENCH] profile=stress mode=all verify=on ok=215 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=4 rel=49 wal=0 risk=100 ops=20512 +[STATS] kv=248/248 ts_samples=7015 rel_rows=8160 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=215 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=4 rel=50 wal=34 risk=100 ops=20751 +[STATS] kv=248/248 ts_samples=7096 rel_rows=8261 wal=11304/32736 +[BENCH] profile=stress mode=all verify=on ok=217 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=4 rel=51 wal=0 risk=100 ops=20999 +[STATS] kv=248/248 ts_samples=7175 rel_rows=8360 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=220 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=4 rel=51 wal=41 risk=100 ops=21286 +[STATS] kv=248/248 ts_samples=7261 rel_rows=8483 wal=13540/32736 +[BENCH] profile=stress mode=all verify=on ok=223 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=4 rel=52 wal=0 risk=100 ops=21485 +[STATS] kv=248/248 ts_samples=7322 rel_rows=8568 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=225 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=4 rel=52 wal=42 risk=100 ops=21782 +[STATS] kv=248/248 ts_samples=7439 rel_rows=8678 wal=13888/32736 +[BENCH] profile=stress mode=all verify=on ok=228 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=4 rel=53 wal=0 risk=100 ops=21975 +[STATS] kv=248/248 ts_samples=7502 rel_rows=8753 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=230 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=4 rel=54 wal=39 risk=100 ops=22245 +[STATS] kv=248/248 ts_samples=7584 rel_rows=8874 wal=12816/32736 +[BENCH] profile=stress mode=all verify=on ok=234 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=4 rel=54 wal=0 risk=100 ops=22461 +[STATS] kv=248/248 ts_samples=7662 rel_rows=8957 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=237 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=4 rel=55 wal=37 risk=100 ops=22724 +[STATS] kv=248/248 ts_samples=7768 rel_rows=9054 wal=12296/32736 +[BENCH] profile=stress mode=all verify=on ok=239 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=4 rel=55 wal=0 risk=100 ops=22952 +[STATS] kv=248/248 ts_samples=7853 rel_rows=9138 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=241 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=5 rel=56 wal=40 risk=100 ops=23236 +[STATS] kv=248/248 ts_samples=7946 rel_rows=9258 wal=13412/32736 +[BENCH] profile=stress mode=all verify=on ok=244 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=5 rel=56 wal=65 risk=100 ops=23411 +[STATS] kv=248/248 ts_samples=8005 rel_rows=9326 wal=21600/32736 +[BENCH] profile=stress mode=all verify=on ok=245 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=5 rel=56 wal=0 risk=100 ops=23441 +[STATS] kv=248/248 ts_samples=8012 rel_rows=9335 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=246 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=5 rel=57 wal=41 risk=100 ops=23731 +[STATS] kv=248/248 ts_samples=8110 rel_rows=9452 wal=13636/32736 +[BENCH] profile=stress mode=all verify=on ok=248 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=5 rel=58 wal=0 risk=100 ops=23930 +[STATS] kv=248/248 ts_samples=8165 rel_rows=9531 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=251 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=5 rel=58 wal=39 risk=100 ops=24202 +[STATS] kv=248/248 ts_samples=8267 rel_rows=9638 wal=12792/32736 +[BENCH] profile=stress mode=all verify=on ok=255 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=5 rel=59 wal=0 risk=100 ops=24417 +[STATS] kv=248/248 ts_samples=8348 rel_rows=9726 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=257 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=5 rel=59 wal=36 risk=100 ops=24675 +[STATS] kv=248/248 ts_samples=8447 rel_rows=9820 wal=12032/32736 +[BENCH] profile=stress mode=all verify=on ok=260 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=5 rel=60 wal=0 risk=100 ops=24910 +[STATS] kv=248/248 ts_samples=8525 rel_rows=9904 wal=0/32736 +[BENCH] profile=stress mode=all verify=on ok=263 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 +[PRESSURE] kv=100 ts=5 rel=61 wal=40 risk=100 ops=25193 +[STATS] kv=248/248 ts_samples=8622 rel_rows=10018 wal=13304/32736 +[BENCH] profile=stress mode=all verify=on ok=265 fail=0 compact=0 last_compact_ms=0 streams=8 tables=4 diff --git a/docs/results/esp32_sd_stress_20260511_113124_4d1fd65_com19.md b/docs/results/esp32_sd_stress_20260511_113124_4d1fd65_com19.md new file mode 100644 index 0000000..83c0b2c --- /dev/null +++ b/docs/results/esp32_sd_stress_20260511_113124_4d1fd65_com19.md @@ -0,0 +1,23 @@ +# SD stress run - 20260511_113124 + +- port: COM19 +- repo commit: 4d1fd65 +- profile: stress +- mode: all +- verify: on +- duration: 600s +- resetdb: yes +- formatdb: no + +Admission: + +- [OK] admitted profile=stress-B ram_kb=4096 split=40/30/30 wal_th=70 + +Artifacts: + +- raw log: esp32_sd_stress_20260511_113124_4d1fd65_com19.log +- pressure CSV: esp32_sd_stress_20260511_113124_4d1fd65_com19.csv + +Notes: + +- The bench prints [PRESSURE], [STATS], [BENCH] lines periodically; see the raw log for full context. \ No newline at end of file diff --git a/docs/results/esp32_stress_20260511_102425_1a1c569_com19.log b/docs/results/esp32_stress_20260511_102425_1a1c569_com19.log new file mode 100644 index 0000000..abb4c31 --- /dev/null +++ b/docs/results/esp32_stress_20260511_102425_1a1c569_com19.log @@ -0,0 +1,59 @@ +loxdb-bench> [PROFILE] switched to stress paced=OFF +[OK] DB ready (wipe=1, profile=stress) +loxdb-bench> +=== loxdb ESP32-S3 benchmark start (profile=stress) === +[EFFECTIVE] kv_capacity=248 (target=900) wal_total=32736B +[KV] capped ops to capacity: 248 +[BENCH] kv_put total=6.525 ms avg=26.310 us p50=26 p95=27 min=24 max=72 max_op~0 xmax/p50=2.8 spk>1ms=0@0 spk>5ms=0@0 ops/s=38007.7 MB/s=0.145 ops=248 samp=248 heap_d=0 +[SLO] kv_put OK (max=72<=25000, spk>5ms=0<=30) +[PHASE] kv_put cold_ops=64 cold_avg=26.406 us steady_ops=184 steady_avg=26.277 us +[BENCH] kv_get total=2.219 ms avg=8.948 us p50=9 p95=9 min=6 max=25 max_op~0 xmax/p50=2.8 spk>1ms=0@0 spk>5ms=0@0 ops/s=111762.1 MB/s=0.426 ops=248 samp=248 heap_d=0 +[SLO] kv_get OK (max=25<=25000, spk>5ms=0<=30) +[PHASE] kv_get cold_ops=64 cold_avg=9.031 us steady_ops=184 steady_avg=8.918 us +[BENCH] kv_del total=5.773 ms avg=23.278 us p50=22 p95=23 min=20 max=116 max_op~124 xmax/p50=5.3 spk>1ms=0@0 spk>5ms=0@0 ops/s=42958.6 MB/s=0.000 ops=248 samp=248 heap_d=0 +[SLO] kv_del OK (max=116<=25000, spk>5ms=0<=30) +[PHASE] kv_del cold_ops=64 cold_avg=21.531 us steady_ops=184 steady_avg=23.886 us +[CHECK] KV benchmark: PASS +[BENCH] ts_insert total=81.612 ms avg=34.005 us p50=19 p95=20 min=18 max=14584 max_op~2312 xmax/p50=767.6 spk>1ms=2@494 spk>5ms=2@494 ops/s=29407.4 MB/s=0.112 ops=2400 samp=1200 heap_d=0 +[SLO] ts_insert OK (max=14584<=25000, spk>5ms=2<=30) +[PHASE] ts_insert cold_ops=64 cold_avg=19.859 us steady_ops=2336 steady_avg=34.393 us +[TS] target=2400 retained=1792 dropped=608 +[BENCH] ts_query_buf total=1.471 ms avg=1471.000 us p50=1471 p95=1471 min=1471 max=1471 max_op~0 xmax/p50=1.0 spk>1ms=1@0 spk>5ms=0@0 ops/s=679.8 MB/s=0.830 ops=1 samp=1 heap_d=0 +[CHECK] TS benchmark: PASS +[BENCH] rel_insert total=288.920 ms avg=240.767 us p50=170 p95=628 min=22 max=16534 max_op~737 xmax/p50=97.3 spk>1ms=1@737 spk>5ms=1@737 ops/s=4153.4 MB/s=0.032 ops=1200 samp=1200 heap_d=0 +[SLO] rel_insert OK (max=16534<=25000, spk>5ms=1<=30) +[PHASE] rel_insert cold_ops=64 cold_avg=23.812 us steady_ops=1136 steady_avg=252.989 us +[BENCH] rel_find(index) total=0.013 ms avg=13.000 us p50=13 p95=13 min=13 max=13 max_op~0 xmax/p50=1.0 spk>1ms=0@0 spk>5ms=0@0 ops/s=76923.1 MB/s=0.587 ops=1 samp=1 heap_d=0 +[REL] rows_expected=1200 rows_actual=1200 +[CHECK] REL benchmark: PASS +[WAL] baseline before warmup: used=0 total=32736 fill=0% +[WAL] key_span adjusted to 246 to keep probe key resident. +[BENCH] wal_kv_put total=13.424 ms avg=41.950 us p50=42 p95=45 min=39 max=72 max_op~0 xmax/p50=1.7 spk>1ms=0@0 spk>5ms=0@0 ops/s=23837.9 MB/s=1.455 ops=320 samp=25 heap_d=0 +[SLO] wal_kv_put OK (max=72<=25000, spk>5ms=0<=30) +[WAL] warmup target_fill=80% reached=93% peak=93% ops_done=320/25600 steady_ops=256 (min=256) +[PHASE] wal_kv_put cold_ops=64 cold_avg=42.625 us steady_ops=256 steady_avg=41.781 us +[WAL] before compact: used=30768 total=32736 fill=93% +[BENCH] compact total=22.798 ms avg=22798.000 us p50=22798 p95=22798 min=22798 max=22798 max_op~0 xmax/p50=1.0 spk>1ms=1@0 spk>5ms=1@0 ops/s=43.9 MB/s=0.000 ops=1 samp=1 heap_d=0 +[WAL] after compact: used=0 total=32736 fill=0% +[CHECK] WAL compact: PASS +[BENCH] reopen total=277.816 ms avg=277816.000 us p50=277816 p95=277816 min=277816 max=277816 max_op~0 xmax/p50=1.0 spk>1ms=1@0 spk>5ms=1@0 ops/s=3.6 MB/s=0.000 ops=1 samp=1 heap_d=0 +[CHECK] Reopen integrity: PASS +[MIGRATE] calls=1 old=1 new=2 +[CHECK] Migration callback: PASS +[CHECK] TXN commit/rollback: PASS +[STATS] kv=248/248 (100%) coll=0 evict=0 +[STATS] ts_streams=1 ts_samples=1792 ts_fill=100% +[STATS] wal=96/32736 (0%) rel_tables=2 rel_rows=1200 +=== loxdb ESP32-S3 benchmark end === +loxdb-bench> [METRICS] count=10 +[METRIC] kv_put total=6.525ms avg=26.310us p50=26 p95=27 max=72@0 xmax/p50=2.8 spk>1ms=0@0 spk>5ms=0@0 ops/s=38007.7 MB/s=0.145 heap_d=0 +[METRIC] kv_get total=2.219ms avg=8.948us p50=9 p95=9 max=25@0 xmax/p50=2.8 spk>1ms=0@0 spk>5ms=0@0 ops/s=111762.1 MB/s=0.426 heap_d=0 +[METRIC] kv_del total=5.773ms avg=23.278us p50=22 p95=23 max=116@124 xmax/p50=5.3 spk>1ms=0@0 spk>5ms=0@0 ops/s=42958.6 MB/s=0.000 heap_d=0 +[METRIC] ts_insert total=81.612ms avg=34.005us p50=19 p95=20 max=14584@2312 xmax/p50=767.6 spk>1ms=2@494 spk>5ms=2@494 ops/s=29407.4 MB/s=0.112 heap_d=0 +[METRIC] ts_query_buf total=1.471ms avg=1471.000us p50=1471 p95=1471 max=1471@0 xmax/p50=1.0 spk>1ms=1@0 spk>5ms=0@0 ops/s=679.8 MB/s=0.830 heap_d=0 +[METRIC] rel_insert total=288.920ms avg=240.767us p50=170 p95=628 max=16534@737 xmax/p50=97.3 spk>1ms=1@737 spk>5ms=1@737 ops/s=4153.4 MB/s=0.032 heap_d=0 +[METRIC] rel_find(index) total=0.013ms avg=13.000us p50=13 p95=13 max=13@0 xmax/p50=1.0 spk>1ms=0@0 spk>5ms=0@0 ops/s=76923.1 MB/s=0.587 heap_d=0 +[METRIC] wal_kv_put total=13.424ms avg=41.950us p50=42 p95=45 max=72@0 xmax/p50=1.7 spk>1ms=0@0 spk>5ms=0@0 ops/s=23837.9 MB/s=1.455 heap_d=0 +[METRIC] compact total=22.798ms avg=22798.000us p50=22798 p95=22798 max=22798@0 xmax/p50=1.0 spk>1ms=1@0 spk>5ms=1@0 ops/s=43.9 MB/s=0.000 heap_d=0 +[METRIC] reopen total=277.816ms avg=277816.000us p50=277816 p95=277816 max=277816@0 xmax/p50=1.0 spk>1ms=1@0 spk>5ms=1@0 ops/s=3.6 MB/s=0.000 heap_d=0 +loxdb-bench> \ No newline at end of file diff --git a/docs/results/esp32_stress_20260511_104504_1a1c569_com19.log b/docs/results/esp32_stress_20260511_104504_1a1c569_com19.log new file mode 100644 index 0000000..1d645ea --- /dev/null +++ b/docs/results/esp32_stress_20260511_104504_1a1c569_com19.log @@ -0,0 +1,58 @@ +loxdb-bench> [PROFILE] switched to stress paced=OFF +[OK] DB ready (wipe=1, profile=stress) +loxdb-bench> +=== loxdb ESP32-S3 benchmark start (profile=stress) === +[EFFECTIVE] kv_capacity=376 (target=900) wal_total=8160B +[KV] capped ops to capacity: 376 +[BENCH] kv_put total=31.809 ms avg=84.598 us p50=65 p95=69 min=61 max=7439 max_op~226 xmax/p50=114.4 spk>1ms=1@226 spk>5ms=1@226 ops/s=11820.6 MB/s=0.045 ops=376 samp=376 heap_d=0 +[SLO] kv_put OK (max=7439<=25000, spk>5ms=1<=30) +[PHASE] kv_put cold_ops=64 cold_avg=64.484 us steady_ops=312 steady_avg=88.724 us +[BENCH] kv_get total=3.323 ms avg=8.838 us p50=9 p95=10 min=6 max=18 max_op~353 xmax/p50=2.0 spk>1ms=0@0 spk>5ms=0@0 ops/s=113150.8 MB/s=0.432 ops=376 samp=376 heap_d=0 +[SLO] kv_get OK (max=18<=25000, spk>5ms=0<=30) +[PHASE] kv_get cold_ops=64 cold_avg=8.109 us steady_ops=312 steady_avg=8.987 us +[BENCH] kv_del total=44.980 ms avg=119.628 us p50=98 p95=103 min=96 max=7737 max_op~115 xmax/p50=78.9 spk>1ms=1@115 spk>5ms=1@115 ops/s=8359.3 MB/s=0.000 ops=376 samp=376 heap_d=0 +[SLO] kv_del OK (max=7737<=25000, spk>5ms=1<=30) +[PHASE] kv_del cold_ops=64 cold_avg=99.578 us steady_ops=312 steady_avg=123.740 us +[CHECK] KV benchmark: PASS +[BENCH] ts_insert total=142.388 ms avg=59.328 us p50=20 p95=21 min=20 max=44 max_op~0 xmax/p50=2.2 spk>1ms=0@0 spk>5ms=0@0 ops/s=16855.4 MB/s=0.064 ops=2400 samp=1200 heap_d=0 +[SLO] ts_insert OK (max=44<=25000, spk>5ms=0<=30) +[PHASE] ts_insert cold_ops=64 cold_avg=116.969 us steady_ops=2336 steady_avg=57.749 us +[TS] target=2400 retained=716 dropped=1684 +[BENCH] ts_query_buf total=0.154 ms avg=154.000 us p50=154 p95=154 min=154 max=154 max_op~0 xmax/p50=1.0 spk>1ms=0@0 spk>5ms=0@0 ops/s=6493.5 MB/s=7.927 ops=1 samp=1 heap_d=0 +[CHECK] TS benchmark: PASS +[BENCH] rel_insert total=350.528 ms avg=292.107 us p50=182 p95=664 min=23 max=12554 max_op~1141 xmax/p50=69.0 spk>1ms=6@121 spk>5ms=6@121 ops/s=3423.4 MB/s=0.026 ops=1200 samp=1200 heap_d=0 +[SLO] rel_insert OK (max=12554<=25000, spk>5ms=6<=30) +[PHASE] rel_insert cold_ops=64 cold_avg=25.609 us steady_ops=1136 steady_avg=307.121 us +[BENCH] rel_find(index) total=0.017 ms avg=17.000 us p50=17 p95=17 min=17 max=17 max_op~0 xmax/p50=1.0 spk>1ms=0@0 spk>5ms=0@0 ops/s=58823.5 MB/s=0.449 ops=1 samp=1 heap_d=0 +[REL] rows_expected=1200 rows_actual=1200 +[CHECK] REL benchmark: PASS +[WAL] baseline before warmup: used=0 total=8160 fill=0% +[BENCH] wal_kv_put total=229.437 ms avg=275.766 us p50=80 p95=86 min=78 max=15866 max_op~169 xmax/p50=198.3 spk>1ms=1@169 spk>5ms=1@169 ops/s=3626.3 MB/s=0.221 ops=832 samp=64 heap_d=0 +[SLO] wal_kv_put OK (max=15866<=25000, spk>5ms=1<=30) +[WAL] warmup target_fill=80% reached=80% peak=80% ops_done=832/25600 steady_ops=768 (min=256) +[PHASE] wal_kv_put cold_ops=64 cold_avg=79.062 us steady_ops=768 steady_avg=292.158 us +[WAL] before compact: used=6528 total=8160 fill=80% +[BENCH] compact total=18.734 ms avg=18734.000 us p50=18734 p95=18734 min=18734 max=18734 max_op~0 xmax/p50=1.0 spk>1ms=1@0 spk>5ms=1@0 ops/s=53.4 MB/s=0.000 ops=1 samp=1 heap_d=0 +[WAL] after compact: used=0 total=8160 fill=0% +[CHECK] WAL compact: PASS +[BENCH] reopen total=295.806 ms avg=295806.000 us p50=295806 p95=295806 min=295806 max=295806 max_op~0 xmax/p50=1.0 spk>1ms=1@0 spk>5ms=1@0 ops/s=3.4 MB/s=0.000 ops=1 samp=1 heap_d=0 +[CHECK] Reopen integrity: PASS +[MIGRATE] calls=1 old=1 new=2 +[CHECK] Migration callback: PASS +[CHECK] TXN commit/rollback: PASS +[STATS] kv=322/376 (85%) coll=126 evict=0 +[STATS] ts_streams=1 ts_samples=716 ts_fill=12% +[STATS] wal=96/8160 (1%) rel_tables=2 rel_rows=1200 +=== loxdb ESP32-S3 benchmark end === +loxdb-bench> [METRICS] count=10 +[METRIC] kv_put total=31.809ms avg=84.598us p50=65 p95=69 max=7439@226 xmax/p50=114.4 spk>1ms=1@226 spk>5ms=1@226 ops/s=11820.6 MB/s=0.045 heap_d=0 +[METRIC] kv_get total=3.323ms avg=8.838us p50=9 p95=10 max=18@353 xmax/p50=2.0 spk>1ms=0@0 spk>5ms=0@0 ops/s=113150.8 MB/s=0.432 heap_d=0 +[METRIC] kv_del total=44.980ms avg=119.628us p50=98 p95=103 max=7737@115 xmax/p50=78.9 spk>1ms=1@115 spk>5ms=1@115 ops/s=8359.3 MB/s=0.000 heap_d=0 +[METRIC] ts_insert total=142.388ms avg=59.328us p50=20 p95=21 max=44@0 xmax/p50=2.2 spk>1ms=0@0 spk>5ms=0@0 ops/s=16855.4 MB/s=0.064 heap_d=0 +[METRIC] ts_query_buf total=0.154ms avg=154.000us p50=154 p95=154 max=154@0 xmax/p50=1.0 spk>1ms=0@0 spk>5ms=0@0 ops/s=6493.5 MB/s=7.927 heap_d=0 +[METRIC] rel_insert total=350.528ms avg=292.107us p50=182 p95=664 max=12554@1141 xmax/p50=69.0 spk>1ms=6@121 spk>5ms=6@121 ops/s=3423.4 MB/s=0.026 heap_d=0 +[METRIC] rel_find(index) total=0.017ms avg=17.000us p50=17 p95=17 max=17@0 xmax/p50=1.0 spk>1ms=0@0 spk>5ms=0@0 ops/s=58823.5 MB/s=0.449 heap_d=0 +[METRIC] wal_kv_put total=229.437ms avg=275.766us p50=80 p95=86 max=15866@169 xmax/p50=198.3 spk>1ms=1@169 spk>5ms=1@169 ops/s=3626.3 MB/s=0.221 heap_d=0 +[METRIC] compact total=18.734ms avg=18734.000us p50=18734 p95=18734 max=18734@0 xmax/p50=1.0 spk>1ms=1@0 spk>5ms=1@0 ops/s=53.4 MB/s=0.000 heap_d=0 +[METRIC] reopen total=295.806ms avg=295806.000us p50=295806 p95=295806 max=295806@0 xmax/p50=1.0 spk>1ms=1@0 spk>5ms=1@0 ops/s=3.4 MB/s=0.000 heap_d=0 +loxdb-bench> \ No newline at end of file diff --git a/docs/results/sd_stress_template.md b/docs/results/sd_stress_template.md new file mode 100644 index 0000000..1e07c09 --- /dev/null +++ b/docs/results/sd_stress_template.md @@ -0,0 +1,37 @@ +# SD stress run template + +Fill this file after a significant SD endurance run (soak/stress) to capture release evidence. + +## Setup + +- Date: +- Repo commit: +- Board: +- SD card (brand/model/capacity): +- Arduino-ESP32 core: +- FQBN: +- Power supply: + +## Run configuration + +- Profile: `smoke | soak | stress` +- Mode: `all | kv | ts | rel` +- Verify: `on | off` +- Duration: +- Storage file: `/loxdb_stress_store.bin` + +## Artifacts + +- Raw log: `docs/results/esp32_sd_stress_...log` +- Pressure CSV: `docs/results/esp32_sd_stress_...csv` +- Run note: `docs/results/esp32_sd_stress_...md` + +## Summary + +- Start pressure: +- End pressure: +- Max risk%: +- Compactions: +- Verify failures: +- Notes (spikes, errors, resets): + diff --git a/docs/social-preview-1280x640.png b/docs/social-preview-1280x640.png new file mode 100644 index 0000000..e2eb423 Binary files /dev/null and b/docs/social-preview-1280x640.png differ diff --git a/library.json b/library.json new file mode 100644 index 0000000..6c60fd5 --- /dev/null +++ b/library.json @@ -0,0 +1,30 @@ +{ + "name": "loxdb", + "version": "1.4.0", + "description": "Predictable-memory embedded database (C99) with KV, time-series, and relational engines plus WAL recovery.", + "keywords": [ + "embedded", + "database", + "wal", + "kv", + "timeseries", + "relational", + "esp32" + ], + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/Vanderhell/loxdb.git" + }, + "frameworks": [ + "arduino", + "espidf" + ], + "platforms": "*", + "headers": "lox.h", + "build": { + "includeDir": "include", + "srcDir": "src" + } +} + diff --git a/library.properties b/library.properties new file mode 100644 index 0000000..7a78987 --- /dev/null +++ b/library.properties @@ -0,0 +1,9 @@ +name=loxdb +version=1.4.0 +author=Vanderhell +maintainer=Vanderhell +sentence=Predictable-memory embedded database (C99) with KV, time-series, relational engines, and WAL recovery. +paragraph=loxdb is a compact embedded database for microcontrollers and edge runtimes. One API surface over KV/TS/REL engines, with optional persistence via a storage HAL and WAL recovery. +category=Data Storage +url=https://github.com/Vanderhell/loxdb +architectures=esp32 diff --git a/scripts/run_esp32_bench_and_update_docs.ps1 b/scripts/run_esp32_bench_and_update_docs.ps1 new file mode 100644 index 0000000..9480ea7 --- /dev/null +++ b/scripts/run_esp32_bench_and_update_docs.ps1 @@ -0,0 +1,81 @@ +# SPDX-License-Identifier: MIT +param( + [Parameter(Mandatory = $true)][string]$Port, + [int]$Baud = 115200, + [int]$OpenTimeoutSec = 30, + [int]$RunTimeoutSec = 240, + [switch]$RunStress, + [string]$BenchRoot = "bench/loxdb_esp32_s3_bench_head", + [string]$OutDir = "docs/results" +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +function Ensure-Dir { + param([Parameter(Mandatory = $true)][string]$Path) + if (-not (Test-Path -LiteralPath $Path)) { + New-Item -ItemType Directory -Path $Path | Out-Null + } +} + +Ensure-Dir -Path $OutDir + +$sha = (git rev-parse --short HEAD).Trim() +$ts = Get-Date -Format "yyyyMMdd_HHmmss" + +$runBench = Join-Path $BenchRoot "run_bench.ps1" +if (-not (Test-Path -LiteralPath $runBench)) { + throw "Bench runner not found: $runBench" +} + +$detLog = Join-Path $OutDir "esp32_deterministic_${ts}_${sha}_$($Port.ToLower()).log" +$balLog = Join-Path $OutDir "esp32_balanced_${ts}_${sha}_$($Port.ToLower()).log" +$stressLog = Join-Path $OutDir "esp32_stress_${ts}_${sha}_$($Port.ToLower()).log" + +Write-Host "== Running deterministic on $Port ==" +& $runBench ` + -Port $Port ` + -Baud $Baud ` + -OpenTimeoutSec $OpenTimeoutSec ` + -RunTimeoutSec $RunTimeoutSec ` + -CommandScript "resetdb;run_det;metrics" ` + -LogPath $detLog + +Write-Host "" +Write-Host "== Running balanced on $Port ==" +& $runBench ` + -Port $Port ` + -Baud $Baud ` + -OpenTimeoutSec $OpenTimeoutSec ` + -RunTimeoutSec $RunTimeoutSec ` + -CommandScript "profile balanced;run;metrics" ` + -LogPath $balLog + +if ($RunStress) { + Write-Host "" + Write-Host "== Running stress on $Port ==" + & $runBench ` + -Port $Port ` + -Baud $Baud ` + -OpenTimeoutSec $OpenTimeoutSec ` + -RunTimeoutSec ([Math]::Max($RunTimeoutSec, 900)) ` + -CommandScript "profile stress;run;metrics" ` + -LogPath $stressLog +} + +Write-Host "" +Write-Host "== Updating docs/BENCHMARKS.md ==" +if ($RunStress) { + & "scripts/update_benchmarks_md.ps1" -DeterministicLog $detLog -BalancedLog $balLog -StressLog $stressLog -OutPath "docs/BENCHMARKS.md" +} else { + & "scripts/update_benchmarks_md.ps1" -DeterministicLog $detLog -BalancedLog $balLog -OutPath "docs/BENCHMARKS.md" +} + +Write-Host "" +Write-Host "Logs:" +Write-Host " $detLog" +Write-Host " $balLog" +if ($RunStress) { + Write-Host " $stressLog" +} diff --git a/scripts/run_sd_stress_bench.ps1 b/scripts/run_sd_stress_bench.ps1 new file mode 100644 index 0000000..e3748f0 --- /dev/null +++ b/scripts/run_sd_stress_bench.ps1 @@ -0,0 +1,222 @@ +# SPDX-License-Identifier: MIT +param( + [Parameter(Mandatory = $true)][string]$Port, + [int]$Baud = 115200, + [int]$OpenTimeoutSec = 45, + [int]$DurationSec = 120, + [string]$Profile = "soak", # smoke|soak|stress + [string]$Mode = "all", # all|kv|ts|rel + [string]$Verify = "on", # on|off + [switch]$ResetDb, + [switch]$FormatDb, + [string]$OutDir = "docs/results" +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +function Ensure-Dir { + param([Parameter(Mandatory = $true)][string]$Path) + if (-not (Test-Path -LiteralPath $Path)) { + New-Item -ItemType Directory -Path $Path | Out-Null + } +} + +function Read-UntilAnyPattern { + param( + [Parameter(Mandatory = $true)][System.IO.Ports.SerialPort]$Serial, + [Parameter(Mandatory = $true)][string[]]$Patterns, + [Parameter(Mandatory = $true)][int]$TimeoutSec, + [ref]$Buffer, + [ref]$MatchedPattern + ) + $deadline = (Get-Date).AddSeconds($TimeoutSec) + while ((Get-Date) -lt $deadline) { + try { + $chunk = $Serial.ReadExisting() + if (-not [string]::IsNullOrEmpty($chunk)) { + $Buffer.Value += $chunk + foreach ($pattern in $Patterns) { + if ($Buffer.Value -match [regex]::Escape($pattern)) { + $MatchedPattern.Value = $pattern + return $true + } + } + } + } catch { + } + Start-Sleep -Milliseconds 50 + } + return $false +} + +function Write-Cmd { + param( + [Parameter(Mandatory = $true)][System.IO.Ports.SerialPort]$Serial, + [Parameter(Mandatory = $true)][string]$Cmd + ) + $Serial.WriteLine($Cmd) +} + +Ensure-Dir -Path $OutDir + +$sha = (git rev-parse --short HEAD).Trim() +$ts = Get-Date -Format "yyyyMMdd_HHmmss" +$portTag = $Port.ToLower() + +$rawLogPath = Join-Path $OutDir "esp32_sd_stress_${ts}_${sha}_${portTag}.log" +$csvPath = Join-Path $OutDir "esp32_sd_stress_${ts}_${sha}_${portTag}.csv" +$mdPath = Join-Path $OutDir "esp32_sd_stress_${ts}_${sha}_${portTag}.md" + +$serial = New-Object System.IO.Ports.SerialPort $Port, $Baud, ([System.IO.Ports.Parity]::None), 8, ([System.IO.Ports.StopBits]::One) +$serial.NewLine = "`n" +$serial.ReadTimeout = 100 +$serial.WriteTimeout = 1000 +$serial.DtrEnable = $false +$serial.RtsEnable = $false + +$fullLog = "" +$csvRows = New-Object System.Collections.Generic.List[object] +$admissionLine = $null +$admittedProfileLine = $null +$sdReady = "[OK] loxdb SD stress bench ready" +$fatal = "[FATAL]" +$wrongBenchPrompts = @("loxdb-bench>", "microdb-bench>") +$readyPatterns = @($sdReady, $fatal) + $wrongBenchPrompts + +try { + Write-Host "Opening $Port @ $Baud..." + $serial.Open() + Start-Sleep -Milliseconds 800 + + # Wake output. + $serial.WriteLine("") + + $buf = "" + $matched = "" + $ready = Read-UntilAnyPattern -Serial $serial -Patterns $readyPatterns -TimeoutSec $OpenTimeoutSec -Buffer ([ref]$buf) -MatchedPattern ([ref]$matched) + $fullLog += $buf + if (-not $ready) { + throw "SD stress bench ready marker not detected on $Port within $OpenTimeoutSec s." + } + if ($matched -eq $fatal) { + throw "Device reported [FATAL] during startup." + } + if ($wrongBenchPrompts -contains $matched) { + throw "Detected terminal bench firmware ($matched). Flash bench/loxdb_esp32_s3_sd_stress_bench/loxdb_esp32_s3_sd_stress_bench.ino to $Port and retry." + } + + # Deterministic start: pause -> (format/reset) -> config -> run. + Write-Cmd -Serial $serial -Cmd "pause" + Start-Sleep -Milliseconds 100 + + # Admission is controlled by profile + (re)init. Select profile first. + Write-Cmd -Serial $serial -Cmd ("profile {0}" -f $Profile) + Start-Sleep -Milliseconds 80 + + if ($FormatDb) { + Write-Host "Formatting DB image..." + Write-Cmd -Serial $serial -Cmd "formatdb" + Start-Sleep -Milliseconds 200 + } elseif ($ResetDb) { + Write-Cmd -Serial $serial -Cmd "resetdb" + Start-Sleep -Milliseconds 200 + } else { + # Re-admit without wiping image (still deterministic for admission config). + Write-Cmd -Serial $serial -Cmd "reinit" + Start-Sleep -Milliseconds 200 + } + + Write-Cmd -Serial $serial -Cmd ("mode {0}" -f $Mode) + Write-Cmd -Serial $serial -Cmd ("verify {0}" -f $Verify) + Write-Cmd -Serial $serial -Cmd "stats" + Write-Cmd -Serial $serial -Cmd "run" + + $start = Get-Date + $deadline = $start.AddSeconds($DurationSec) + while ((Get-Date) -lt $deadline) { + $chunk = "" + try { $chunk = $serial.ReadExisting() } catch { $chunk = "" } + + if (-not [string]::IsNullOrEmpty($chunk)) { + $fullLog += $chunk + + foreach ($line in ($chunk -split "`r?`n")) { + if ($line -match "^\[PRESSURE\]\s+kv=(\d+)\s+ts=(\d+)\s+rel=(\d+)\s+wal=(\d+)\s+risk=(\d+)\s+ops=(\d+)") { + $csvRows.Add([pscustomobject]@{ + ts_iso = (Get-Date).ToString("o") + kv_pct = [int]$Matches[1] + ts_pct = [int]$Matches[2] + rel_pct = [int]$Matches[3] + wal_pct = [int]$Matches[4] + risk_pct = [int]$Matches[5] + ops = [int]$Matches[6] + }) + } + if ($line -match "^ADMISSION\s+") { $admissionLine = $line.Trim() } + if ($line -match "^\[OK\]\s+admitted\s+profile=") { $admittedProfileLine = $line.Trim() } + if ($line -match "^\[FATAL\]") { + throw "Device reported [FATAL] during run." + } + } + } + + Start-Sleep -Milliseconds 50 + } + + Write-Cmd -Serial $serial -Cmd "pause" + Write-Cmd -Serial $serial -Cmd "stats" + Start-Sleep -Milliseconds 250 + + # Drain tail output. + $tail = "" + try { $tail = $serial.ReadExisting() } catch { $tail = "" } + if (-not [string]::IsNullOrEmpty($tail)) { $fullLog += $tail } +} +finally { + $utf8Bom = New-Object System.Text.UTF8Encoding $true + [System.IO.File]::WriteAllText($rawLogPath, $fullLog, $utf8Bom) + + if ($csvRows.Count -gt 0) { + $csvRows | Export-Csv -NoTypeInformation -Encoding UTF8 -Path $csvPath + } + + $md = @() + $md += ("# SD stress run - {0}" -f $ts) + $md += "" + $md += ("- port: {0}" -f $Port) + $md += ("- repo commit: {0}" -f $sha) + $md += ("- profile: {0}" -f $Profile) + $md += ("- mode: {0}" -f $Mode) + $md += ("- verify: {0}" -f $Verify) + $md += ("- duration: {0}s" -f $DurationSec) + $md += ("- resetdb: {0}" -f ($(if ($ResetDb) { "yes" } else { "no" }))) + $md += ("- formatdb: {0}" -f ($(if ($FormatDb) { "yes" } else { "no" }))) + $md += "" + if ($admissionLine -or $admittedProfileLine) { + $md += "Admission:" + $md += "" + if ($admittedProfileLine) { $md += ("- {0}" -f $admittedProfileLine) } + if ($admissionLine) { $md += ("- {0}" -f $admissionLine) } + $md += "" + } + $md += "Artifacts:" + $md += "" + $md += ("- raw log: {0}" -f ([IO.Path]::GetFileName($rawLogPath))) + if (Test-Path -LiteralPath $csvPath) { + $md += ("- pressure CSV: {0}" -f ([IO.Path]::GetFileName($csvPath))) + } + $md += "" + $md += "Notes:" + $md += "" + $md += "- The bench prints `[PRESSURE]`, `[STATS]`, `[BENCH]` lines periodically; see the raw log for full context." + + [System.IO.File]::WriteAllText($mdPath, ($md -join "`r`n"), $utf8Bom) + + if ($serial.IsOpen) { $serial.Close() } +} + +Write-Host "Saved:" +Write-Host " $rawLogPath" +Write-Host " $csvPath" +Write-Host " $mdPath" diff --git a/scripts/update_benchmarks_md.ps1 b/scripts/update_benchmarks_md.ps1 new file mode 100644 index 0000000..569abf9 --- /dev/null +++ b/scripts/update_benchmarks_md.ps1 @@ -0,0 +1,199 @@ +# SPDX-License-Identifier: MIT +param( + [Parameter(Mandatory = $true)][string]$DeterministicLog, + [string]$BalancedLog = "", + [string]$StressLog = "", + [string]$OutPath = "docs/BENCHMARKS.md" +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +function Parse-MetricsFromLog { + param([Parameter(Mandatory = $true)][string]$Path) + + if (-not (Test-Path -LiteralPath $Path)) { + throw "Log not found: $Path" + } + + $text = Get-Content -LiteralPath $Path -Raw + + $rowsExpected = $null + $tsRetained = $null + foreach ($line in ($text -split "`r?`n")) { + if ($null -eq $rowsExpected -and $line -match "^\[REL\]\s+rows_expected=(\d+)") { + $rowsExpected = [int]$Matches[1] + } + if ($null -eq $tsRetained -and $line -match "^\[TS\]\s+target=\d+\s+retained=(\d+)") { + $tsRetained = [int]$Matches[1] + } + } + + $metrics = @{} + foreach ($line in ($text -split "`r?`n")) { + if ($line -match "^\[METRIC\]\s+(?.+?)\s+total=(?[0-9.]+)ms\s+avg=(?[0-9.]+)us\s+p50=(?\d+)\s+p95=(?\d+)\s+max=(?\d+)@(?\d+).*?\s+ops/s=(?[0-9.]+)\b") { + $op = $Matches["op"].Trim() + $metrics[$op] = [pscustomobject]@{ + op = $op + total_ms = [double]$Matches["total_ms"] + avg_us = [double]$Matches["avg_us"] + p50_us = [int]$Matches["p50"] + p95_us = [int]$Matches["p95"] + max_us = [int]$Matches["max"] + ops_s = [double]$Matches["ops_s"] + } + } + } + + return [pscustomobject]@{ + text = $text + metrics = $metrics + rowsExpected = $rowsExpected + tsRetained = $tsRetained + } +} + +function Fmt-Num { + param([Parameter(Mandatory = $true)]$Value) + if ($Value -is [double]) { + return ([string]::Format([System.Globalization.CultureInfo]::InvariantCulture, "{0:0.0}", $Value)) + } + return "$Value" +} + +function Require-Metric { + param( + [Parameter(Mandatory = $true)][hashtable]$Metrics, + [Parameter(Mandatory = $true)][string]$Op + ) + if (-not $Metrics.ContainsKey($Op)) { + $known = ($Metrics.Keys | Sort-Object) -join ", " + throw "Metric '$Op' not found in log. Known: $known" + } + return $Metrics[$Op] +} + +$det = Parse-MetricsFromLog -Path $DeterministicLog +$bal = $null +if (-not [string]::IsNullOrWhiteSpace($BalancedLog)) { + $bal = Parse-MetricsFromLog -Path $BalancedLog +} +$stress = $null +if (-not [string]::IsNullOrWhiteSpace($StressLog)) { + $stress = Parse-MetricsFromLog -Path $StressLog +} + +$kv_put = Require-Metric -Metrics $det.metrics -Op "kv_put" +$kv_get = Require-Metric -Metrics $det.metrics -Op "kv_get" +$kv_del = Require-Metric -Metrics $det.metrics -Op "kv_del" +$ts_ins = Require-Metric -Metrics $det.metrics -Op "ts_insert" +$ts_q = Require-Metric -Metrics $det.metrics -Op "ts_query_buf" +$rel_i = Require-Metric -Metrics $det.metrics -Op "rel_insert" +$rel_f = Require-Metric -Metrics $det.metrics -Op "rel_find(index)" +$wal_kv = Require-Metric -Metrics $det.metrics -Op "wal_kv_put" +$compact = Require-Metric -Metrics $det.metrics -Op "compact" +$reopen = Require-Metric -Metrics $det.metrics -Op "reopen" + +$detName = Split-Path -Leaf $DeterministicLog +$balName = if ($bal) { Split-Path -Leaf $BalancedLog } else { "" } + +$relRows = if ($null -ne $det.rowsExpected) { $det.rowsExpected } else { "TBD" } +$tsTypeNote = if ($null -ne $det.tsRetained) { "retained=$($det.tsRetained)" } else { "retained=TBD" } + +$generatedLines = @() +$generatedLines += '## Results - KV engine (deterministic profile)' +$generatedLines += '' +$generatedLines += '| Operation | p50 (us) | p95 (us) | max (us) | throughput (ops/s) | Notes |' +$generatedLines += '|---|---:|---:|---:|---:|---|' +$generatedLines += ('| `kv_put` | {0} | {1} | {2} | {3} | `{4}` |' -f $kv_put.p50_us, $kv_put.p95_us, $kv_put.max_us, (Fmt-Num $kv_put.ops_s), $detName) +$generatedLines += ('| `kv_get` | {0} | {1} | {2} | {3} | `{4}` |' -f $kv_get.p50_us, $kv_get.p95_us, $kv_get.max_us, (Fmt-Num $kv_get.ops_s), $detName) +$generatedLines += ('| `kv_del` | {0} | {1} | {2} | {3} | `{4}` |' -f $kv_del.p50_us, $kv_del.p95_us, $kv_del.max_us, (Fmt-Num $kv_del.ops_s), $detName) +$generatedLines += '' +$generatedLines += 'WAL impact (KV):' +$generatedLines += ('- `wal_kv_put` p50/p95/max: {0}/{1}/{2} us (`{3}`)' -f $wal_kv.p50_us, $wal_kv.p95_us, $wal_kv.max_us, $detName) +$generatedLines += '' +$generatedLines += '## Results - TS engine (deterministic profile)' +$generatedLines += '' +$generatedLines += '| Stream type | insert rate (samples/s) | query p50 (us) | query p95 (us) | Notes |' +$generatedLines += '|---|---:|---:|---:|---|' +$generatedLines += ('| `F32` | {0} | {1} | {2} | `{3}` ({4}) |' -f (Fmt-Num $ts_ins.ops_s), $ts_q.p50_us, $ts_q.p95_us, $detName, $tsTypeNote) +$generatedLines += '| `I32` | TBD | TBD | TBD | |' +$generatedLines += '| `U32` | TBD | TBD | TBD | |' +$generatedLines += '| `RAW` | TBD | TBD | TBD | |' +$generatedLines += '' +$generatedLines += '## Results - REL engine (deterministic profile)' +$generatedLines += '' +$generatedLines += '| Rows (N) | insert p50 (us) | find_by_index p50 (us) | Notes |' +$generatedLines += '|---:|---:|---:|---|' +$generatedLines += ('| {0} | {1} | {2} | `{3}` |' -f $relRows, $rel_i.p50_us, $rel_f.p50_us, $detName) +$generatedLines += '' +$generatedLines += '## WAL / maintenance (deterministic profile)' +$generatedLines += '' +$generatedLines += '| Operation | total (ms) | Notes |' +$generatedLines += '|---|---:|---|' +$generatedLines += ('| `compact` | {0} | `{1}` |' -f ([string]::Format([System.Globalization.CultureInfo]::InvariantCulture, "{0:0.###}", $compact.total_ms)), $detName) +$generatedLines += ('| `reopen` | {0} | `{1}` |' -f ([string]::Format([System.Globalization.CultureInfo]::InvariantCulture, "{0:0.###}", $reopen.total_ms)), $detName) + +$generated = ($generatedLines -join "`r`n") + +if ($bal) { + $b_kv_put = Require-Metric -Metrics $bal.metrics -Op "kv_put" + $b_kv_get = Require-Metric -Metrics $bal.metrics -Op "kv_get" + $b_kv_del = Require-Metric -Metrics $bal.metrics -Op "kv_del" + $b_ts_ins = Require-Metric -Metrics $bal.metrics -Op "ts_insert" + $b_rel_i = Require-Metric -Metrics $bal.metrics -Op "rel_insert" + $generated += "`r`n`r`n## Throughput reference - balanced profile`r`n`r`n" + $generated += "| Operation | throughput (ops/s) | Notes |`r`n" + $generated += "|---|---:|---|`r`n" + $generated += ('| `kv_put` | {0} | `{1}` |' -f (Fmt-Num $b_kv_put.ops_s), $balName) + "`r`n" + $generated += ('| `kv_get` | {0} | `{1}` |' -f (Fmt-Num $b_kv_get.ops_s), $balName) + "`r`n" + $generated += ('| `kv_del` | {0} | `{1}` |' -f (Fmt-Num $b_kv_del.ops_s), $balName) + "`r`n" + $generated += ('| `ts_insert` | {0} | `{1}` |' -f (Fmt-Num $b_ts_ins.ops_s), $balName) + "`r`n" + $generated += ('| `rel_insert` | {0} | `{1}` |' -f (Fmt-Num $b_rel_i.ops_s), $balName) + "`r`n" +} + +if ($stress) { + $s_kv_put = Require-Metric -Metrics $stress.metrics -Op "kv_put" + $s_kv_get = Require-Metric -Metrics $stress.metrics -Op "kv_get" + $s_kv_del = Require-Metric -Metrics $stress.metrics -Op "kv_del" + $s_ts_ins = Require-Metric -Metrics $stress.metrics -Op "ts_insert" + $s_rel_i = Require-Metric -Metrics $stress.metrics -Op "rel_insert" + $s_wal_kv = Require-Metric -Metrics $stress.metrics -Op "wal_kv_put" + $s_compact = Require-Metric -Metrics $stress.metrics -Op "compact" + $s_reopen = Require-Metric -Metrics $stress.metrics -Op "reopen" + $stressName = Split-Path -Leaf $StressLog + + $stressTsNote = if ($null -ne $stress.tsRetained) { "retained=$($stress.tsRetained)" } else { "retained=TBD" } + $stressRelRows = if ($null -ne $stress.rowsExpected) { $stress.rowsExpected } else { "TBD" } + + $generated += "`r`n`r`n## Stress profile reference`r`n`r`n" + $generated += "| Metric | Value | Notes |`r`n" + $generated += "|---|---:|---|`r`n" + $generated += ('| `kv_put` throughput (ops/s) | {0} | `{1}` |' -f (Fmt-Num $s_kv_put.ops_s), $stressName) + "`r`n" + $generated += ('| `kv_get` throughput (ops/s) | {0} | `{1}` |' -f (Fmt-Num $s_kv_get.ops_s), $stressName) + "`r`n" + $generated += ('| `kv_del` throughput (ops/s) | {0} | `{1}` |' -f (Fmt-Num $s_kv_del.ops_s), $stressName) + "`r`n" + $generated += ('| `ts_insert` throughput (samples/s) | {0} | `{1}` ({2}) |' -f (Fmt-Num $s_ts_ins.ops_s), $stressName, $stressTsNote) + "`r`n" + $generated += ('| `rel_insert` throughput (rows/s) | {0} | `{1}` (N={2}) |' -f (Fmt-Num $s_rel_i.ops_s), $stressName, $stressRelRows) + "`r`n" + $generated += ('| `wal_kv_put` throughput (ops/s) | {0} | `{1}` |' -f (Fmt-Num $s_wal_kv.ops_s), $stressName) + "`r`n" + $generated += ('| `compact` total (ms) | {0} | `{1}` |' -f ([string]::Format([System.Globalization.CultureInfo]::InvariantCulture, "{0:0.###}", $s_compact.total_ms)), $stressName) + "`r`n" + $generated += ('| `reopen` total (ms) | {0} | `{1}` |' -f ([string]::Format([System.Globalization.CultureInfo]::InvariantCulture, "{0:0.###}", $s_reopen.total_ms)), $stressName) + "`r`n" +} + +$md = Get-Content -LiteralPath $OutPath -Raw +$begin = "" +$end = "" + +$beginIndex = $md.IndexOf($begin) +$endIndex = $md.IndexOf($end) +if ($beginIndex -lt 0 -or $endIndex -lt 0 -or $endIndex -le $beginIndex) { + throw "Markers not found or invalid in $OutPath. Expected $begin ... $end" +} + +$before = $md.Substring(0, $beginIndex + $begin.Length) +$after = $md.Substring($endIndex) + +$newMd = $before + "`r`n" + $generated.Trim() + "`r`n" + $after +$utf8Bom = New-Object System.Text.UTF8Encoding $true +[System.IO.File]::WriteAllText($OutPath, $newMd, $utf8Bom) + +Write-Host "Updated: $OutPath" diff --git a/scripts/validate_arduino_bench_layout.ps1 b/scripts/validate_arduino_bench_layout.ps1 new file mode 100644 index 0000000..6c4e686 --- /dev/null +++ b/scripts/validate_arduino_bench_layout.ps1 @@ -0,0 +1,91 @@ +# SPDX-License-Identifier: MIT +param( + [string]$HeadBench = "bench/loxdb_esp32_s3_bench_head", + [string]$BaseBench = "bench/loxdb_esp32_s3_bench_base" +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +function Assert-Exists { + param([Parameter(Mandatory = $true)][string]$Path) + if (-not (Test-Path -LiteralPath $Path)) { + throw "Missing: $Path" + } +} + +function Get-Includes { + param([Parameter(Mandatory = $true)][string]$Root) + $includes = @() + Get-ChildItem -LiteralPath $Root -Recurse -File -Include *.c,*.h,*.ino | ForEach-Object { + $file = $_.FullName + Get-Content -LiteralPath $file | ForEach-Object { + if ($_ -match '^\s*#include\s+"([^"]+)"') { + $includes += [pscustomobject]@{ file = $file; include = $Matches[1] } + } + } + } + return $includes +} + +function Validate-Bench { + param( + [Parameter(Mandatory = $true)][string]$Name, + [Parameter(Mandatory = $true)][string]$Root, + [Parameter(Mandatory = $true)][string[]]$SearchPaths + ) + + Write-Host "== $Name ==" + Assert-Exists $Root + + $missing = @() + $includes = Get-Includes -Root $Root + foreach ($inc in $includes) { + # Only validate project-local headers; platform/toolchain headers may not exist in the repo. + if ($inc.include -notmatch '^(lox|microdb)[^\\/]*\.h$') { + continue + } + $found = $false + foreach ($sp in $SearchPaths) { + $cand = Join-Path $sp $inc.include + if (Test-Path -LiteralPath $cand) { $found = $true; break } + } + if (-not $found) { + $missing += [pscustomobject]@{ file = $inc.file; include = $inc.include } + } + } + + if ($missing.Count -gt 0) { + Write-Host "Missing includes:" + $missing | Select-Object -First 30 | ForEach-Object { + Write-Host (" {0} -> {1}" -f $_.file, $_.include) + } + if ($missing.Count -gt 30) { + Write-Host " ... ($($missing.Count-30) more)" + } + exit 2 + } + + Write-Host "OK: include graph resolves within search paths." + Write-Host "" +} + +Validate-Bench ` + -Name "HEAD bench (Arduino folder + repo helpers)" ` + -Root $HeadBench ` + -SearchPaths @( + (Join-Path $HeadBench "lox_esp32_s3_bench"), + (Join-Path $HeadBench "lox_esp32_s3_bench/src"), + $HeadBench, + (Join-Path $HeadBench "src") + ) + +Validate-Bench ` + -Name "BASE bench" ` + -Root $BaseBench ` + -SearchPaths @( + $BaseBench, + (Join-Path $BaseBench "src") + ) + +Write-Host "All bench layouts look consistent." diff --git a/tests/fuzz/README.md b/tests/fuzz/README.md new file mode 100644 index 0000000..a48bfc8 --- /dev/null +++ b/tests/fuzz/README.md @@ -0,0 +1,32 @@ +# Fuzzing (libFuzzer) + +This directory contains libFuzzer harness scaffolding for loxdb’s most safety-critical parsers and decoders. + +Important: **scaffolding ≠ proven fuzz coverage**. The initial harnesses provide only minimal input plumbing and should be extended with WAL-format-aware mutators/dictionaries and additional targets as issues/coverage guide the work. + +## Requirements + +- Linux (recommended) with clang/llvm installed +- libFuzzer (ships with clang) + +## Harnesses + +- `fuzz_wal_parser.cpp`: minimal harness that exercises WAL header/entry parsing logic (via `tools/lox_verify.c` WAL inspector). It is not a production-ready fuzz target yet. + +## How to add a new harness + +1. Create a new `fuzz_*.cpp` file with the standard entry point: + - `extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size);` +2. Prefer targeting **pure parsing/decoding** functions (no network/IO), and keep runtime bounded: + - cap input size + - avoid unbounded loops +3. Add a build snippet to `tests/fuzz/build.sh` and a run snippet to `tests/fuzz/run_one.sh`. + +## Local build + run (Linux) + +```bash +./tests/fuzz/build.sh +./tests/fuzz/run_one.sh fuzz_wal_parser 600 +``` + +The second argument is the max runtime in seconds. diff --git a/tests/fuzz/build.sh b/tests/fuzz/build.sh new file mode 100644 index 0000000..baa3e13 --- /dev/null +++ b/tests/fuzz/build.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MIT +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +OUT_DIR="${ROOT_DIR}/build/fuzz" + +mkdir -p "${OUT_DIR}" + +CC=clang +CXX=clang++ + +COMMON_CFLAGS=( + -O1 -g + -fno-omit-frame-pointer +) + +FUZZ_CXXFLAGS=( + -std=c++17 + -fsanitize=fuzzer,address,undefined +) + +INCLUDES=( + -I"${ROOT_DIR}/include" + -I"${ROOT_DIR}/src" +) + +echo "=== build fuzz_wal_parser ===" +${CXX} \ + "${ROOT_DIR}/tests/fuzz/fuzz_wal_parser.cpp" \ + "${ROOT_DIR}/src/lox_crc.c" \ + ${COMMON_CFLAGS[@]} \ + ${FUZZ_CXXFLAGS[@]} \ + ${INCLUDES[@]} \ + -o "${OUT_DIR}/fuzz_wal_parser" + +echo "Built: ${OUT_DIR}/fuzz_wal_parser" diff --git a/tests/fuzz/fuzz_wal_parser.cpp b/tests/fuzz/fuzz_wal_parser.cpp new file mode 100644 index 0000000..18c5776 --- /dev/null +++ b/tests/fuzz/fuzz_wal_parser.cpp @@ -0,0 +1,54 @@ +// SPDX-License-Identifier: MIT +#include +#include +#include +#include +#include + +// Scaffolding-only fuzz harness: +// - Reuses the WAL inspector from the offline verifier to exercise WAL parsing. +// - This is minimal input plumbing (no WAL-format-aware mutators/dictionaries yet). +// - Do not interpret its presence as “WAL parser is fuzz-tested” in a coverage sense. +// Rename verifier's main() so the harness can link. +#define main lox_verify_main +#include "../../tools/lox_verify.c" +#undef main + +static FILE *mem_to_tmpfile(const uint8_t *data, size_t size) { + FILE *fp = tmpfile(); + if (!fp) { + return nullptr; + } + if (size > 0) { + (void)fwrite(data, 1, size, fp); + } + (void)fflush(fp); + (void)fseek(fp, 0, SEEK_SET); + return fp; +} + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { + // Keep runtime bounded and avoid pathological allocations/IO. + if (!data || size == 0) { + return 0; + } + if (size > (128u * 1024u)) { + size = 128u * 1024u; + } + + FILE *fp = mem_to_tmpfile(data, size); + if (!fp) { + return 0; + } + + verify_layout_t layout; + memset(&layout, 0, sizeof(layout)); + layout.wal_offset = 0u; + layout.wal_size = (uint32_t)size; + + // Fuzz WAL header + entry parsing. We ignore the returned verdict; crashes/UB are what matter. + (void)inspect_wal(fp, &layout); + + (void)fclose(fp); + return 0; +} diff --git a/tests/fuzz/run_one.sh b/tests/fuzz/run_one.sh new file mode 100644 index 0000000..204009a --- /dev/null +++ b/tests/fuzz/run_one.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MIT +set -euo pipefail + +NAME="${1:-}" +MAX_TOTAL_TIME_SEC="${2:-600}" + +if [[ -z "${NAME}" ]]; then + echo "Usage: $0 [max_total_time_sec]" + exit 2 +fi + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +BIN="${ROOT_DIR}/build/fuzz/${NAME}" + +if [[ ! -x "${BIN}" ]]; then + echo "Missing harness binary: ${BIN}" + echo "Build first: ./tests/fuzz/build.sh" + exit 2 +fi + +exec "${BIN}" \ + -max_total_time="${MAX_TOTAL_TIME_SEC}" \ + -timeout=5 \ + -rss_limit_mb=2048 \ + -print_final_stats=1 + diff --git a/wiki/Home.md b/wiki/Home.md index 6871a22..245cdb7 100644 --- a/wiki/Home.md +++ b/wiki/Home.md @@ -24,7 +24,7 @@ It combines three engines behind one API: - [Storage HAL](Storage-HAL) - [Testing](Testing) - Programmer manual (repo doc): `docs/PROGRAMMER_MANUAL.md` -- Docs map (repo doc): `docs/DOCS_MAP.md` +- Docs index (repo doc): `docs/README.md` ## Repository diff --git a/wiki/Integration.md b/wiki/Integration.md index fc000bc..fe34cb3 100644 --- a/wiki/Integration.md +++ b/wiki/Integration.md @@ -34,5 +34,5 @@ Use `lox_cfg_t` lock callbacks for multithreaded builds: ## Full Docs Map -- `https://github.com/Vanderhell/loxdb/blob/master/docs/DOCS_MAP.md` +- `https://github.com/Vanderhell/loxdb/blob/master/docs/README.md`