diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b642f56..aa247da 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,7 +2,7 @@ name: CI on: push: - branches: [ main, master ] + branches: [ main, master, 'feature/**' ] tags: ['v*'] pull_request: workflow_dispatch: diff --git a/CHANGELOG.md b/CHANGELOG.md index 47a2f4b..942a8f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,40 @@ All notable changes to this project are documented in this file. The format follows Keep a Changelog and the project adheres to Semantic Versioning. ## [Unreleased] +### Changed +- Moved mutable DB ownership behind an internal runtime and moved file upload / path handling behind a real `FileStore` subsystem. +- `Collection` now uses an internal backing store, enforces `maxDecodedViews` / `maxRecordsInMemory`, and applies revision-based conflict checks in update paths. +- `ESPJsonDB` and `Collection` headers are now thin façades over internal runtime / store state instead of exposing runtime-owned reference members. +- `.jdb` writes now use a single authoritative prefix `flags` field; decode still accepts the interim duplicated-`flags` v2 envelope. + +### Removed +- Compatibility aliases `getDiag()`, `getAllCollectionName()`, and `unRegisterSchema()`. +- Direct file helper methods from `ESPJsonDB`; use `db.files()` only. + +## [2.0.0] - 2026-03-27 +### Added +- Durable `.jdb` record format with persisted `_id`, `createdAtMs`, `updatedAtMs`, `revision`, and `flags`. +- `SnapshotMode::{OnDiskOnly, InMemoryConsistent}` for explicit snapshot semantics. +- `CollectionLoadPolicy::{Eager, Lazy, Delayed}` and `configureCollection(...)`. +- First-class schema `required` fields and typed defaults via `JsonDefaultValue`. +- `FileStore` facade exposed through `db.files()`. +- New status codes including `NotInitialized`, `Conflict`, `Timeout`, `Unsupported`, `SchemaMismatch`, and `CorruptionDetected`. + +### Changed +- Snapshot payloads now include `_meta` alongside `_id` and document fields. +- `getDiagnostics()` replaces `getDiag()`, `listCollectionNames()` replaces `getAllCollectionName()`, and `unregisterSchema()` replaces `unRegisterSchema()`. +- File operations now live only behind `db.files()`, with async state access renamed to `cancelUpload()` / `getUploadState()`. +- Collection persistence now uses `.jdb` files instead of raw `.mp` MessagePack payload files. +- Unique constraints are enforced through in-memory per-field indexes instead of full collection scans. +- The top-level CMake build now uses C++17 to match the library metadata. + +### Removed +- `ESPJsonDBConfig::cacheEnabled` +- `ESPJsonDBConfig::coldSync` +- `ESPJsonDBConfig::delayedCollectionSyncArray` +- Automatic on-disk compatibility with legacy v1 `.mp` record files. + +## [1.1.1] - 2026-03-13 ### Added - `ESPJsonDBConfig::usePSRAMBuffers` to prefer PSRAM for ESPJsonDB-owned byte buffers when available (with automatic heap fallback). - `ESPJsonDBConfig::delayedCollectionSyncArray` to defer selected collection preloads at boot and load them later. @@ -103,7 +137,9 @@ The format follows Keep a Changelog and the project adheres to Semantic Versioni - Event callbacks, diagnostics reporting, and automatic `createdAt` / `updatedAt` timestamps on documents. - Example sketches covering quick start, collections, bulk operations, schema validation, and references. -[Unreleased]: https://github.com/ESPToolKit/esp-jsondb/compare/v1.1.0...HEAD +[Unreleased]: https://github.com/ESPToolKit/esp-jsondb/compare/v2.0.0...HEAD +[2.0.0]: https://github.com/ESPToolKit/esp-jsondb/releases/tag/v2.0.0 +[1.1.1]: https://github.com/ESPToolKit/esp-jsondb/releases/tag/v1.1.1 [1.1.0]: https://github.com/ESPToolKit/esp-jsondb/releases/tag/v1.1.0 [1.0.5]: https://github.com/ESPToolKit/esp-jsondb/releases/tag/v1.0.5 [1.0.4]: https://github.com/ESPToolKit/esp-jsondb/releases/tag/v1.0.4 diff --git a/CMakeLists.txt b/CMakeLists.txt index d8f408c..b86800f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,7 +7,7 @@ project(ESPJsonDB) enable_testing() -set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) if(${COVERAGE}) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4af4a20..355e973 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -120,7 +120,7 @@ Please keep these in mind when contributing: 4. **Autosync task (FreeRTOS)** - The DB may run a background task that flushes dirty documents every `intervalMs`. Keep callbacks **non-blocking**; tune stack/priority/core via `ESPJsonDBConfig`. 5. **On-disk layout & atomic writes** - - Documents are saved as `.mp` under `/baseDir//`. Writes should be **atomic**: write to `*.tmp` then `rename()`. + - Documents are saved as `.jdb` under `/baseDir//`. Writes should be **atomic**: write to `*.tmp` then `rename()`. 6. **Validation hooks** - Collections may have a `Schema` validator. Mutations should run pre-save validation and fail with a clear status when invalid. diff --git a/README.md b/README.md index 1f889dc..b5517df 100644 --- a/README.md +++ b/README.md @@ -1,26 +1,33 @@ # ESPJsonDB -A lightweight document database for ESP32 devices. ESPJsonDB borrows the ergonomics of MongoDB/Mongoose while embracing embedded constraints: collections live as JSON on LittleFS, memory use is capped through an optional cache, and every API leans on ArduinoJson types so you can stay inside a single document representation. +ESPJsonDB is an embedded document database for ESP32 boards. Version 2 stores document payloads as MessagePack inside durable `.jdb` records, keeps document metadata on disk, and separates document storage from generic file storage. ## CI / Release / License [![CI](https://github.com/ESPToolKit/esp-jsondb/actions/workflows/ci.yml/badge.svg)](https://github.com/ESPToolKit/esp-jsondb/actions/workflows/ci.yml) [![Release](https://img.shields.io/github/v/release/ESPToolKit/esp-jsondb?sort=semver)](https://github.com/ESPToolKit/esp-jsondb/releases) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE.md) -## Features -- Simple, mongoose-like API for embedded projects (create/update/remove/find with predicates or JSON filters). -- Optional in-memory cache with dirty-document tracking and change detection to avoid needless flash I/O. -- Automatic LittleFS synchronisation on a background FreeRTOS task (`ESPJsonDBConfig` controls interval, stack, priority, and core affinity). -- Configurable delayed boot preload for selected collections (`delayedCollectionSyncArray`) to reduce cold-start RAM sync work. -- MessagePack compression + StreamUtils for efficient read/write pipelines. -- Schema registry with required fields, defaults, type validation, and collection-level unique constraints. -- Event + error callbacks so firmware can observe sync cycles or take action when validation fails. -- Snapshot/restore helpers for backups plus diagnostics that report per-collection counts and config details. -- Generic file storage helpers under `//_files` with chunked stream read/write for any file type (text, binary, etc.). - -## Examples -Quick start: +## What Changed In v2 +- Documents are persisted as `.jdb` records with durable `_id`, `createdAtMs`, `updatedAtMs`, `revision`, and `flags`. +- Collection loading is policy-driven with `Eager`, `Lazy`, and `Delayed` modes. +- Snapshots are explicit: `SnapshotMode::OnDiskOnly` or `SnapshotMode::InMemoryConsistent`. +- Schema fields support first-class `required`, `unique`, and typed defaults. +- Generic file storage is accessed through `db.files()`. +- The old `cacheEnabled`, `coldSync`, and `delayedCollectionSyncArray` config knobs are gone. +## Features +- MessagePack payload storage with lazy `DocView` decoding. +- Durable per-document metadata and revision counters. +- The current `.jdb` writer uses a prefix-authoritative record envelope and still reads the interim duplicated-`flags` v2 envelope for compatibility. +- Background sync worker for record flush and collection cleanup. +- Per-collection load policy configuration via `configureCollection()`. +- Schema validation with typed defaults and required fields. +- Unique field enforcement backed by in-memory indexes. +- Snapshot / restore for document collections. +- Async file uploads and chunked file I/O through `FileStore`. +- PSRAM-aware internal allocators for payload and buffer-heavy paths. + +## Quick Start ```cpp #include @@ -30,171 +37,124 @@ void setup() { Serial.begin(115200); ESPJsonDBConfig cfg; - cfg.intervalMs = 3000; // autosync every 3s + cfg.intervalMs = 2000; cfg.autosync = true; - cfg.usePSRAMBuffers = true; // optional: prefer PSRAM for internal byte buffers + cfg.defaultLoadPolicy = CollectionLoadPolicy::Eager; + + db.configureCollection("audit", CollectionConfig{CollectionLoadPolicy::Delayed, 0, 0}); - if (!db.init("/test_db", cfg).ok()) { + if (!db.init("/jsondb_v2", cfg).ok()) { Serial.println("DB init failed"); return; } - db.onEvent([](DBEventType evt){ - Serial.printf("Event: %s\n", dbEventTypeToString(evt)); - }); - db.onSyncStatus([](const DBSyncStatus& status){ + Schema users; + users.fields = { + SchemaField{"email", FieldType::String, std::string("a@b.c")}, + SchemaField{"username", FieldType::String}, + SchemaField{"role", FieldType::String, std::string("user")}, + SchemaField{"password", FieldType::String}, + SchemaField{"age", FieldType::Int32}, + }; + users.fields[1].required = true; + users.fields[3].required = true; + db.registerSchema("users", users); + + JsonDocument doc; + doc["username"] = "esp-jsondb"; + doc["password"] = "secret"; + auto created = db.create("users", doc.as()); + if (!created.status.ok()) { + Serial.printf("Create failed: %s\n", created.status.message); + return; + } + + auto found = db.findById("users", created.value); + if (found.status.ok()) { Serial.printf( - "Sync: %s source=%s collection=%s (%lu/%lu)\n", - dbSyncStageToString(status.stage), - dbSyncSourceToString(status.source), - status.collectionName.c_str(), - static_cast(status.collectionsCompleted), - static_cast(status.collectionsTotal) + "revision=%lu createdAtMs=%llu\n", + static_cast(found.value.meta().revision), + static_cast(found.value.meta().createdAtMs) ); - }); - db.onError([](const DbStatus &st){ - Serial.printf("Error: %s\n", st.message); - }); -} + } -void loop() { - // Call db.deinit() before shutting down the feature/task that owns the DB. + auto snap = db.getSnapshot(SnapshotMode::InMemoryConsistent); + serializeJsonPretty(snap, Serial); } -``` -Working with documents is intentionally `JsonDocument`-centric: - -```cpp -JsonDocument doc; -doc["email"] = "user@example.com"; -doc["role"] = "admin"; -auto createRes = db.create("users", doc.as()); - -if (createRes.status.ok()) { - const std::string& id = createRes.value; - auto found = db.findById("users", id); - if (found.status.ok()) { - Serial.printf("Role: %s\n", found.value["role"].as()); - } - db.updateById("users", id, [](DocView& view){ - view["role"].set("owner"); - }); - db.removeById("users", id); +void loop() { } ``` -See the sketches under `examples/` for end-to-end flows: -- `QuickStart` – database initialisation and simple CRUD. -- `Collections` – create/drop collections at runtime. -- `CacheDisabled` – migration note for the removed cache-disabled mode. -- `BulkOperations` – batch inserts, updates, and queries. -- `SchemaValidation` – enforce required fields and custom validators. -- `UniqueFields` – per-collection uniqueness guarantees. -- `References` – store one-to-many relations and populate them lazily. -- `FileStreaming` – store and stream `txt` / `json` / `csv` / `bin` / custom extension payloads. -- `LargeFileStreaming` – chunked upload + chunked verification for a large binary payload without full-buffer RAM copies. -- `AsyncFileUpload` – non-blocking, callback-driven chunk upload on a background task. -- `AsyncLargeFileUpload` – background chunk upload for a large binary payload with progress polling and streaming hash verification. - -File storage example: +## File Storage +Document records and arbitrary file blobs are separate subsystems. ```cpp -ESPJsonDBFileOptions fileOpts; -fileOpts.chunkSize = 256; -db.writeTextFile("notes/readme.txt", "hello from esp-jsondb"); +ESPJsonDBFileOptions opts; +opts.chunkSize = 256; -db.writeFileFromPath("firmware/chunk.bin", "/fw/chunk.bin", fileOpts); +db.files().writeTextFile("notes/readme.txt", "hello"); -db.writeFileStream( - "firmware/chunk_cb.bin", +auto uploadId = db.files().writeFileStreamAsync( + "firmware/chunk.bin", [](size_t requested, uint8_t* buffer, size_t& produced, bool& eof) -> DbStatus { - // fill `buffer` with up to `requested` bytes, set produced/eof produced = 0; eof = true; return {DbStatusCode::Ok, ""}; - }, - fileOpts + } ); -auto fileInfo = db.getFileInfo("notes/readme.txt"); -auto fileTree = db.listFiles("firmware", true); +auto info = db.files().getFileInfo("notes/readme.txt"); ``` -## Gotchas -- Each collection lives in RAM; add PSRAM when handling large documents. -- All payloads are JSON; converting to structs is optional but deserialisation still costs memory—size your `JsonDocument` objects carefully. -- `onSyncStatus()` immediately invokes the callback once on the caller task with the latest snapshot, then invokes future updates from the task producing them (`init()` caller or sync task). Keep callbacks short. -- Unique constraints and validators run inside write operations. Long-running validators will increase latency for the calling task. -- `writeFileStream()` and `readFileStream()` hold the filesystem lock while processing the stream; use reasonable chunk sizes and avoid blocking stream sources/sinks. -- `writeFileStreamAsync()` runs producer callbacks on a background task; callbacks must be short and thread-safe. -- `getFileUploadState(uploadId)` retains terminal states for a bounded number of recent uploads; older upload IDs eventually return `NotFound`. -- Uploaded files are not surfaced as collections or snapshots; use `getFileInfo()` / `listFiles()` to inspect persisted file storage under `/_files`. -- `dropCollection()` only schedules on-disk removal; the collection directory and document files are deleted on the next autosync pass or `syncNow()`. -- `/_files` is an internal reserved directory used for file storage and cannot be used as a collection name. -- `getSnapshot()` and `restoreFromSnapshot()` currently cover document collections only; file storage under `/_files` is not included. -- `usePSRAMBuffers` affects ESPJsonDB-owned byte buffers, decoded `DocView` `JsonDocument` pools on ArduinoJson v7, and long-lived internal DB containers (collection/schema/upload/diag maps and queues). Public return containers like `readFile()` still use the existing API types. - -## API Reference -- `DbStatus init(const char* baseDir = "/db", const ESPJsonDBConfig& cfg = {})` – mount LittleFS (`cfg.initFileSystem`), preload collections into RAM cache (except names listed in `cfg.delayedCollectionSyncArray`), and start the sync worker task. -- `void deinit()` – stop background tasks, cancel pending async uploads, and release runtime state. Safe before `init()` and safe to call repeatedly. -- `bool isInitialized() const` – reports whether this instance is initialized and ready for DB operations. -- `void onEvent(std::function)` / `void onError(std::function)` – receive sync, CRUD, and validation events. -- `void onSyncStatus(std::function)` – observe cold preload and `syncNow()` progress with stage/source/current collection counters. -- `onSync(std::function)` was removed; migrate to `onSyncStatus(...)`. -- Collection management: `collection(name)`, `dropCollection(name)`, `dropAll()`, `getAllCollectionName()`. - - `dropCollection(name)` removes in-memory state immediately and deletes the corresponding filesystem directory on the next autosync pass or explicit `syncNow()`. -- Document helpers: - - Create: `create`, `createMany` (JSON array) plus direct `Collection::create*` variants. - - Read: `findById`, `findOne`, `findMany` (predicate or JSON filter) returning `DocView` so you can read/write lazily. - - Update/delete: `updateOne`, `updateById`, `updateMany`, `removeById`, `removeMany` (predicate or JSON filter). -- Schemas: `registerSchema(name, Schema)`, `unRegisterSchema(name)`; `Schema` exposes fields with type/default/unique flags plus optional custom `validate` callables. -- References: store `{ "collection": "authors", "_id": "..." }` inside a document and call `DocView::populate(fieldName)` to expand the reference into an embedded object. -- Sync + diagnostics: `syncNow()`, `getDiag()` (JSON summary), `getSnapshot()` / `restoreFromSnapshot()` for backups. - - `getDiag()` does not touch the filesystem; it reports cached counters overlaid with currently loaded collection sizes. -- File storage: - - `writeFileStream(path, in, bytesToWrite, opts)` / `readFileStream(path, out, chunkSize)` for chunked stream transfer. - - `writeFileStream(path, pullCb, opts)` for synchronous callback-driven chunk production. - - `writeFileFromPath(path, sourceFsPath, opts)` to copy a source file path into DB-managed file storage. - - `writeFileStreamAsync(path, pullCb, opts, doneCb)` for non-blocking producer-driven uploads. - - `cancelFileUpload(uploadId)`, `getFileUploadState(uploadId)` for async job control (terminal states are retained for a bounded recent window). - - `writeFile(path, data, size)` / `readFile(path)` for direct byte buffers. - - `writeTextFile(path, text)` / `readTextFile(path)` for UTF-8 or plain text payloads. - - `getFileInfo(path)` returns a JSON object with `path`, `name`, `exists`, `isDirectory`, and `size`. - - `listFiles(prefix, recursive)` returns a JSON document with `prefix`, `recursive`, and an `entries` array of file/directory metadata objects. - - `fileExists(path)`, `fileSize(path)`, `removeFile(path)` for file lifecycle utilities. - - File paths are relative to `//_files` and path traversal segments are rejected. - - `ESPJsonDBFileOptions`: `overwrite` and `chunkSize` controls for stream writes. - - `DbFileUploadPullCb`: callback receives `(requested, buffer, produced, eof)` and fills bytes into `buffer`. - -`ESPJsonDBConfig` knobs: -- `intervalMs`, `stackSize`, `priority`, `coreId` – background autosync cadence & FreeRTOS tuning. -- `autosync`, `coldSync`, `cacheEnabled` – sync behavior. `cacheEnabled=false` is rejected so writes stay on the sync task; init preloads collections unless they are listed in `delayedCollectionSyncArray`. -- `delayedCollectionSyncArray` – collection names to skip during `init()` preload. Delayed collections load on first periodic autosync tick; if `autosync=false`, first `syncNow()` triggers one-time delayed preload. Accessing `collection(name)` earlier loads that delayed collection immediately. -- `fs`, `initFileSystem`, `formatOnFail`, `partitionLabel`, `maxOpenFiles` – file system integration; pass your own `fs::FS` if you mount LittleFS elsewhere. -- `usePSRAMBuffers` – prefer PSRAM for internal msgpack + file stream byte buffers, decoded `DocView` `JsonDocument` pools (ArduinoJson v7), and long-lived DB runtime container nodes, with safe fallback to default heap. Task stacks are always created from internal RAM. - -Stack sizes are expressed in bytes. - -## Restrictions -- Designed for ESP32 + LittleFS. Other platforms/FSes are untested. -- Large documents are only practical on boards with PSRAM when the cache is enabled. -- Requires ArduinoJson 6+, StreamUtils, and a FreeRTOS-capable environment (Arduino-ESP32 or ESP-IDF with C++17). - -## Tests -An integration harness (`test/`) runs CRUD, bulk, schema, reference, and diagnostic scenarios via the `DbTester` class. Build it as a PlatformIO test or ESP-IDF component (include `test/dbTest.cpp` in your project) and run it on hardware to validate changes. Contributions that expand automated coverage are welcome. +## Core API +- `DbStatus init(const char* baseDir = "/db", const ESPJsonDBConfig& cfg = {})` +- `DbStatus configureCollection(const std::string& name, const CollectionConfig& cfg)` +- `DbResult collection(name)` +- `DbStatus registerSchema(name, schema)` +- `DbStatus unregisterSchema(name)` +- `JsonDocument getDiagnostics()` +- `JsonDocument getSnapshot(SnapshotMode mode = SnapshotMode::OnDiskOnly)` +- `DbStatus restoreFromSnapshot(const JsonDocument& snapshot)` +- `FileStore& files()` + +## Snapshot Format +Snapshots are document-only and exclude `/_files`. + +```json +{ + "collections": { + "users": [ + { + "_id": "0123456789abcdef01234567", + "_meta": { + "createdAtMs": 1743100000000, + "updatedAtMs": 1743100005000, + "revision": 2, + "flags": 0 + }, + "username": "esp-jsondb" + } + ] + } +} +``` -## Formatting Baseline +## Notes +- `SnapshotMode::InMemoryConsistent` triggers `syncNow()` before reading persisted state. +- `CollectionLoadPolicy::Lazy` loads a collection on first access; `Delayed` defers load to background sync or explicit access. +- `DocView::commit()` is the only write intent; metadata returned by `meta()` is durable record metadata. +- New `.jdb` writes use the current v2 envelope; decode also accepts the earlier unreleased duplicated-`flags` envelope variant. +- `/_files` remains reserved and is not a valid collection name. +- v2 is a breaking release and does not read legacy v1 `.mp` files directly. -This repository follows the firmware formatting baseline from `esptoolkit-template`: -- `.clang-format` is the source of truth for C/C++/INO layout. -- `.editorconfig` enforces tabs (`tab_width = 4`), LF endings, and final newline. -- Format all tracked firmware sources with `bash scripts/format_cpp.sh`. +## Tests +The hardware-oriented test harness under `test/` exercises CRUD, schema validation, delayed loading, snapshots, diagnostics, and file storage. Run it in the same environment used for the library examples. ## License MIT — see [LICENSE.md](LICENSE.md). ## ESPToolKit -- Check out other libraries: -- Hang out on Discord: -- Support the project: -- Visit the website: +- Website: +- GitHub: +- Ko-Fi: diff --git a/examples/AsyncFileUpload/AsyncFileUpload.ino b/examples/AsyncFileUpload/AsyncFileUpload.ino index 917f2d2..40ff3e1 100644 --- a/examples/AsyncFileUpload/AsyncFileUpload.ino +++ b/examples/AsyncFileUpload/AsyncFileUpload.ino @@ -72,7 +72,7 @@ void setup() { opts.chunkSize = 128; opts.overwrite = true; - auto start = db.writeFileStreamAsync("uploads/telemetry.bin", uploadPull, opts, onUploadDone); + auto start = db.files().writeFileStreamAsync("uploads/telemetry.bin", uploadPull, opts, onUploadDone); if (!start.status.ok()) { Serial.printf("writeFileStreamAsync failed: %s\n", start.status.message); return; @@ -91,7 +91,7 @@ void loop() { } if (!gUploadDone) { - auto state = db.getFileUploadState(gUploadId); + auto state = db.files().getUploadState(gUploadId); if (state.status.ok()) { Serial.printf("Upload state: %u\n", static_cast(state.value)); } @@ -109,7 +109,7 @@ void loop() { return; } - auto file = db.readFile("uploads/telemetry.bin"); + auto file = db.files().readFile("uploads/telemetry.bin"); if (!file.status.ok()) { Serial.printf("readFile failed: %s\n", file.status.message); return; diff --git a/examples/AsyncLargeFileUpload/AsyncLargeFileUpload.ino b/examples/AsyncLargeFileUpload/AsyncLargeFileUpload.ino index d2b9e1b..7ddb0d4 100644 --- a/examples/AsyncLargeFileUpload/AsyncLargeFileUpload.ino +++ b/examples/AsyncLargeFileUpload/AsyncLargeFileUpload.ino @@ -127,7 +127,7 @@ void setup() { opts.chunkSize = kChunkSize; opts.overwrite = true; - auto start = db.writeFileStreamAsync( + auto start = db.files().writeFileStreamAsync( "uploads/large_payload.bin", largeUploadPull, opts, @@ -153,7 +153,7 @@ void loop() { } if (!gUploadDone) { - auto state = db.getFileUploadState(gUploadId); + auto state = db.files().getUploadState(gUploadId); if (state.status.ok()) { Serial.printf( "state=%u progress=%u/%u\n", @@ -175,7 +175,7 @@ void loop() { return; } - auto sizeRes = db.fileSize("uploads/large_payload.bin"); + auto sizeRes = db.files().fileSize("uploads/large_payload.bin"); if (!sizeRes.status.ok() || sizeRes.value != kPayloadSize) { Serial.printf( "fileSize mismatch: %s size=%u\n", @@ -186,7 +186,7 @@ void loop() { } HashingSink sink; - auto readRes = db.readFileStream("uploads/large_payload.bin", sink, kReadChunkSize); + auto readRes = db.files().readFileStream("uploads/large_payload.bin", sink, kReadChunkSize); if (!readRes.status.ok()) { Serial.printf("readFileStream failed: %s\n", readRes.status.message); return; diff --git a/examples/FileStreaming/FileStreaming.ino b/examples/FileStreaming/FileStreaming.ino index 88a8870..27b0bdd 100644 --- a/examples/FileStreaming/FileStreaming.ino +++ b/examples/FileStreaming/FileStreaming.ino @@ -37,21 +37,21 @@ void setup() { } // 1) Plain text file - st = db.writeTextFile("docs/readme.txt", "Hello from ESPJsonDB file storage.\nLine #2."); + st = db.files().writeTextFile("docs/readme.txt", "Hello from ESPJsonDB file storage.\nLine #2."); if (!st.ok()) { Serial.printf("writeTextFile(txt) failed: %s\n", st.message); return; } // 2) JSON file (stored as text payload) - st = db.writeTextFile("configs/app.json", "{\"mode\":\"demo\",\"intervalMs\":1000}"); + st = db.files().writeTextFile("configs/app.json", "{\"mode\":\"demo\",\"intervalMs\":1000}"); if (!st.ok()) { Serial.printf("writeTextFile(json) failed: %s\n", st.message); return; } // 3) CSV file - st = db.writeTextFile("exports/metrics.csv", "ts,temp,humidity\n1,21.4,48\n2,21.7,47\n"); + st = db.files().writeTextFile("exports/metrics.csv", "ts,temp,humidity\n1,21.4,48\n2,21.7,47\n"); if (!st.ok()) { Serial.printf("writeTextFile(csv) failed: %s\n", st.message); return; @@ -59,7 +59,7 @@ void setup() { // 4) Binary file using direct byte write const uint8_t binPayload[] = {0x00, 0xA1, 0xFF, 0x10, 0x22, 0x33, 0x44, 0x55}; - st = db.writeFile("firmware/chunk.bin", binPayload, sizeof(binPayload)); + st = db.files().writeFile("firmware/chunk.bin", binPayload, sizeof(binPayload)); if (!st.ok()) { Serial.printf("writeFile(bin) failed: %s\n", st.message); return; @@ -67,7 +67,7 @@ void setup() { // 5) Custom extension file (any file type works) const uint8_t customPayload[] = {'M', 'O', 'D', 'L', 0x01, 0x02, 0x03, 0x04}; - st = db.writeFile("assets/model.dat", customPayload, sizeof(customPayload)); + st = db.files().writeFile("assets/model.dat", customPayload, sizeof(customPayload)); if (!st.ok()) { Serial.printf("writeFile(dat) failed: %s\n", st.message); return; @@ -83,7 +83,7 @@ void setup() { ESPJsonDBFileOptions opts; opts.overwrite = true; opts.chunkSize = 128; - st = db.writeFileFromPath("streams/raw_capture.raw", seedPath, opts); + st = db.files().writeFileFromPath("streams/raw_capture.raw", seedPath, opts); if (!st.ok()) { Serial.printf("writeFileFromPath failed: %s\n", st.message); return; @@ -100,7 +100,7 @@ void setup() { return; } - auto streamOut = db.readFileStream("streams/raw_capture.raw", sink, 96); + auto streamOut = db.files().readFileStream("streams/raw_capture.raw", sink, 96); sink.close(); if (!streamOut.status.ok()) { Serial.printf("readFileStream failed: %s\n", streamOut.status.message); @@ -108,11 +108,11 @@ void setup() { } // 8) Verify with helper reads - auto txt = db.readTextFile("docs/readme.txt"); - auto jsn = db.readTextFile("configs/app.json"); - auto csv = db.readTextFile("exports/metrics.csv"); - auto bin = db.readFile("firmware/chunk.bin"); - auto modelSize = db.fileSize("assets/model.dat"); + auto txt = db.files().readTextFile("docs/readme.txt"); + auto jsn = db.files().readTextFile("configs/app.json"); + auto csv = db.files().readTextFile("exports/metrics.csv"); + auto bin = db.files().readFile("firmware/chunk.bin"); + auto modelSize = db.files().fileSize("assets/model.dat"); if (!txt.status.ok() || !jsn.status.ok() || !csv.status.ok() || !bin.status.ok() || !modelSize.status.ok()) { @@ -128,7 +128,7 @@ void setup() { Serial.printf("model.dat bytes: %u\n", static_cast(modelSize.value)); Serial.printf("streamed raw bytes out: %u\n", static_cast(streamOut.value)); - auto exists = db.fileExists("streams/raw_capture.raw"); + auto exists = db.files().fileExists("streams/raw_capture.raw"); Serial.printf( "streams/raw_capture.raw exists: %s\n", (exists.status.ok() && exists.value) ? "yes" : "no" diff --git a/examples/LargeFileStreaming/LargeFileStreaming.ino b/examples/LargeFileStreaming/LargeFileStreaming.ino index 4317159..8697d79 100644 --- a/examples/LargeFileStreaming/LargeFileStreaming.ino +++ b/examples/LargeFileStreaming/LargeFileStreaming.ino @@ -108,13 +108,13 @@ void setup() { opts.chunkSize = kWriteChunkSize; const char *path = "firmware/large_payload.bin"; - st = db.writeFileStream(path, pullCb, opts); + st = db.files().writeFileStream(path, pullCb, opts); if (!st.ok()) { Serial.printf("writeFileStream failed: %s\n", st.message); return; } - auto sizeRes = db.fileSize(path); + auto sizeRes = db.files().fileSize(path); if (!sizeRes.status.ok() || sizeRes.value != kPayloadSize) { Serial.printf( "fileSize mismatch: status=%s size=%u\n", @@ -125,7 +125,7 @@ void setup() { } HashingSink sink; - auto readRes = db.readFileStream(path, sink, kReadChunkSize); + auto readRes = db.files().readFileStream(path, sink, kReadChunkSize); if (!readRes.status.ok()) { Serial.printf("readFileStream failed: %s\n", readRes.status.message); return; diff --git a/examples/SchemaValidation/SchemaValidation.ino b/examples/SchemaValidation/SchemaValidation.ino index c3c697f..f029cab 100644 --- a/examples/SchemaValidation/SchemaValidation.ino +++ b/examples/SchemaValidation/SchemaValidation.ino @@ -18,12 +18,14 @@ void setup() { Schema userSchema; userSchema.fields = { - {"email", FieldType::String, "a@b.c"}, - {"username", FieldType::String}, - {"role", FieldType::String, "user"}, - {"password", FieldType::String}, - {"age", FieldType::Int} + SchemaField{"email", FieldType::String, std::string("a@b.c")}, + SchemaField{"username", FieldType::String}, + SchemaField{"role", FieldType::String, std::string("user")}, + SchemaField{"password", FieldType::String}, + SchemaField{"age", FieldType::Int32} }; + userSchema.fields[1].required = true; + userSchema.fields[3].required = true; userSchema.validate = usersValidate; db.registerSchema("users", userSchema); diff --git a/library.json b/library.json index ba3c739..9793f79 100644 --- a/library.json +++ b/library.json @@ -1,7 +1,7 @@ { "name": "ESPJsonDB", - "version": "1.1.1", - "description": "Lightweight JSON document database for ESP32 (Arduino). MongoDB-like API with optional autosync to LittleFS.", + "version": "2.0.0", + "description": "Embedded MessagePack-backed document database for ESP32 with durable metadata, load policies, snapshots, and async file storage.", "keywords": [ "esp32", "arduino", diff --git a/library.properties b/library.properties index 00dd9ed..dba7f2c 100644 --- a/library.properties +++ b/library.properties @@ -1,9 +1,9 @@ name=ESPJsonDB -version=1.1.1 +version=2.0.0 author=zekageri maintainer=zekageri -sentence=Lightweight document database for ESP32 devices. -paragraph=Provides a MongoDB-like interface with optional automatic sync between memory and flash storage. +sentence=MessagePack-backed document database for ESP32 devices. +paragraph=Provides durable metadata, collection load policies, explicit snapshots, and async file storage for embedded workloads. category=Data Storage url=https://github.com/ESPToolKit/esp-jsondb repository=https://github.com/ESPToolKit/esp-jsondb.git diff --git a/src/esp_jsondb/collection/collection.cpp b/src/esp_jsondb/collection/collection.cpp index b2591f3..17f5c7e 100644 --- a/src/esp_jsondb/collection/collection.cpp +++ b/src/esp_jsondb/collection/collection.cpp @@ -1,137 +1,411 @@ #include "collection.h" #include "../db.h" +#include "../db_runtime.h" #include "../utils/fs_utils.h" #include "../utils/time_utils.h" +#include +#include + namespace { std::shared_ptr makeSharedDocumentRecord(bool usePSRAMBuffers) { return std::allocate_shared( JsonDbAllocator(usePSRAMBuffers), usePSRAMBuffers ); } + +using DocumentRecordPtr = std::shared_ptr; +using DocumentMapValue = std::pair; +using DocumentMapAllocator = JsonDbAllocator; +using DocumentMap = std::map; +using UniqueValueMap = std::map, + JsonDbAllocator>>; +using UniqueIndexMap = + std::map, + JsonDbAllocator>>; } // namespace +struct CollectionStore { + DocumentMap docs; + JsonDbVector deletedIds; + JsonDbVector knownIds; + DbRuntime *rt = nullptr; + std::string name; + Schema schema; + CollectionConfig config{}; + bool dirty = false; + FrMutex mu; + std::string baseDir; + bool usePSRAMBuffers = false; + fs::FS *fs = nullptr; + RecordStore recordStore; + UniqueIndexMap uniqueIndexes; + uint64_t accessClock = 0; + size_t activeDecodedViews = 0; + + CollectionStore( + DbRuntime &rtRef, + const std::string &collectionName, + const Schema &collectionSchema, + std::string baseDirValue, + const CollectionConfig &collectionConfig, + bool psram, + fs::FS &filesystem + ) + : docs(DocumentMap(DocIdLess{}, DocumentMapAllocator(psram))), + deletedIds(JsonDbAllocator(psram)), knownIds(JsonDbAllocator(psram)), + rt(&rtRef), name(collectionName), schema(collectionSchema), config(collectionConfig), + baseDir(std::move(baseDirValue)), usePSRAMBuffers(psram), fs(&filesystem), + recordStore(filesystem, psram), + uniqueIndexes( + std::less{}, + JsonDbAllocator>(psram) + ) { + } +}; + Collection::Collection( - ESPJsonDB &db, + DbRuntime &rt, const std::string &name, const Schema &schema, std::string baseDir, - bool cacheEnabled, + const CollectionConfig &config, bool usePSRAMBuffers, fs::FS &fs ) - : _db(&db), _name(name), _schema(schema), - _docs(DocIdLess{}, DocumentMapAllocator(usePSRAMBuffers)), - _deletedIds(JsonDbAllocator(usePSRAMBuffers)), - _baseDir(std::move(baseDir)), _cacheEnabled(true), _usePSRAMBuffers(usePSRAMBuffers), - _fs(&fs) { - (void)cacheEnabled; + : _store(std::make_unique( + rt, name, schema, std::move(baseDir), config, usePSRAMBuffers, fs + )) { +} + +Collection::~Collection() = default; + +#define _rt (_store->rt) +#define _name (_store->name) +#define _schema (_store->schema) +#define _config (_store->config) +#define _docs (_store->docs) +#define _dirty (_store->dirty) +#define _deletedIds (_store->deletedIds) +#define _mu (_store->mu) +#define _baseDir (_store->baseDir) +#define _usePSRAMBuffers (_store->usePSRAMBuffers) +#define _fs (_store->fs) +#define _recordStore (_store->recordStore) +#define _uniqueIndexes (_store->uniqueIndexes) + +const std::string &Collection::name() const { + return _name; +} + +const CollectionConfig &Collection::config() const { + return _config; } -void Collection::setCacheEnabled(bool enabled) { - (void)enabled; - _cacheEnabled = true; +bool Collection::isDirty() const { + return _dirty; +} + +void Collection::clearDirty() { + _dirty = false; +} + +size_t Collection::size() const { + return _docs.size(); +} + +void Collection::markAllRemoved() { + FrLock lk(_mu); + for (auto &kv : _docs) { + kv.second->meta.removed = true; + } +} + +void Collection::setConfig(const CollectionConfig &config) { + FrLock lk(_mu); + _config = config; + (void)ensureResidentCapacityLocked(0); +} + +void Collection::setSchema(const Schema &schema) { + FrLock lk(_mu); + _schema = schema; + (void)rebuildUniqueIndexesLocked(); } DbStatus Collection::recordStatus(const DbStatus &st) const { - return _db ? _db->recordStatus(st) : st; + return _rt ? _rt->recordStatus(st) : st; } void Collection::emitEvent(DBEventType ev) const { - if (_db) - _db->emitEvent(ev); + if (_rt) + _rt->emitEvent(ev); } void Collection::noteDeletedInDiag(size_t count) const { - if (count == 0 || !_db) + if (count == 0 || !_rt) return; - _db->noteDocumentDeleted(_name, static_cast(count)); + _rt->noteDocumentDeleted(_name, static_cast(count)); } -DbStatus Collection::checkUniqueFieldsInCache(JsonObjectConst obj, const DocId *selfId) { - // Scan schema for fields marked unique and ensure no other doc has same value - for (const auto &f : _schema.fields) { - if (!f.unique) - continue; - // Only enforce on scalar types - if (f.type == FieldType::Object || f.type == FieldType::Array) - continue; - JsonVariantConst v = obj[f.name]; - if (v.isNull()) - continue; - for (const auto &kv : _docs) { - if (selfId && kv.first == *selfId) +bool Collection::isResidentBudgetEnforced() const { + return _config.maxRecordsInMemory > 0 && + (_config.loadPolicy == CollectionLoadPolicy::Lazy || + _config.loadPolicy == CollectionLoadPolicy::Delayed); +} + +bool Collection::isDecodedBudgetEnforced() const { + return _config.maxDecodedViews > 0; +} + +void Collection::touchRecordLocked(const std::shared_ptr &rec) { + if (!rec) + return; + rec->lastAccessSeq = ++_store->accessClock; +} + +void Collection::rememberKnownIdLocked(const DocId &id) { + if (!containsKnownIdLocked(id)) { + _store->knownIds.push_back(id); + } +} + +void Collection::forgetKnownIdLocked(const DocId &id) { + _store->knownIds.erase( + std::remove(_store->knownIds.begin(), _store->knownIds.end(), id), + _store->knownIds.end() + ); +} + +bool Collection::containsKnownIdLocked(const DocId &id) const { + return std::find(_store->knownIds.begin(), _store->knownIds.end(), id) != _store->knownIds.end(); +} + +DbStatus Collection::ensureResidentCapacityLocked(size_t additional, const DocId *protectId) { + if (!isResidentBudgetEnforced()) + return {DbStatusCode::Ok, ""}; + + while ((_docs.size() + additional) > _config.maxRecordsInMemory) { + auto victimIt = _docs.end(); + uint64_t oldestSeq = UINT64_MAX; + for (auto it = _docs.begin(); it != _docs.end(); ++it) { + const auto &rec = it->second; + if (!rec || rec->meta.dirty || rec->meta.removed || rec->pinCount > 0) + continue; + if (protectId && it->first == *protectId) continue; - DocView other(kv.second, &_schema, nullptr, _db); - JsonVariantConst ov = other[f.name]; - if (!ov.isNull() && ov == v) { - return recordStatus({DbStatusCode::ValidationFailed, "unique constraint violated"}); + if (rec->lastAccessSeq < oldestSeq) { + oldestSeq = rec->lastAccessSeq; + victimIt = it; } } + if (victimIt == _docs.end()) { + return {DbStatusCode::Busy, "record memory budget exceeded"}; + } + _docs.erase(victimIt); } - return recordStatus({DbStatusCode::Ok, ""}); + return {DbStatusCode::Ok, ""}; } -DbStatus Collection::checkUniqueFieldsOnDisk(JsonObjectConst obj, const DocId *selfId) { - bool hasUnique = false; - for (const auto &field : _schema.fields) { - if (field.unique) { - hasUnique = true; - break; +DbResult> Collection::ensureRecordLoaded(const DocId &id) { + DbResult> res{}; + { + FrLock lk(_mu); + auto it = _docs.find(id); + if (it != _docs.end()) { + touchRecordLocked(it->second); + res.status = {DbStatusCode::Ok, ""}; + res.value = it->second; + return res; + } + if (!containsKnownIdLocked(id)) { + res.status = {DbStatusCode::NotFound, "document not found"}; + return res; } } - if (!hasUnique) { - return recordStatus({DbStatusCode::Ok, ""}); + + auto rr = readDocFromFile(_baseDir, id.c_str()); + if (!rr.status.ok()) { + res.status = rr.status; + return res; } - // Enumerate all documents on disk and compare declared unique fields - JsonDbVector ids{JsonDbAllocator(_usePSRAMBuffers)}; - std::string dir = joinPath(_baseDir, _name); + { - FrLock fs(g_fsMutex); - if (_fs->exists(dir.c_str())) { - File d = _fs->open(dir.c_str()); - if (!d || !d.isDirectory()) { - return recordStatus({DbStatusCode::IoError, "open dir failed"}); - } - for (File f = d.openNextFile(); f; f = d.openNextFile()) { - if (f.isDirectory()) - continue; - String name = f.name(); - std::string fname = name.c_str(); - if (fname.size() < 3 || fname.substr(fname.size() - 3) != ".mp") - continue; - DocId parsedId; - if (parsedId.assign(fname.substr(0, fname.size() - 3))) { - ids.push_back(parsedId); - } - } + FrLock lk(_mu); + auto existing = _docs.find(id); + if (existing != _docs.end()) { + touchRecordLocked(existing->second); + res.status = {DbStatusCode::Ok, ""}; + res.value = existing->second; + return res; + } + auto cap = ensureResidentCapacityLocked(1, &id); + if (!cap.ok()) { + res.status = recordStatus(cap); + return res; + } + touchRecordLocked(rr.value); + auto [it, inserted] = _docs.emplace(id, rr.value); + (void)inserted; + res.status = {DbStatusCode::Ok, ""}; + res.value = it->second; + } + return res; +} + +DbStatus Collection::pinRecord(const std::shared_ptr &rec) { + if (!rec) + return {DbStatusCode::Ok, ""}; + FrLock lk(_mu); + ++rec->pinCount; + touchRecordLocked(rec); + return {DbStatusCode::Ok, ""}; +} + +void Collection::unpinRecord(const std::shared_ptr &rec) { + if (!rec) + return; + FrLock lk(_mu); + if (rec->pinCount > 0) + --rec->pinCount; +} + +DbStatus Collection::acquireDecodedViewSlot() { + if (!isDecodedBudgetEnforced()) + return {DbStatusCode::Ok, ""}; + FrLock lk(_mu); + if (_store->activeDecodedViews >= _config.maxDecodedViews) { + return {DbStatusCode::Busy, "decoded view budget exceeded"}; + } + ++_store->activeDecodedViews; + return {DbStatusCode::Ok, ""}; +} + +void Collection::releaseDecodedViewSlot() { + if (!isDecodedBudgetEnforced()) + return; + FrLock lk(_mu); + if (_store->activeDecodedViews > 0) + --_store->activeDecodedViews; +} + +std::string Collection::collectionDir() const { + return joinPath(_baseDir, _name); +} + +std::string Collection::uniqueValueKey(const SchemaField &field, JsonVariantConst value) const { + if (value.isNull()) + return {}; + char buffer[48]; + switch (field.type) { + case FieldType::String: + return std::string("s:") + value.as(); + case FieldType::Int32: + return std::string("i32:") + std::to_string(value.as()); + case FieldType::Int64: + return std::string("i64:") + std::to_string(value.as()); + case FieldType::UInt32: + return std::string("u32:") + std::to_string(value.as()); + case FieldType::UInt64: + return std::string("u64:") + std::to_string(value.as()); + case FieldType::Float: + snprintf(buffer, sizeof(buffer), "f:%0.7g", static_cast(value.as())); + return buffer; + case FieldType::Double: + snprintf(buffer, sizeof(buffer), "d:%0.17g", value.as()); + return buffer; + case FieldType::Bool: + return value.as() ? "b:true" : "b:false"; + case FieldType::Object: + case FieldType::Array: + break; + } + return {}; +} + +DbStatus Collection::addUniqueValuesLocked(JsonObjectConst obj, const DocId &id) { + for (const auto &field : _schema.fields) { + if (!field.unique || field.type == FieldType::Object || field.type == FieldType::Array) + continue; + const auto key = uniqueValueKey(field, obj[field.name]); + if (key.empty()) + continue; + auto &fieldIndex = _uniqueIndexes[field.name ? field.name : ""]; + auto it = fieldIndex.find(key); + if (it != fieldIndex.end() && it->second != id) { + return {DbStatusCode::ValidationFailed, "unique constraint violated"}; } + fieldIndex[key] = id; } - for (const auto &docId : ids) { - if (selfId && docId == *selfId) + return {DbStatusCode::Ok, ""}; +} + +void Collection::removeUniqueValuesLocked(JsonObjectConst obj, const DocId &id) { + for (const auto &field : _schema.fields) { + if (!field.unique || field.type == FieldType::Object || field.type == FieldType::Array) continue; - auto rr = readDocFromFile(_baseDir, docId.c_str()); - if (!rr.status.ok()) { - return rr.status; + const auto fieldName = field.name ? std::string(field.name) : std::string{}; + auto fieldIt = _uniqueIndexes.find(fieldName); + if (fieldIt == _uniqueIndexes.end()) + continue; + const auto key = uniqueValueKey(field, obj[field.name]); + if (key.empty()) + continue; + auto valueIt = fieldIt->second.find(key); + if (valueIt != fieldIt->second.end() && valueIt->second == id) { + fieldIt->second.erase(valueIt); } - DocView view(rr.value, &_schema, nullptr, _db); - for (const auto &field : _schema.fields) { - if (!field.unique) - continue; - if (field.type == FieldType::Object || field.type == FieldType::Array) - continue; - JsonVariantConst newVal = obj[field.name]; - if (newVal.isNull()) - continue; - JsonVariantConst existingVal = view[field.name]; - if (!existingVal.isNull() && existingVal == newVal) { - return recordStatus({DbStatusCode::ValidationFailed, "unique constraint violated"}); - } + if (fieldIt->second.empty()) { + _uniqueIndexes.erase(fieldIt); + } + } +} + +DbStatus Collection::rebuildUniqueIndexesLocked() { + _uniqueIndexes.clear(); + for (const auto &kv : _docs) { + JsonDocument doc; + if (!kv.second || kv.second->msgpack.empty()) { + continue; + } + auto err = deserializeMsgPack(doc, kv.second->msgpack.data(), kv.second->msgpack.size()); + if (err) { + return {DbStatusCode::CorruptionDetected, "msgpack decode failed while rebuilding index"}; + } + auto st = addUniqueValuesLocked(doc.as(), kv.first); + if (!st.ok()) + return st; + } + return {DbStatusCode::Ok, ""}; +} + +DbStatus Collection::checkUniqueFieldsInCache(JsonObjectConst obj, const DocId *selfId) { + for (const auto &f : _schema.fields) { + if (!f.unique) + continue; + if (f.type == FieldType::Object || f.type == FieldType::Array) + continue; + const std::string fieldName = f.name ? std::string(f.name) : std::string{}; + auto fit = _uniqueIndexes.find(fieldName); + if (fit == _uniqueIndexes.end()) + continue; + const std::string key = uniqueValueKey(f, obj[f.name]); + if (key.empty()) + continue; + auto vit = fit->second.find(key); + if (vit != fit->second.end() && (!selfId || vit->second != *selfId)) { + return recordStatus({DbStatusCode::ValidationFailed, "unique constraint violated"}); } } return recordStatus({DbStatusCode::Ok, ""}); } +DbStatus Collection::checkUniqueFieldsOnDisk(JsonObjectConst obj, const DocId *selfId) { + return checkUniqueFieldsInCache(obj, selfId); +} + DbStatus Collection::checkUniqueFields(JsonObjectConst obj, const DocId *selfId) { return checkUniqueFieldsInCache(obj, selfId); } @@ -162,10 +436,12 @@ DbResult Collection::create(JsonObjectConst data) { return res; } rec = makeSharedDocumentRecord(_usePSRAMBuffers); - rec->meta.createdAt = nowUtcMs(); - rec->meta.updatedAt = rec->meta.createdAt; + rec->meta.createdAtMs = nowUtcMs(); + rec->meta.updatedAtMs = rec->meta.createdAtMs; rec->meta.id = ObjectId().toDocId(); + rec->meta.revision = 1; rec->meta.dirty = true; + touchRecordLocked(rec); // Serialize input data to MsgPack size_t sz = measureMsgPack(obj); @@ -178,7 +454,20 @@ DbResult Collection::create(JsonObjectConst data) { } id = rec->meta.id.c_str(); + auto cap = ensureResidentCapacityLocked(1, &rec->meta.id); + if (!cap.ok()) { + res.status = recordStatus(cap); + return res; + } _docs.emplace(rec->meta.id, rec); + rememberKnownIdLocked(rec->meta.id); + auto uniqueStatus = addUniqueValuesLocked(obj, rec->meta.id); + if (!uniqueStatus.ok()) { + _docs.erase(rec->meta.id); + res.status = uniqueStatus; + recordStatus(res.status); + return res; + } _dirty = true; res.status = {DbStatusCode::Ok, ""}; @@ -187,8 +476,8 @@ DbResult Collection::create(JsonObjectConst data) { emit = true; } if (emit) { - if (_db) - _db->noteDocumentCreated(_name); + if (_rt) + _rt->noteDocumentCreated(_name); emitEvent(DBEventType::DocumentCreated); } return res; @@ -241,30 +530,52 @@ DbResult Collection::findById(const std::string &id) { if (!lookupId.assign(id)) { DbStatus st{DbStatusCode::NotFound, "document not found"}; recordStatus(st); - return {st, DocView(nullptr, &_schema, &_mu, _db, nullptr, _usePSRAMBuffers)}; - } - { - FrLock lk(_mu); - auto it = _docs.find(lookupId); - if (it != _docs.end()) { - DbStatus st{DbStatusCode::Ok, ""}; - recordStatus(st); - return {st, makeView(it->second)}; - } + return {st, + DocView(nullptr, + &_schema, + nullptr, + _rt ? _rt->owner : nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + false, + _usePSRAMBuffers)}; } - DbStatus st{DbStatusCode::NotFound, "document not found"}; + auto loaded = ensureRecordLoaded(lookupId); + if (!loaded.status.ok()) { + recordStatus(loaded.status); + return {loaded.status, + DocView(nullptr, + &_schema, + nullptr, + _rt ? _rt->owner : nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + false, + _usePSRAMBuffers)}; + } + DbStatus st{DbStatusCode::Ok, ""}; recordStatus(st); - return {st, DocView(nullptr, &_schema, &_mu, _db, nullptr, _usePSRAMBuffers)}; + return {st, makeView(loaded.value)}; } DbResult> Collection::findMany(std::function pred) { DbResult> res{}; - FrLock lk(_mu); - for (auto &kv : _docs) { - DocView v(kv.second, &_schema, nullptr, _db); - if (!pred || pred(v)) { - res.value.emplace_back(makeView(kv.second)); + auto idsRes = collectMatchingIds(std::move(pred)); + if (!idsRes.status.ok()) { + res.status = idsRes.status; + return res; + } + for (const auto &id : idsRes.value) { + auto loaded = ensureRecordLoaded(id); + if (loaded.status.ok()) { + res.value.emplace_back(makeView(loaded.value)); } } res.status = {DbStatusCode::Ok, ""}; @@ -273,18 +584,43 @@ DbResult> Collection::findMany(std::function Collection::findOne(std::function pred) { - FrLock lk(_mu); - for (auto &kv : _docs) { - DocView v(kv.second, &_schema, nullptr, _db); - if (!pred || pred(v)) { + auto idsRes = collectMatchingIds(std::move(pred)); + if (!idsRes.status.ok()) { + return {idsRes.status, + DocView(nullptr, + &_schema, + nullptr, + _rt ? _rt->owner : nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + false, + _usePSRAMBuffers)}; + } + if (!idsRes.value.empty()) { + auto loaded = ensureRecordLoaded(idsRes.value.front()); + if (loaded.status.ok()) { DbStatus st{DbStatusCode::Ok, ""}; recordStatus(st); - return {st, makeView(kv.second)}; + return {st, makeView(loaded.value)}; } } DbStatus st{DbStatusCode::NotFound, "document not found"}; recordStatus(st); - return {st, DocView(nullptr, &_schema, &_mu, _db, nullptr, _usePSRAMBuffers)}; + return {st, + DocView(nullptr, + &_schema, + nullptr, + _rt ? _rt->owner : nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + false, + _usePSRAMBuffers)}; } DbResult Collection::findOne(const JsonDocument &filter) { @@ -300,53 +636,94 @@ DbResult Collection::findOne(const JsonDocument &filter) { return findOne(std::move(pred)); } +DbResult> Collection::collectMatchingIds( + std::function pred +) { + DbResult> res{}; + res.value = JsonDbVector(JsonDbAllocator(_usePSRAMBuffers)); + JsonDbVector ids{JsonDbAllocator(_usePSRAMBuffers)}; + { + FrLock lk(_mu); + ids = _store->knownIds; + } + for (const auto &id : ids) { + auto loaded = ensureRecordLoaded(id); + if (!loaded.status.ok()) { + if (loaded.status.code == DbStatusCode::Busy) { + res.status = loaded.status; + return res; + } + continue; + } + bool matched = false; + { + FrLock lk(_mu); + auto it = _docs.find(id); + if (it == _docs.end()) + continue; + touchRecordLocked(it->second); + DocView v(it->second, + &_schema, + nullptr, + _rt ? _rt->owner : nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + false, + _usePSRAMBuffers); + matched = !pred || pred(v); + } + if (matched) { + res.value.push_back(id); + } + } + res.status = {DbStatusCode::Ok, ""}; + return res; +} + DbStatus Collection::updateOne( std::function pred, std::function mutator, bool create ) { + auto matches = collectMatchingIds(std::move(pred)); + if (!matches.status.ok()) + return recordStatus(matches.status); + bool updated = false; bool created = false; DbStatus st{DbStatusCode::NotFound, "document not found"}; - FrLock lk(_mu); - // Search existing docs first - for (auto &kv : _docs) { - DocView v(kv.second, &_schema, nullptr, _db); // use outer lock - if (!pred || pred(v)) { - mutator(v); - if (_schema.hasValidate()) { - auto obj = v.asObject(); - auto ve = _schema.runPreSave(obj); - if (!ve.valid) { - v.discard(); - return recordStatus({DbStatusCode::ValidationFailed, ve.message}); - } - // Unique constraints - auto ust = checkUniqueFields(obj, &kv.second->meta.id); - if (!ust.ok()) { - v.discard(); - return recordStatus(ust); - } - } - st = v.commit(); - if (!st.ok()) - return recordStatus(st); - // Only flag collection and emit update if record actually changed - if (kv.second->meta.dirty) { - _dirty = true; - updated = true; - } - break; - } + if (!matches.value.empty()) { + st = updateByIdWithDecision( + matches.value.front().c_str(), + std::function([&mutator](DocView &view) { + mutator(view); + return true; + }), + updated + ); } - // If not found and create requested, create a new record and apply mutator if (!updated && create) { auto rec = makeSharedDocumentRecord(_usePSRAMBuffers); - rec->meta.createdAt = nowUtcMs(); - rec->meta.updatedAt = rec->meta.createdAt; + rec->meta.createdAtMs = nowUtcMs(); + rec->meta.updatedAtMs = rec->meta.createdAtMs; rec->meta.id = ObjectId().toDocId(); + rec->meta.revision = 1; rec->meta.dirty = true; + touchRecordLocked(rec); - DocView v(rec, &_schema, nullptr, _db); + DocView v(rec, + &_schema, + nullptr, + _rt ? _rt->owner : nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + false, + _usePSRAMBuffers); // Initialize as empty object then let mutator fill values v.asObject(); mutator(v); @@ -367,14 +744,22 @@ DbStatus Collection::updateOne( st = v.commit(); if (!st.ok()) return recordStatus(st); + FrLock lk(_mu); + auto cap = ensureResidentCapacityLocked(1, &rec->meta.id); + if (!cap.ok()) + return recordStatus(cap); _docs.emplace(rec->meta.id, std::move(rec)); + rememberKnownIdLocked(v.meta().id); + auto uniqueStatus = addUniqueValuesLocked(v.asObjectConst(), v.meta().id); + if (!uniqueStatus.ok()) + return recordStatus(uniqueStatus); _dirty = true; created = true; st = {DbStatusCode::Ok, ""}; } if (created) { - if (_db) - _db->noteDocumentCreated(_name); + if (_rt) + _rt->noteDocumentCreated(_name); emitEvent(DBEventType::DocumentCreated); } else if (updated) { emitEvent(DBEventType::DocumentUpdated); @@ -383,58 +768,54 @@ DbStatus Collection::updateOne( } DbStatus Collection::updateOne(const JsonDocument &filter, const JsonDocument &patch, bool create) { - bool updated = false; - bool created = false; - DbStatus st{DbStatusCode::NotFound, "document not found"}; - FrLock lk(_mu); - // Look for first matching doc - for (auto &kv : _docs) { - DocView v(kv.second, &_schema, nullptr, _db); - bool match = true; + auto matches = collectMatchingIds([&filter](const DocView &v) { for (auto kvf : filter.as()) { if (v[kvf.key().c_str()] != kvf.value()) { - match = false; - break; - } - } - if (match) { - for (auto kvp : patch.as()) { - v[kvp.key().c_str()].set(kvp.value()); - } - if (_schema.hasValidate()) { - auto obj = v.asObject(); - auto ve = _schema.runPreSave(obj); - if (!ve.valid) { - v.discard(); - return recordStatus({DbStatusCode::ValidationFailed, ve.message}); - } - // Unique constraints - auto ust = checkUniqueFields(obj, &kv.second->meta.id); - if (!ust.ok()) { - v.discard(); - return recordStatus(ust); - } - } - st = v.commit(); - if (!st.ok()) - return recordStatus(st); - if (kv.second->meta.dirty) { - _dirty = true; - updated = true; + return false; } - break; } + return true; + }); + if (!matches.status.ok()) + return recordStatus(matches.status); + + bool updated = false; + bool created = false; + DbStatus st{DbStatusCode::NotFound, "document not found"}; + if (!matches.value.empty()) { + st = updateByIdWithDecision( + matches.value.front().c_str(), + std::function([&patch](DocView &view) { + for (auto kvp : patch.as()) { + view[kvp.key().c_str()].set(kvp.value()); + } + return true; + }), + updated + ); } if (!updated && create) { // Create a new document merging filter and patch auto rec = makeSharedDocumentRecord(_usePSRAMBuffers); - rec->meta.createdAt = nowUtcMs(); - rec->meta.updatedAt = rec->meta.createdAt; + rec->meta.createdAtMs = nowUtcMs(); + rec->meta.updatedAtMs = rec->meta.createdAtMs; rec->meta.id = ObjectId().toDocId(); + rec->meta.revision = 1; rec->meta.dirty = true; + touchRecordLocked(rec); - DocView v(rec, &_schema, nullptr, _db); + DocView v(rec, + &_schema, + nullptr, + _rt ? _rt->owner : nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + false, + _usePSRAMBuffers); auto obj = v.asObject(); for (auto kvf : filter.as()) { obj[kvf.key().c_str()] = kvf.value(); @@ -458,14 +839,22 @@ DbStatus Collection::updateOne(const JsonDocument &filter, const JsonDocument &p st = v.commit(); if (!st.ok()) return recordStatus(st); + FrLock lk(_mu); + auto cap = ensureResidentCapacityLocked(1, &rec->meta.id); + if (!cap.ok()) + return recordStatus(cap); _docs.emplace(rec->meta.id, std::move(rec)); + rememberKnownIdLocked(v.meta().id); + auto uniqueStatus = addUniqueValuesLocked(v.asObjectConst(), v.meta().id); + if (!uniqueStatus.ok()) + return recordStatus(uniqueStatus); _dirty = true; created = true; st = {DbStatusCode::Ok, ""}; } if (created) { - if (_db) - _db->noteDocumentCreated(_name); + if (_rt) + _rt->noteDocumentCreated(_name); emitEvent(DBEventType::DocumentCreated); } else if (updated) { emitEvent(DBEventType::DocumentUpdated); @@ -475,42 +864,119 @@ DbStatus Collection::updateOne(const JsonDocument &filter, const JsonDocument &p DbStatus Collection::updateById(const std::string &id, std::function mutator) { bool updated = false; - DbStatus st{DbStatusCode::Ok, ""}; + auto st = updateByIdWithDecision( + id, + std::function([&mutator](DocView &view) { + mutator(view); + return true; + }), + updated + ); + if (updated) + emitEvent(DBEventType::DocumentUpdated); + return recordStatus(st); +} + +DbStatus Collection::updateByIdWithDecision( + const std::string &id, std::function mutator, bool &updated +) { + updated = false; DocId lookupId; if (!lookupId.assign(id)) { return recordStatus({DbStatusCode::NotFound, "document not found"}); } - FrLock lk(_mu); - auto it = _docs.find(lookupId); - if (it == _docs.end()) { - return recordStatus({DbStatusCode::NotFound, "document not found"}); + + auto loaded = ensureRecordLoaded(lookupId); + if (!loaded.status.ok()) + return recordStatus(loaded.status); + + std::shared_ptr liveRec; + uint32_t startRevision = 0; + JsonDocument beforeDoc; + { + FrLock lk(_mu); + auto it = _docs.find(lookupId); + if (it == _docs.end()) + return recordStatus({DbStatusCode::NotFound, "document not found"}); + liveRec = it->second; + startRevision = liveRec->meta.revision; + touchRecordLocked(liveRec); + if (!liveRec->msgpack.empty()) { + auto err = deserializeMsgPack(beforeDoc, liveRec->msgpack.data(), liveRec->msgpack.size()); + if (err) { + return recordStatus({DbStatusCode::CorruptionDetected, "msgpack decode failed"}); + } + } else { + beforeDoc.to(); + } + } + + auto candidate = makeSharedDocumentRecord(_usePSRAMBuffers); + candidate->meta = liveRec->meta; + candidate->msgpack = liveRec->msgpack; + DocView working( + candidate, + &_schema, + nullptr, + _rt ? _rt->owner : nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + false, + _usePSRAMBuffers + ); + bool shouldCommit = mutator ? mutator(working) : true; + if (!shouldCommit) { + working.discard(); + return recordStatus({DbStatusCode::Ok, ""}); } - DocView v(it->second, &_schema, nullptr, _db); // using outer lock - mutator(v); if (_schema.hasValidate()) { - auto obj = v.asObject(); + auto obj = working.asObject(); auto ve = _schema.runPreSave(obj); if (!ve.valid) { - v.discard(); + working.discard(); return recordStatus({DbStatusCode::ValidationFailed, ve.message}); } - // Unique constraints - auto ust = checkUniqueFields(obj, &it->second->meta.id); - if (!ust.ok()) { - v.discard(); - return recordStatus(ust); - } } - st = v.commit(); + auto st = working.commit(); if (!st.ok()) return recordStatus(st); - if (it->second->meta.dirty) { - _dirty = true; - updated = true; + + { + FrLock lk(_mu); + auto it = _docs.find(lookupId); + if (it == _docs.end()) + return recordStatus({DbStatusCode::Conflict, "document changed during update"}); + if (it->second->meta.revision != startRevision) { + return recordStatus({DbStatusCode::Conflict, "document changed during update"}); + } + auto uniqueStatus = + checkUniqueFields(candidate->msgpack.empty() + ? JsonObjectConst() + : working.asObjectConst(), + &lookupId); + if (!uniqueStatus.ok()) { + return recordStatus(uniqueStatus); + } + if (candidate->meta.revision != startRevision) { + removeUniqueValuesLocked(beforeDoc.as(), lookupId); + auto addStatus = addUniqueValuesLocked(working.asObjectConst(), lookupId); + if (!addStatus.ok()) { + addUniqueValuesLocked(beforeDoc.as(), lookupId); + return recordStatus(addStatus); + } + it->second->msgpack = candidate->msgpack; + it->second->meta.updatedAtMs = candidate->meta.updatedAtMs; + it->second->meta.revision = candidate->meta.revision; + it->second->meta.dirty = true; + touchRecordLocked(it->second); + _dirty = true; + updated = true; + } } - if (updated) - emitEvent(DBEventType::DocumentUpdated); - return recordStatus(st); + return recordStatus({DbStatusCode::Ok, ""}); } DbStatus Collection::removeById(const std::string &id) { @@ -520,112 +986,56 @@ DbStatus Collection::removeById(const std::string &id) { if (!lookupId.assign(id)) { return recordStatus({DbStatusCode::NotFound, "document not found"}); } - FrLock lk(_mu); - auto it = _docs.find(lookupId); - if (it == _docs.end()) - return recordStatus({DbStatusCode::NotFound, "document not found"}); - // Mark record as logically removed so outstanding views fail on commit - it->second->meta.removed = true; - _deletedIds.push_back(it->first); // ensure file removal on sync - _docs.erase(it); - _dirty = true; - removed = true; + auto loaded = ensureRecordLoaded(lookupId); + if (!loaded.status.ok()) + return recordStatus(loaded.status); + { + FrLock lk(_mu); + auto it = _docs.find(lookupId); + if (it == _docs.end()) + return recordStatus({DbStatusCode::NotFound, "document not found"}); + DocView view(it->second, + &_schema, + nullptr, + _rt ? _rt->owner : nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + nullptr, + false, + _usePSRAMBuffers); + removeUniqueValuesLocked(view.asObjectConst(), it->first); + it->second->meta.removed = true; + _deletedIds.push_back(it->first); + forgetKnownIdLocked(it->first); + _docs.erase(it); + _dirty = true; + removed = true; + } if (removed) { - if (_db) - _db->noteDocumentDeleted(_name); + if (_rt) + _rt->noteDocumentDeleted(_name); emitEvent(DBEventType::DocumentDeleted); } return recordStatus(st); } DbStatus Collection::writeDocToFile(const std::string &baseDir, const DocumentRecord &r) { - FrLock fs(g_fsMutex); - std::string dir = joinPath(baseDir, _name); - if (!fsEnsureDir(*_fs, dir)) { - return recordStatus({DbStatusCode::IoError, "mkdir failed"}); - } - std::string finalPath = joinPath(dir, std::string(r.meta.id.c_str()) + ".mp"); - std::string tmpPath = finalPath + ".tmp"; - // Write to temp then rename for atomicity - File f = _fs->open(tmpPath.c_str(), FILE_WRITE); - if (!f) - return recordStatus({DbStatusCode::IoError, "open for write failed"}); - // Buffer writes to coalesce small chunks if any - WriteBufferingStream bufferedFile(f, 256); - size_t w = bufferedFile.write(r.msgpack.data(), r.msgpack.size()); - bufferedFile.flush(); - f.close(); - if (w != r.msgpack.size()) { - _fs->remove(tmpPath.c_str()); - return recordStatus({DbStatusCode::IoError, "write failed"}); - } - if (!_fs->rename(tmpPath.c_str(), finalPath.c_str())) { - _fs->remove(tmpPath.c_str()); - return recordStatus({DbStatusCode::IoError, "rename failed"}); - } - return recordStatus({DbStatusCode::Ok, ""}); + (void)baseDir; + return recordStatus(_recordStore.write(collectionDir(), r)); } DbResult> Collection::readDocFromFile(const std::string &baseDir, const std::string &id) { - DbResult> res{}; - std::string path = joinPath(joinPath(baseDir, _name), id + ".mp"); - FrLock fs(g_fsMutex); - File f = _fs->open(path.c_str(), FILE_READ); - if (!f) { - res.status = {DbStatusCode::NotFound, "file not found"}; - recordStatus(res.status); - return res; - } - auto rec = makeSharedDocumentRecord(_usePSRAMBuffers); - if (!rec->meta.id.assign(id)) { - res.status = {DbStatusCode::Corrupted, "invalid document id"}; - recordStatus(res.status); - return res; - } - rec->meta.createdAt = nowUtcMs(); - rec->meta.updatedAt = rec->meta.createdAt; - rec->meta.dirty = false; - - size_t sz = f.size(); - rec->msgpack.resize(sz); - size_t r = f.read(rec->msgpack.data(), sz); - f.close(); - if (r != sz) { - res.status = {DbStatusCode::IoError, "read failed"}; - recordStatus(res.status); - return res; - } - res.status = {DbStatusCode::Ok, ""}; + (void)baseDir; + auto res = _recordStore.read(collectionDir(), id); recordStatus(res.status); - res.value = std::move(rec); return res; } JsonDbVector Collection::listDocumentIdsFromFs() const { - JsonDbVector ids{JsonDbAllocator(_usePSRAMBuffers)}; - std::string dir = joinPath(_baseDir, _name); - { - FrLock fs(g_fsMutex); - if (!_fs->exists(dir.c_str())) - return ids; - File d = _fs->open(dir.c_str()); - if (!d || !d.isDirectory()) - return ids; - for (File f = d.openNextFile(); f; f = d.openNextFile()) { - if (f.isDirectory()) - continue; - String name = f.name(); - std::string fname = name.c_str(); - if (fname.size() < 3 || fname.substr(fname.size() - 3) != ".mp") - continue; - DocId parsedId; - if (parsedId.assign(fname.substr(0, fname.size() - 3))) { - ids.push_back(parsedId); - } - } - } - return ids; + return _recordStore.listIds(collectionDir()); } size_t Collection::countDocumentsFromFs() const { @@ -647,7 +1057,33 @@ DbStatus Collection::persistImmediate(const std::shared_ptr &rec } DocView Collection::makeView(std::shared_ptr rec) { - return DocView(std::move(rec), &_schema, &_mu, _db, nullptr, _usePSRAMBuffers); + { + FrLock lk(_mu); + if (rec) { + ++rec->pinCount; + touchRecordLocked(rec); + } + } + auto releasePin = [this, weakRec = std::weak_ptr(rec)]() { + if (auto locked = weakRec.lock()) { + unpinRecord(locked); + } + }; + auto acquireDecode = [this]() { return acquireDecodedViewSlot(); }; + auto releaseDecode = [this]() { releaseDecodedViewSlot(); }; + return DocView( + std::move(rec), + &_schema, + nullptr, + _rt ? _rt->owner : nullptr, + nullptr, + acquireDecode, + releaseDecode, + nullptr, + releasePin, + true, + _usePSRAMBuffers + ); } DbStatus Collection::updateOneNoCache( @@ -690,9 +1126,10 @@ DbStatus Collection::updateOneNoCache( } if (create) { auto rec = makeSharedDocumentRecord(_usePSRAMBuffers); - rec->meta.createdAt = nowUtcMs(); - rec->meta.updatedAt = rec->meta.createdAt; + rec->meta.createdAtMs = nowUtcMs(); + rec->meta.updatedAtMs = rec->meta.createdAtMs; rec->meta.id = ObjectId().toDocId(); + rec->meta.revision = 1; rec->meta.dirty = true; auto view = makeView(rec); view.asObject(); @@ -764,9 +1201,10 @@ DbStatus Collection::updateOneJsonNoCache( } if (create) { auto rec = makeSharedDocumentRecord(_usePSRAMBuffers); - rec->meta.createdAt = nowUtcMs(); - rec->meta.updatedAt = rec->meta.createdAt; + rec->meta.createdAtMs = nowUtcMs(); + rec->meta.updatedAtMs = rec->meta.createdAtMs; rec->meta.id = ObjectId().toDocId(); + rec->meta.revision = 1; rec->meta.dirty = true; auto view = makeView(rec); auto obj = view.asObject(); @@ -826,55 +1264,49 @@ DbStatus Collection::updateByIdNoCache( } DbStatus Collection::removeByIdNoCache(const std::string &id, bool &removed) { - std::string path = joinPath(joinPath(_baseDir, _name), id + ".mp"); - { - FrLock fs(g_fsMutex); - if (!_fs->exists(path.c_str())) { - return recordStatus({DbStatusCode::NotFound, "document not found"}); - } - if (!_fs->remove(path.c_str())) { - return recordStatus({DbStatusCode::IoError, "remove failed"}); - } + DocId docId; + if (!docId.assign(id)) { + return recordStatus({DbStatusCode::NotFound, "document not found"}); } + auto st = _recordStore.remove(collectionDir(), docId); + if (!st.ok()) + return recordStatus(st); removed = true; return recordStatus({DbStatusCode::Ok, ""}); } DbStatus Collection::loadFromFs(const std::string &baseDir) { - // First, under FS mutex, collect the list of document IDs to load. JsonDbVector ids{JsonDbAllocator(_usePSRAMBuffers)}; - std::string dir = joinPath(baseDir, _name); + (void)baseDir; + ids = listDocumentIdsFromFs(); + { - FrLock fs(g_fsMutex); - if (!_fs->exists(dir.c_str())) { - // Nothing to load yet; create directory lazily on write - return recordStatus({DbStatusCode::Ok, ""}); - } - File d = _fs->open(dir.c_str()); - if (!d || !d.isDirectory()) { - return recordStatus({DbStatusCode::IoError, "open dir failed"}); - } - for (File f = d.openNextFile(); f; f = d.openNextFile()) { - if (f.isDirectory()) - continue; - String name = f.name(); - f.close(); - std::string n = name.c_str(); - // Expect .mp - if (n.size() >= 3 && n.substr(n.size() - 3) == ".mp") { - DocId parsedId; - if (parsedId.assign(n.substr(0, n.size() - 3))) { - ids.push_back(parsedId); - } - } - } + FrLock lk(_mu); + _docs.clear(); + _store->knownIds = ids; + _uniqueIndexes.clear(); } - // Now, outside FS mutex, read each document file (readDocFromFile acquires FS mutex per file) for (const auto &id : ids) { - auto rr = readDocFromFile(baseDir, id.c_str()); - if (rr.status.ok()) { - _docs.emplace(rr.value->meta.id, std::move(rr.value)); + auto rr = readDocFromFile(_baseDir, id.c_str()); + if (!rr.status.ok()) { + continue; + } + JsonDocument doc; + auto err = deserializeMsgPack(doc, rr.value->msgpack.data(), rr.value->msgpack.size()); + if (err) { + return recordStatus({DbStatusCode::CorruptionDetected, "msgpack decode failed"}); + } + { + FrLock lk(_mu); + auto uniqueStatus = addUniqueValuesLocked(doc.as(), rr.value->meta.id); + if (!uniqueStatus.ok()) + return recordStatus(uniqueStatus); + if (_config.loadPolicy == CollectionLoadPolicy::Eager && + (!isResidentBudgetEnforced() || _docs.size() < _config.maxRecordsInMemory)) { + touchRecordLocked(rr.value); + _docs.emplace(rr.value->meta.id, rr.value); + } } } return recordStatus({DbStatusCode::Ok, ""}); @@ -885,7 +1317,7 @@ DbStatus Collection::flushDirtyToFs(const std::string &baseDir, bool &didWork) { // Snapshot work under lock JsonDbVector toDelete{JsonDbAllocator(_usePSRAMBuffers)}; struct PendingWrite { - DocId id; + DocumentMeta meta; JsonDbVector bytes; explicit PendingWrite(bool usePSRAMBuffers) @@ -900,7 +1332,7 @@ DbStatus Collection::flushDirtyToFs(const std::string &baseDir, bool &didWork) { auto &rec = kv.second; if (rec->meta.dirty) { PendingWrite pending(_usePSRAMBuffers); - pending.id = rec->meta.id; + pending.meta = rec->meta; pending.bytes = rec->msgpack; toWrite.push_back(std::move(pending)); rec->meta.dirty = false; @@ -912,24 +1344,17 @@ DbStatus Collection::flushDirtyToFs(const std::string &baseDir, bool &didWork) { // Process deletions (FS serialized by global mutex) if (!toDelete.empty()) { didWork = true; - std::string dir = joinPath(baseDir, _name); for (const auto &id : toDelete) { - std::string path = joinPath(dir, std::string(id.c_str()) + ".mp"); - { - FrLock fs(g_fsMutex); - if (_fs->exists(path.c_str())) { - if (!_fs->remove(path.c_str())) { - return recordStatus({DbStatusCode::IoError, "document delete failed"}); - } - } - } + auto st = _recordStore.remove(collectionDir(), id); + if (!st.ok() && st.code != DbStatusCode::NotFound) + return recordStatus({DbStatusCode::IoError, "document delete failed"}); } } // Flush writes for (auto &pw : toWrite) { DocumentRecord tmp(_usePSRAMBuffers); - tmp.meta.id = pw.id; + tmp.meta = pw.meta; tmp.msgpack = std::move(pw.bytes); auto st = writeDocToFile(baseDir, tmp); if (!st.ok()) diff --git a/src/esp_jsondb/collection/collection.h b/src/esp_jsondb/collection/collection.h index f522374..ce8998d 100644 --- a/src/esp_jsondb/collection/collection.h +++ b/src/esp_jsondb/collection/collection.h @@ -19,27 +19,27 @@ #include "../utils/jsondb_allocator.h" #include "../utils/objectId.h" #include "../utils/schema.h" +#include "../storage/record_store.h" -class ESPJsonDB; +struct DbRuntime; +struct CollectionStore; class Collection { public: Collection( - ESPJsonDB &db, + DbRuntime &rt, const std::string &name, const Schema &schema, std::string baseDir, - bool cacheEnabled, + const CollectionConfig &config, bool usePSRAMBuffers, fs::FS &fs ); - const std::string &name() const { - return _name; - } - bool cacheEnabled() const { - return _cacheEnabled; - } - void setCacheEnabled(bool enabled); + ~Collection(); + const std::string &name() const; + const CollectionConfig &config() const; + void setConfig(const CollectionConfig &config); + void setSchema(const Schema &schema); // Create from JsonObjectConst (validated) DbResult create(JsonObjectConst data); // returns new _id @@ -105,12 +105,8 @@ class Collection { DbResult updateMany(const JsonDocument &patch, const JsonDocument &filter); // Dirty tracking - bool isDirty() const { - return _dirty; - } - void clearDirty() { - _dirty = false; - } + bool isDirty() const; + void clearDirty(); // Persistence hooks used by ESPJsonDB DbStatus loadFromFs(const std::string &baseDir); @@ -119,37 +115,13 @@ class Collection { DbStatus flushDirtyToFs(const std::string &baseDir, bool &didWork); // Optional: stats - size_t size() const { - return _docs.size(); - } + size_t size() const; // Mark all records as removed (used when dropping a collection) - void markAllRemoved() { - FrLock lk(_mu); - for (auto &kv : _docs) { - kv.second->meta.removed = true; - } - } + void markAllRemoved(); private: - using DocumentRecordPtr = std::shared_ptr; - using DocumentMapValue = std::pair; - using DocumentMapAllocator = JsonDbAllocator; - using DocumentMap = - std::map; - - ESPJsonDB *_db = nullptr; - std::string _name; - Schema _schema; - // Use shared_ptr to keep records alive while views exist - DocumentMap _docs; - bool _dirty = false; - JsonDbVector _deletedIds; // files to remove on next flush - FrMutex _mu; // guards _docs, _deletedIds - std::string _baseDir; - bool _cacheEnabled = true; - bool _usePSRAMBuffers = false; - fs::FS *_fs = nullptr; // active filesystem (owned by caller) + std::unique_ptr _store; DbStatus writeDocToFile(const std::string &baseDir, const DocumentRecord &r); DbResult> @@ -161,6 +133,29 @@ class Collection { DbStatus persistImmediate(const std::shared_ptr &rec); size_t countDocumentsFromFs() const; DocView makeView(std::shared_ptr rec); + DbResult> collectMatchingIds(std::function pred); + std::string collectionDir() const; + std::string uniqueValueKey(const SchemaField &field, JsonVariantConst value) const; + DbStatus addUniqueValuesLocked(JsonObjectConst obj, const DocId &id); + void removeUniqueValuesLocked(JsonObjectConst obj, const DocId &id); + DbStatus rebuildUniqueIndexesLocked(); + bool isResidentBudgetEnforced() const; + bool isDecodedBudgetEnforced() const; + void touchRecordLocked(const std::shared_ptr &rec); + void rememberKnownIdLocked(const DocId &id); + void forgetKnownIdLocked(const DocId &id); + bool containsKnownIdLocked(const DocId &id) const; + DbStatus ensureResidentCapacityLocked(size_t additional, const DocId *protectId = nullptr); + DbResult> ensureRecordLoaded(const DocId &id); + DbStatus pinRecord(const std::shared_ptr &rec); + void unpinRecord(const std::shared_ptr &rec); + DbStatus acquireDecodedViewSlot(); + void releaseDecodedViewSlot(); + DbStatus updateByIdWithDecision( + const std::string &id, + std::function mutator, + bool &updated + ); DbStatus updateOneNoCache( std::function pred, std::function mutator, @@ -186,29 +181,16 @@ class Collection { template DbResult Collection::removeMany(Pred &&p) { DbResult res{}; - JsonDbVector toErase{JsonDbAllocator(_usePSRAMBuffers)}; - { - FrLock lk(_mu); - toErase.reserve(_docs.size()); - for (auto &kv : _docs) { - DocView v(kv.second, &_schema, nullptr, _db); - if (p(v)) { - toErase.push_back(kv.first); - } - } - for (auto &id : toErase) { - auto it = _docs.find(id); - if (it != _docs.end()) { - it->second->meta.removed = true; - _deletedIds.push_back(id); - _docs.erase(it); - } - } - if (!toErase.empty()) - _dirty = true; + auto matches = collectMatchingIds(std::function(std::forward(p))); + if (!matches.status.ok()) { + res.status = matches.status; + return res; + } + for (const auto &id : matches.value) { + auto st = removeById(id.c_str()); + if (st.ok()) + ++res.value; } - res.value = toErase.size(); - noteDeletedInDiag(res.value); res.status = {DbStatusCode::Ok, ""}; recordStatus(res.status); return res; @@ -217,74 +199,64 @@ template DbResult Collection::removeMany(Pred &&p) { template DbResult Collection::updateMany(Pred &&p, Mut &&m) { DbResult res{}; - size_t count = 0; - FrLock lk(_mu); - for (auto &kv : _docs) { - DocView v(kv.second, &_schema, nullptr, _db); - if (p(v)) { - m(v); - if (_schema.hasValidate()) { - auto obj = v.asObject(); - auto ve = _schema.runPreSave(obj); - if (!ve.valid) { - v.discard(); - continue; - } - // Unique constraints - auto ust = checkUniqueFields(obj, &kv.second->meta.id); - if (!ust.ok()) { - v.discard(); - continue; - } - } - auto st = v.commit(); - if (st.ok()) { - ++count; - } - } + bool sawConflict = false; + auto matches = collectMatchingIds(std::function(std::forward(p))); + if (!matches.status.ok()) { + res.status = matches.status; + return res; } - if (count) - _dirty = true; - res.status = {DbStatusCode::Ok, ""}; + for (const auto &id : matches.value) { + bool updated = false; + auto st = updateByIdWithDecision( + id.c_str(), + std::function([&m](DocView &view) { + m(view); + return true; + }), + updated + ); + if (st.code == DbStatusCode::Conflict) + sawConflict = true; + if (st.ok() && updated) + ++res.value; + } + res.status = sawConflict ? DbStatus{DbStatusCode::Conflict, "concurrent modification"} : + DbStatus{DbStatusCode::Ok, ""}; recordStatus(res.status); - res.value = count; return res; } template DbResult Collection::updateMany(Mut &&m) { DbResult res{}; - size_t count = 0; - FrLock lk(_mu); - for (auto &kv : _docs) { - DocView v(kv.second, &_schema, nullptr, _db); - if (m(v)) { - if (_schema.hasValidate()) { - auto obj = v.asObject(); - auto ve = _schema.runPreSave(obj); - if (!ve.valid) { - v.discard(); - continue; - } - // Unique constraints - auto ust = checkUniqueFields(obj, &kv.second->meta.id); - if (!ust.ok()) { - v.discard(); - continue; - } - } - auto st = v.commit(); - if (st.ok()) { - ++count; - } - } else { - v.discard(); - } + bool sawConflict = false; + auto matches = collectMatchingIds({}); + if (!matches.status.ok()) { + res.status = matches.status; + return res; } - if (count) - _dirty = true; - res.status = {DbStatusCode::Ok, ""}; + for (const auto &id : matches.value) { + bool updated = false; + auto st = updateByIdWithDecision( + id.c_str(), + std::function([&m](DocView &view) { + using Ret = std::invoke_result_t; + if constexpr (std::is_same_v) { + return m(view); + } else { + m(view); + return true; + } + }), + updated + ); + if (st.code == DbStatusCode::Conflict) + sawConflict = true; + if (st.ok() && updated) + ++res.value; + } + res.status = sawConflict ? DbStatus{DbStatusCode::Conflict, "concurrent modification"} : + DbStatus{DbStatusCode::Ok, ""}; recordStatus(res.status); - res.value = count; return res; } diff --git a/src/esp_jsondb/db.cpp b/src/esp_jsondb/db.cpp index 170031c..5627b11 100644 --- a/src/esp_jsondb/db.cpp +++ b/src/esp_jsondb/db.cpp @@ -1,6 +1,9 @@ #include "db.h" +#include "db_runtime.h" +#include "files/file_store_impl.h" #include "utils/fs_utils.h" #include "utils/jsondb_allocator.h" +#include "utils/time_utils.h" #include #include #include @@ -14,31 +17,227 @@ using DirEntryVector = JsonDbVector; FrMutex g_fsMutex; // definition of global FS mutex +DbRuntime::DbRuntime(bool usePSRAMBuffers) + : cols(std::less{}, DbRuntime::CollectionMap::allocator_type(usePSRAMBuffers)), + schemas(std::less{}, DbRuntime::SchemaMap::allocator_type(usePSRAMBuffers)), + collectionConfigs( + std::less{}, DbRuntime::CollectionConfigMap::allocator_type(usePSRAMBuffers) + ), + colsToDelete(JsonDbAllocator(usePSRAMBuffers)), + eventCbs(JsonDbAllocator>(usePSRAMBuffers)), + errorCbs(JsonDbAllocator>(usePSRAMBuffers)), + syncStatusCbs(JsonDbAllocator>(usePSRAMBuffers)), + pendingDelayedCollections( + std::less{}, DbRuntime::StringBoolMap::allocator_type(usePSRAMBuffers) + ), + diagCache{ + DbRuntime::StringUint32Map( + std::less{}, + DbRuntime::StringUint32Map::allocator_type(usePSRAMBuffers) + ), + 0, + 0} { +} + +DbRuntime::~DbRuntime() = default; + +uint32_t DbRuntime::stackBytesToWords(uint32_t stackBytes) { + const uint32_t wordSize = static_cast(sizeof(StackType_t)); + return (stackBytes + wordSize - 1U) / wordSize; +} + +DbStatus DbRuntime::ensureReady() const { + if (!initialized.load(std::memory_order_acquire) || !fs || baseDir.empty()) { + return {DbStatusCode::NotInitialized, "database not initialized"}; + } + return {DbStatusCode::Ok, ""}; +} + +void DbRuntime::emitError(const DbStatus &st) { + DbRuntime::ErrorCallbackVector callbacks{ + JsonDbAllocator>(cfg.usePSRAMBuffers)}; + { + FrLock lk(mu); + callbacks.reserve(errorCbs.size()); + for (auto &cb : errorCbs) { + callbacks.push_back(cb); + } + } + for (const auto &cb : callbacks) { + if (cb) + cb(st); + } +} + +DbStatus DbRuntime::recordStatus(const DbStatus &st) { + lastError = st; + if (!st.ok()) { + emitError(st); + } + return st; +} + +void DbRuntime::emitEvent(DBEventType ev) { + DbRuntime::EventCallbackVector callbacks{ + JsonDbAllocator>(cfg.usePSRAMBuffers)}; + { + FrLock lk(mu); + callbacks.reserve(eventCbs.size()); + for (auto &cb : eventCbs) { + callbacks.push_back(cb); + } + } + for (const auto &cb : callbacks) { + if (cb) + cb(ev); + } +} + +void DbRuntime::noteDocumentCreated(const std::string &collectionName, uint32_t count) { + if (collectionName.empty() || count == 0) + return; + FrLock lk(mu); + if (!diagCachePrimed) + return; + uint32_t &docs = diagCache.docsPerCollection[collectionName]; + if (docs == 0) { + ++diagCache.collections; + } + docs += count; + diagCache.lastRefreshMs = millis(); +} + +void DbRuntime::noteDocumentDeleted(const std::string &collectionName, uint32_t count) { + if (collectionName.empty() || count == 0) + return; + FrLock lk(mu); + if (!diagCachePrimed) + return; + auto it = diagCache.docsPerCollection.find(collectionName); + if (it == diagCache.docsPerCollection.end()) + return; + if (it->second <= count) { + diagCache.docsPerCollection.erase(it); + if (diagCache.collections > 0) + --diagCache.collections; + } else { + it->second -= count; + } + diagCache.lastRefreshMs = millis(); +} + +std::string DbRuntime::fileRootDir() const { + return joinPath(baseDir, "_files"); +} + +bool DbRuntime::createTask( + TaskFunction_t entry, const char *name, void *arg, TaskHandle_t &outHandle +) { + const uint32_t stackDepthWords = stackBytesToWords(cfg.stackSize); + BaseType_t rc = + xTaskCreatePinnedToCore(entry, name, stackDepthWords, arg, cfg.priority, &outHandle, cfg.coreId); + return rc == pdPASS; +} + +void DbRuntime::stopTask( + TaskHandle_t &taskHandle, std::atomic &stopRequested, std::atomic &taskExited +) { + if (taskHandle == nullptr) + return; + stopRequested.store(true, std::memory_order_release); + const uint32_t startMs = millis(); + while (!taskExited.load(std::memory_order_acquire)) { + if ((millis() - startMs) >= kTaskStopTimeoutMs) { + break; + } + vTaskDelay(pdMS_TO_TICKS(1)); + } + if (!taskExited.load(std::memory_order_acquire)) { + vTaskDelete(taskHandle); + taskExited.store(true, std::memory_order_release); + } + taskHandle = nullptr; +} + +ESPJsonDB::ESPJsonDB() : _rt(std::make_unique()) { + _rt->owner = this; + _rt->fileStoreImpl = std::make_unique(*_rt); + _rt->fileStore = std::make_unique(_rt->fileStoreImpl.get()); +} + +#define _baseDir (_rt->baseDir) +#define _cfg (_rt->cfg) +#define _cols (_rt->cols) +#define _schemas (_rt->schemas) +#define _collectionConfigs (_rt->collectionConfigs) +#define _colsToDelete (_rt->colsToDelete) +#define _eventCbs (_rt->eventCbs) +#define _errorCbs (_rt->errorCbs) +#define _syncStatusCbs (_rt->syncStatusCbs) +#define _fs (_rt->fs) +#define _mu (_rt->mu) +#define _lastError (_rt->lastError) +#define _lastSyncStatus (_rt->lastSyncStatus) +#define _diagCache (_rt->diagCache) +#define _diagCachePrimed (_rt->diagCachePrimed) +#define _initialized (_rt->initialized) +#define _syncTask (_rt->syncTask) +#define _syncStopRequested (_rt->syncStopRequested) +#define _syncTaskExited (_rt->syncTaskExited) +#define _syncKickRequested (_rt->syncKickRequested) +#define _syncRequestSeq (_rt->syncRequestSeq) +#define _syncCompletedSeq (_rt->syncCompletedSeq) +#define _pendingDelayedCollections (_rt->pendingDelayedCollections) +#define _delayedPreloadPhaseCompleted (_rt->delayedPreloadPhaseCompleted) +#define _dropAllRequested (_rt->dropAllRequested) +#define _fileStore (_rt->fileStore) + +bool ESPJsonDB::isInitialized() const { + return _initialized.load(std::memory_order_acquire); +} + +DbStatus ESPJsonDB::lastError() const { + return _lastError; +} + +DbStatus ESPJsonDB::recordStatus(const DbStatus &st) { + return _rt->recordStatus(st); +} + +DbStatus ESPJsonDB::setLastError(const DbStatus &st) { + return _rt->recordStatus(st); +} + +FileStore &ESPJsonDB::files() { + return *_fileStore; +} + +const FileStore &ESPJsonDB::files() const { + return *_fileStore; +} + DbStatus ESPJsonDB::ensureFsReady() { _fs = _cfg.fs ? _cfg.fs : &LittleFS; if (!_fs) { - return setLastError({DbStatusCode::InvalidArgument, "filesystem handle is null"}); + return _rt->recordStatus({DbStatusCode::InvalidArgument, "filesystem handle is null"}); } if (_cfg.initFileSystem && _fs == &LittleFS) { if (!LittleFS .begin(_cfg.formatOnFail, "/littlefs", _cfg.maxOpenFiles, _cfg.partitionLabel)) { - return setLastError({DbStatusCode::IoError, "LittleFS.begin failed"}); + return _rt->recordStatus({DbStatusCode::IoError, "LittleFS.begin failed"}); } } if (!fsEnsureDir(*_fs, _baseDir)) { - return setLastError({DbStatusCode::IoError, "mkdir baseDir failed"}); + return _rt->recordStatus({DbStatusCode::IoError, "mkdir baseDir failed"}); } if (!fsEnsureDir(*_fs, fileRootDir())) { - return setLastError({DbStatusCode::IoError, "mkdir file root failed"}); + return _rt->recordStatus({DbStatusCode::IoError, "mkdir file root failed"}); } - return setLastError({DbStatusCode::Ok, ""}); + return _rt->recordStatus({DbStatusCode::Ok, ""}); } DbStatus ESPJsonDB::ensureReady() const { - if (!isInitialized() || !_fs || _baseDir.empty()) { - return {DbStatusCode::InvalidArgument, "database not initialized"}; - } - return {DbStatusCode::Ok, ""}; + return _rt->ensureReady(); } void ESPJsonDB::rebindAllocatorAwareStateLocked(bool preserveData) { @@ -46,35 +245,32 @@ void ESPJsonDB::rebindAllocatorAwareStateLocked(bool preserveData) { auto oldCols = std::move(_cols); auto oldSchemas = std::move(_schemas); + auto oldCollectionConfigs = std::move(_collectionConfigs); auto oldColsToDelete = std::move(_colsToDelete); auto oldEventCbs = std::move(_eventCbs); auto oldErrorCbs = std::move(_errorCbs); auto oldSyncStatusCbs = std::move(_syncStatusCbs); auto oldPendingDelayed = std::move(_pendingDelayedCollections); - auto oldUploadQueue = std::move(_uploadQueue); - auto oldUploadJobs = std::move(_uploadJobs); - auto oldTerminalUploadOrder = std::move(_terminalUploadOrder); auto oldDiagDocs = std::move(_diagCache.docsPerCollection); const DBSyncStatus oldLastSyncStatus = _lastSyncStatus; - CollectionMap newCols{ - std::less{}, CollectionMap::allocator_type(usePSRAMBuffers)}; - SchemaMap newSchemas{std::less{}, SchemaMap::allocator_type(usePSRAMBuffers)}; - StringVector newColsToDelete{JsonDbAllocator(usePSRAMBuffers)}; - EventCallbackVector newEventCbs{ + DbRuntime::CollectionMap newCols{ + std::less{}, DbRuntime::CollectionMap::allocator_type(usePSRAMBuffers)}; + DbRuntime::SchemaMap newSchemas{ + std::less{}, DbRuntime::SchemaMap::allocator_type(usePSRAMBuffers)}; + DbRuntime::CollectionConfigMap newCollectionConfigs{ + std::less{}, DbRuntime::CollectionConfigMap::allocator_type(usePSRAMBuffers)}; + DbRuntime::StringVector newColsToDelete{JsonDbAllocator(usePSRAMBuffers)}; + DbRuntime::EventCallbackVector newEventCbs{ JsonDbAllocator>(usePSRAMBuffers)}; - ErrorCallbackVector newErrorCbs{ + DbRuntime::ErrorCallbackVector newErrorCbs{ JsonDbAllocator>(usePSRAMBuffers)}; - SyncStatusCallbackVector newSyncStatusCbs{ + DbRuntime::SyncStatusCallbackVector newSyncStatusCbs{ JsonDbAllocator>(usePSRAMBuffers)}; - StringBoolMap newPendingDelayed{ - std::less{}, StringBoolMap::allocator_type(usePSRAMBuffers)}; - UploadIdDeque newUploadQueue{JsonDbAllocator(usePSRAMBuffers)}; - decltype(_uploadJobs) newUploadJobs{ - std::less{}, decltype(_uploadJobs)::allocator_type(usePSRAMBuffers)}; - UploadIdDeque newTerminalUploadOrder{JsonDbAllocator(usePSRAMBuffers)}; - StringUint32Map newDiagDocs{ - std::less{}, StringUint32Map::allocator_type(usePSRAMBuffers)}; + DbRuntime::StringBoolMap newPendingDelayed{ + std::less{}, DbRuntime::StringBoolMap::allocator_type(usePSRAMBuffers)}; + DbRuntime::StringUint32Map newDiagDocs{ + std::less{}, DbRuntime::StringUint32Map::allocator_type(usePSRAMBuffers)}; if (preserveData) { newColsToDelete.reserve(oldColsToDelete.size()); @@ -88,6 +284,9 @@ void ESPJsonDB::rebindAllocatorAwareStateLocked(bool preserveData) { for (auto &kv : oldSchemas) { newSchemas.emplace(kv.first, kv.second); } + for (auto &kv : oldCollectionConfigs) { + newCollectionConfigs.emplace(kv.first, kv.second); + } for (auto &name : oldColsToDelete) { newColsToDelete.push_back(std::move(name)); } @@ -103,15 +302,6 @@ void ESPJsonDB::rebindAllocatorAwareStateLocked(bool preserveData) { for (auto &kv : oldPendingDelayed) { newPendingDelayed.emplace(kv.first, kv.second); } - for (auto id : oldUploadQueue) { - newUploadQueue.push_back(id); - } - for (auto &kv : oldUploadJobs) { - newUploadJobs.emplace(kv.first, std::move(kv.second)); - } - for (auto id : oldTerminalUploadOrder) { - newTerminalUploadOrder.push_back(id); - } for (auto &kv : oldDiagDocs) { newDiagDocs.emplace(kv.first, kv.second); } @@ -119,14 +309,12 @@ void ESPJsonDB::rebindAllocatorAwareStateLocked(bool preserveData) { _cols = std::move(newCols); _schemas = std::move(newSchemas); + _collectionConfigs = std::move(newCollectionConfigs); _colsToDelete = std::move(newColsToDelete); _eventCbs = std::move(newEventCbs); _errorCbs = std::move(newErrorCbs); _syncStatusCbs = std::move(newSyncStatusCbs); _pendingDelayedCollections = std::move(newPendingDelayed); - _uploadQueue = std::move(newUploadQueue); - _uploadJobs = std::move(newUploadJobs); - _terminalUploadOrder = std::move(newTerminalUploadOrder); _diagCache.docsPerCollection = std::move(newDiagDocs); if (preserveData) { _lastSyncStatus = oldLastSyncStatus; @@ -136,43 +324,14 @@ void ESPJsonDB::rebindAllocatorAwareStateLocked(bool preserveData) { } } -uint32_t ESPJsonDB::stackBytesToWords(uint32_t stackBytes) { - const uint32_t wordSize = static_cast(sizeof(StackType_t)); - return (stackBytes + wordSize - 1U) / wordSize; -} - bool ESPJsonDB::createTask(TaskFunction_t entry, const char *name, TaskHandle_t &outHandle) { - const uint32_t stackDepthWords = stackBytesToWords(_cfg.stackSize); - BaseType_t rc = xTaskCreatePinnedToCore( - entry, - name, - stackDepthWords, - this, - _cfg.priority, - &outHandle, - _cfg.coreId - ); - return rc == pdPASS; + return _rt->createTask(entry, name, this, outHandle); } void ESPJsonDB::stopTask( TaskHandle_t &taskHandle, std::atomic &stopRequested, std::atomic &taskExited ) { - if (taskHandle == nullptr) - return; - stopRequested.store(true, std::memory_order_release); - const uint32_t startMs = millis(); - while (!taskExited.load(std::memory_order_acquire)) { - if ((millis() - startMs) >= kTaskStopTimeoutMs) { - break; - } - vTaskDelay(pdMS_TO_TICKS(1)); - } - if (!taskExited.load(std::memory_order_acquire)) { - vTaskDelete(taskHandle); - taskExited.store(true, std::memory_order_release); - } - taskHandle = nullptr; + _rt->stopTask(taskHandle, stopRequested, taskExited); } bool ESPJsonDB::isReservedName(const std::string &name) const { @@ -180,15 +339,17 @@ bool ESPJsonDB::isReservedName(const std::string &name) const { } std::string ESPJsonDB::fileRootDir() const { - return joinPath(_baseDir, "_files"); + return _rt->fileRootDir(); } void ESPJsonDB::rebuildDelayedCollectionStateFromConfigLocked() { _pendingDelayedCollections.clear(); - for (const auto &name : _cfg.delayedCollectionSyncArray) { - if (name.empty() || isReservedName(name)) + for (const auto &kv : _collectionConfigs) { + if (kv.first.empty() || isReservedName(kv.first)) continue; - _pendingDelayedCollections[name] = true; + if (kv.second.loadPolicy == CollectionLoadPolicy::Delayed) { + _pendingDelayedCollections[kv.first] = true; + } } _delayedPreloadPhaseCompleted = _pendingDelayedCollections.empty(); } @@ -217,6 +378,7 @@ DbStatus ESPJsonDB::preloadCollectionFromFsByName( } Schema sc{}; + CollectionConfig collectionCfg{}; { FrLock lk(_mu); auto it = _cols.find(name); @@ -231,10 +393,17 @@ DbStatus ESPJsonDB::preloadCollectionFromFsByName( auto sit = _schemas.find(name); if (sit != _schemas.end()) sc = sit->second; + auto cit = _collectionConfigs.find(name); + if (cit != _collectionConfigs.end()) { + collectionCfg = cit->second; + } else { + collectionCfg.loadPolicy = _cfg.defaultLoadPolicy; + } } - auto col = - std::make_unique(*this, name, sc, _baseDir, true, _cfg.usePSRAMBuffers, *_fs); + auto col = std::make_unique( + *_rt, name, sc, _baseDir, collectionCfg, _cfg.usePSRAMBuffers, *_fs + ); auto st = col->loadFromFs(_baseDir); if (!st.ok()) return st; @@ -385,14 +554,15 @@ ESPJsonDB::~ESPJsonDB() { } void ESPJsonDB::deinit() { - if (!isInitialized() && _syncTask == nullptr && _fileUploadTask == nullptr) { + if (!isInitialized() && _syncTask == nullptr) { return; } _initialized.store(false, std::memory_order_release); { FrLock lk(_mu); - stopFileUploadTaskUnlocked(true); + if (_rt->fileStoreImpl) + _rt->fileStoreImpl->stopTask(true); stopSyncTaskUnlocked(); for (auto &kv : _cols) { @@ -405,10 +575,6 @@ void ESPJsonDB::deinit() { _eventCbs.clear(); _errorCbs.clear(); _syncStatusCbs.clear(); - _uploadQueue.clear(); - _uploadJobs.clear(); - _terminalUploadOrder.clear(); - _nextUploadId = 1; _pendingDelayedCollections.clear(); _delayedPreloadPhaseCompleted = true; _diagCache.docsPerCollection.clear(); @@ -424,8 +590,6 @@ void ESPJsonDB::deinit() { _syncKickRequested.store(false, std::memory_order_release); _syncRequestSeq.store(0, std::memory_order_release); _syncCompletedSeq.store(0, std::memory_order_release); - _fileUploadStopRequested.store(false, std::memory_order_release); - _fileUploadTaskExited.store(true, std::memory_order_release); _dropAllRequested = false; _baseDir.clear(); _cfg = ESPJsonDBConfig{}; @@ -438,11 +602,6 @@ DbStatus ESPJsonDB::init(const char *baseDir, const ESPJsonDBConfig &cfg) { if (isInitialized()) { deinit(); } - if (!cfg.cacheEnabled) { - return setLastError( - {DbStatusCode::InvalidArgument, "cacheEnabled=false is no longer supported"} - ); - } _initialized.store(false, std::memory_order_release); _baseDir = baseDir ? baseDir : std::string("/db"); // Normalize baseDir: ensure leading '/', drop trailing '/' @@ -456,12 +615,10 @@ DbStatus ESPJsonDB::init(const char *baseDir, const ESPJsonDBConfig &cfg) { _baseDir.pop_back(); } _cfg = cfg; - _cfg.cacheEnabled = true; _syncStopRequested.store(false, std::memory_order_release); _syncKickRequested.store(false, std::memory_order_release); _syncRequestSeq.store(0, std::memory_order_release); _syncCompletedSeq.store(0, std::memory_order_release); - _fileUploadStopRequested.store(false, std::memory_order_release); { FrLock lk(_mu); _lastSyncStatus = { @@ -473,14 +630,13 @@ DbStatus ESPJsonDB::init(const char *baseDir, const ESPJsonDBConfig &cfg) { { FrLock lk(_mu); - stopFileUploadTaskUnlocked(true); + if (_rt->fileStoreImpl) + _rt->fileStoreImpl->stopTask(true); rebindAllocatorAwareStateLocked(true); _cols.clear(); _colsToDelete.clear(); - _uploadQueue.clear(); - _uploadJobs.clear(); - _terminalUploadOrder.clear(); - _nextUploadId = 1; + _rt->fileStoreImpl = std::make_unique(*_rt); + _fileStore = std::make_unique(_rt->fileStoreImpl.get()); rebuildDelayedCollectionStateFromConfigLocked(); _dropAllRequested = false; _diagCache.docsPerCollection.clear(); @@ -507,21 +663,43 @@ DbStatus ESPJsonDB::init(const char *baseDir, const ESPJsonDBConfig &cfg) { return setLastError({DbStatusCode::Ok, ""}); } +DbStatus ESPJsonDB::configureCollection(const std::string &name, const CollectionConfig &cfg) { + if (name.empty() || isReservedName(name)) { + return setLastError({DbStatusCode::InvalidArgument, "reserved collection name"}); + } + FrLock lk(_mu); + _collectionConfigs[name] = cfg; + auto it = _cols.find(name); + if (it != _cols.end() && it->second) { + it->second->setConfig(cfg); + } + rebuildDelayedCollectionStateFromConfigLocked(); + return setLastError({DbStatusCode::Ok, ""}); +} + DbStatus ESPJsonDB::registerSchema(const std::string &name, const Schema &s) { if (isReservedName(name)) { return setLastError({DbStatusCode::InvalidArgument, "reserved collection name"}); } FrLock lk(_mu); _schemas[name] = s; + auto it = _cols.find(name); + if (it != _cols.end() && it->second) { + it->second->setSchema(s); + } return setLastError({DbStatusCode::Ok, ""}); } -DbStatus ESPJsonDB::unRegisterSchema(const std::string &name) { +DbStatus ESPJsonDB::unregisterSchema(const std::string &name) { if (isReservedName(name)) { return setLastError({DbStatusCode::InvalidArgument, "reserved collection name"}); } FrLock lk(_mu); _schemas.erase(name); + auto it = _cols.find(name); + if (it != _cols.end() && it->second) { + it->second->setSchema(Schema{}); + } return setLastError({DbStatusCode::Ok, ""}); } @@ -609,6 +787,7 @@ DbResult ESPJsonDB::collection(const std::string &name) { return res; } bool pendingDelayed = false; + CollectionConfig collectionCfg{}; { FrLock lk(_mu); auto it = _cols.find(name); @@ -618,9 +797,16 @@ DbResult ESPJsonDB::collection(const std::string &name) { return res; } pendingDelayed = _pendingDelayedCollections.find(name) != _pendingDelayedCollections.end(); + auto cit = _collectionConfigs.find(name); + if (cit != _collectionConfigs.end()) { + collectionCfg = cit->second; + } else { + collectionCfg.loadPolicy = _cfg.defaultLoadPolicy; + } } - if (pendingDelayed) { + if (pendingDelayed || (collectionCfg.loadPolicy == CollectionLoadPolicy::Lazy && + collectionDirExistsOnFs(name))) { const bool existedOnFs = collectionDirExistsOnFs(name); bool inserted = false; auto preloadStatus = preloadCollectionFromFsByName(name, true, &inserted); @@ -643,6 +829,7 @@ DbResult ESPJsonDB::collection(const std::string &name) { return res; } Schema sc{}; + collectionCfg = {}; { FrLock lk(_mu); auto existing = _cols.find(name); @@ -654,9 +841,16 @@ DbResult ESPJsonDB::collection(const std::string &name) { auto sit = _schemas.find(name); if (sit != _schemas.end()) sc = sit->second; + auto cit = _collectionConfigs.find(name); + if (cit != _collectionConfigs.end()) { + collectionCfg = cit->second; + } else { + collectionCfg.loadPolicy = _cfg.defaultLoadPolicy; + } } - auto col = - std::make_unique(*this, name, sc, _baseDir, true, _cfg.usePSRAMBuffers, *_fs); + auto col = std::make_unique( + *_rt, name, sc, _baseDir, collectionCfg, _cfg.usePSRAMBuffers, *_fs + ); Collection *ptr = nullptr; bool created = false; { @@ -727,7 +921,8 @@ DbResult ESPJsonDB::findById(const std::string &name, const std::string auto cr = collection(name); if (!cr.status.ok()) { // Return placeholder DocView; caller should check status before use - return {cr.status, DocView(nullptr, nullptr, nullptr, this)}; + return {cr.status, + DocView(nullptr, nullptr, nullptr, this, nullptr, nullptr, nullptr, nullptr, nullptr, false, _cfg.usePSRAMBuffers)}; } return cr.value->findById(id); } @@ -748,7 +943,8 @@ ESPJsonDB::findOne(const std::string &name, std::function auto cr = collection(name); if (!cr.status.ok()) { // Return placeholder DocView; caller should check status before use - return {cr.status, DocView(nullptr, nullptr, nullptr, this)}; + return {cr.status, + DocView(nullptr, nullptr, nullptr, this, nullptr, nullptr, nullptr, nullptr, nullptr, false, _cfg.usePSRAMBuffers)}; } return cr.value->findOne(std::move(pred)); } @@ -757,7 +953,8 @@ DbResult ESPJsonDB::findOne(const std::string &name, const JsonDocument auto cr = collection(name); if (!cr.status.ok()) { // Return placeholder DocView; caller should check status before use - return {cr.status, DocView(nullptr, nullptr, nullptr, this)}; + return {cr.status, + DocView(nullptr, nullptr, nullptr, this, nullptr, nullptr, nullptr, nullptr, nullptr, false, _cfg.usePSRAMBuffers)}; } return cr.value->findOne(filter); } @@ -872,7 +1069,7 @@ DbStatus ESPJsonDB::syncNow() { DbStatus ESPJsonDB::runSyncPass() { // Snapshot work under lock - StringVector colsToDrop{JsonDbAllocator(_cfg.usePSRAMBuffers)}; + DbRuntime::StringVector colsToDrop{JsonDbAllocator(_cfg.usePSRAMBuffers)}; JsonDbVector cols{JsonDbAllocator(_cfg.usePSRAMBuffers)}; bool dropAll = false; { @@ -1255,6 +1452,16 @@ DbStatus ESPJsonDB::preloadCollectionsFromFs(bool emitStatus, DBSyncSource statu continue; if (_pendingDelayedCollections.find(name) != _pendingDelayedCollections.end()) continue; + auto cit = _collectionConfigs.find(name); + const auto policy = + cit != _collectionConfigs.end() ? cit->second.loadPolicy : _cfg.defaultLoadPolicy; + if (policy == CollectionLoadPolicy::Delayed) { + _pendingDelayedCollections[name] = true; + _delayedPreloadPhaseCompleted = false; + continue; + } + if (policy != CollectionLoadPolicy::Eager) + continue; preloadNames.push_back(name); } } @@ -1323,17 +1530,17 @@ DbStatus ESPJsonDB::preloadCollectionsFromFs(bool emitStatus, DBSyncSource statu return setLastError({DbStatusCode::Ok, ""}); } -JsonDocument ESPJsonDB::getDiag() { +JsonDocument ESPJsonDB::getDiagnostics() { // Build diagnostics from cached FS snapshot, overlapped with live loaded collections // No filesystem access here. JsonDocument doc; const bool usePSRAMBuffers = _cfg.usePSRAMBuffers; // Snapshot state under lock - StringUint32Map cached{ - std::less{}, StringUint32Map::allocator_type(usePSRAMBuffers)}; - StringUint32Map live{ - std::less{}, StringUint32Map::allocator_type(usePSRAMBuffers)}; + DbRuntime::StringUint32Map cached{ + std::less{}, DbRuntime::StringUint32Map::allocator_type(usePSRAMBuffers)}; + DbRuntime::StringUint32Map live{ + std::less{}, DbRuntime::StringUint32Map::allocator_type(usePSRAMBuffers)}; uint32_t lastRefreshMs = 0; // Copy of configuration for reporting ESPJsonDBConfig cfgCopy{}; @@ -1354,7 +1561,8 @@ JsonDocument ESPJsonDB::getDiag() { // Per-collection document counts auto per = doc["documentsPerCollection"].to(); // Union of keys: prefer live counts for loaded collections - StringBoolMap seen{std::less{}, StringBoolMap::allocator_type(usePSRAMBuffers)}; + DbRuntime::StringBoolMap seen{ + std::less{}, DbRuntime::StringBoolMap::allocator_type(usePSRAMBuffers)}; for (auto &kv : live) { per[kv.first.c_str()] = kv.second; seen[kv.first] = true; @@ -1383,8 +1591,6 @@ JsonDocument ESPJsonDB::getDiag() { cfg["baseDir"] = baseDirCopy.c_str(); cfg["intervalMs"] = cfgCopy.intervalMs; cfg["autosync"] = cfgCopy.autosync; - cfg["coldSync"] = cfgCopy.coldSync; - cfg["cacheEnabled"] = cfgCopy.cacheEnabled; cfg["initFileSystem"] = cfgCopy.initFileSystem; cfg["formatOnFail"] = cfgCopy.formatOnFail; cfg["maxOpenFiles"] = static_cast(cfgCopy.maxOpenFiles); @@ -1393,9 +1599,14 @@ JsonDocument ESPJsonDB::getDiag() { cfg["priority"] = static_cast(cfgCopy.priority); cfg["coreId"] = static_cast(cfgCopy.coreId); cfg["usePSRAMBuffers"] = cfgCopy.usePSRAMBuffers; - auto delayedArr = cfg["delayedCollectionSyncArray"].to(); - for (const auto &name : cfgCopy.delayedCollectionSyncArray) { - delayedArr.add(name.c_str()); + cfg["defaultLoadPolicy"] = static_cast(cfgCopy.defaultLoadPolicy); + + auto policies = cfg["collectionLoadPolicies"].to(); + { + FrLock lk(_mu); + for (const auto &kv : _collectionConfigs) { + policies[kv.first.c_str()] = static_cast(kv.second.loadPolicy); + } } setLastError({DbStatusCode::Ok, ""}); @@ -1409,19 +1620,17 @@ DbStatus ESPJsonDB::dropAll() { } { FrLock lk(_mu); - stopFileUploadTaskUnlocked(true); + if (_rt->fileStoreImpl) + _rt->fileStoreImpl->stopTask(true); - // Clear in-memory state for (auto &kv : _cols) { if (kv.second) kv.second->markAllRemoved(); } _cols.clear(); _colsToDelete.clear(); - _uploadQueue.clear(); - _uploadJobs.clear(); - _terminalUploadOrder.clear(); - _nextUploadId = 1; + _rt->fileStoreImpl = std::make_unique(*_rt); + _fileStore = std::make_unique(_rt->fileStoreImpl.get()); _pendingDelayedCollections.clear(); _delayedPreloadPhaseCompleted = true; _diagCache.docsPerCollection.clear(); @@ -1434,7 +1643,7 @@ DbStatus ESPJsonDB::dropAll() { return syncNow(); } -std::vector ESPJsonDB::getAllCollectionName() { +std::vector ESPJsonDB::listCollectionNames() { auto ready = ensureReady(); if (!ready.ok()) { setLastError(ready); @@ -1442,8 +1651,8 @@ std::vector ESPJsonDB::getAllCollectionName() { } std::vector names; // Use a set to avoid duplicates - StringBoolMap seen{ - std::less{}, StringBoolMap::allocator_type(_cfg.usePSRAMBuffers)}; + DbRuntime::StringBoolMap seen{ + std::less{}, DbRuntime::StringBoolMap::allocator_type(_cfg.usePSRAMBuffers)}; { FrLock lk(_mu); for (auto &kv : _cols) { @@ -1468,37 +1677,24 @@ DbStatus ESPJsonDB::changeConfig(const ESPJsonDBConfig &cfg) { if (!ready.ok()) { return setLastError(ready); } - if (!cfg.cacheEnabled) { - return setLastError( - {DbStatusCode::InvalidArgument, "cacheEnabled=false is no longer supported"} - ); - } - bool doColdSync = cfg.coldSync; // Stop existing task if running and apply new config { FrLock lk(_mu); - stopFileUploadTaskUnlocked(true); - _uploadQueue.clear(); - _uploadJobs.clear(); - _terminalUploadOrder.clear(); - _nextUploadId = 1; + if (_rt->fileStoreImpl) + _rt->fileStoreImpl->stopTask(true); stopSyncTaskUnlocked(); _cfg = cfg; - _cfg.cacheEnabled = true; rebindAllocatorAwareStateLocked(true); - for (auto &kv : _cols) { - if (kv.second) - kv.second->setCacheEnabled(_cfg.cacheEnabled); - } + _rt->fileStoreImpl = std::make_unique(*_rt); + _fileStore = std::make_unique(_rt->fileStoreImpl.get()); + rebuildDelayedCollectionStateFromConfigLocked(); } auto fsStatus = ensureFsReady(); if (!fsStatus.ok()) return fsStatus; - if (doColdSync) { - auto preloadStatus = preloadCollectionsFromFs(false, DBSyncSource::Init); - if (!preloadStatus.ok()) { - return preloadStatus; - } + auto preloadStatus = preloadCollectionsFromFs(false, DBSyncSource::Init); + if (!preloadStatus.ok()) { + return preloadStatus; } { FrLock lk(_mu); @@ -1510,13 +1706,21 @@ DbStatus ESPJsonDB::changeConfig(const ESPJsonDBConfig &cfg) { return setLastError({DbStatusCode::Ok, ""}); } -JsonDocument ESPJsonDB::getSnapshot() { +JsonDocument ESPJsonDB::getSnapshot(SnapshotMode mode) { JsonDocument snap; + if (mode == SnapshotMode::InMemoryConsistent) { + auto syncStatus = syncNow(); + if (!syncStatus.ok()) { + setLastError(syncStatus); + return snap; + } + } if (!_fs) { setLastError({DbStatusCode::IoError, "filesystem not ready"}); return snap; } auto colsObj = snap["collections"].to(); + RecordStore store(*_fs, _cfg.usePSRAMBuffers); // Scan collections dirs DirEntryVector colDirs{JsonDbAllocator(_cfg.usePSRAMBuffers)}; @@ -1531,39 +1735,25 @@ JsonDocument ESPJsonDB::getSnapshot() { continue; // Iterate files in collection dir - DirEntryVector files{JsonDbAllocator(_cfg.usePSRAMBuffers)}; - listDirEntries(*_fs, full, files); + const auto ids = store.listIds(full); JsonArray arr = colsObj[colName.c_str()].to(); - for (auto &fe : files) { - if (fe.second) - continue; // skip subdirectories - const std::string &fpath = fe.first; - // expect .mp - auto dot = fpath.find_last_of('.'); - if (dot == std::string::npos || fpath.substr(dot) != ".mp") + for (const auto &id : ids) { + auto rec = store.read(full, id.c_str()); + if (!rec.status.ok() || !rec.value) continue; - auto slash = fpath.find_last_of('/'); - std::string fname = (slash == std::string::npos) ? fpath : fpath.substr(slash + 1); - std::string id = fname.substr(0, fname.size() - 3); - - // Read and decode msgpack - DeserializationError derr; JsonDocument tmp; - { - FrLock fs(g_fsMutex); - File f = _fs->open(fpath.c_str(), FILE_READ); - if (f) { - derr = deserializeMsgPack(tmp, f); - f.close(); - } else { - derr = DeserializationError::Code::InvalidInput; - } - } + auto derr = + deserializeMsgPack(tmp, rec.value->msgpack.data(), rec.value->msgpack.size()); if (derr) - continue; // skip unreadable + continue; JsonObject obj = arr.add(); obj.set(tmp.as()); - obj["_id"] = id.c_str(); + obj["_id"] = rec.value->meta.id.c_str(); + auto meta = obj["_meta"].to(); + meta["createdAtMs"] = rec.value->meta.createdAtMs; + meta["updatedAtMs"] = rec.value->meta.updatedAtMs; + meta["revision"] = rec.value->meta.revision; + meta["flags"] = rec.value->meta.flags; } } setLastError({DbStatusCode::Ok, ""}); @@ -1606,6 +1796,7 @@ DbStatus ESPJsonDB::restoreFromSnapshot(const JsonDocument &snapshot) { fsEnsureDir(*_fs, dir); } + RecordStore store(*_fs, _cfg.usePSRAMBuffers); for (JsonObjectConst obj : arr) { const char *id = obj["_id"].is() ? obj["_id"].as() : nullptr; @@ -1619,36 +1810,33 @@ DbStatus ESPJsonDB::restoreFromSnapshot(const JsonDocument &snapshot) { JsonDocument tmp; tmp.to().set(obj); tmp.remove("_id"); + tmp.remove("_meta"); - // Serialize to MsgPack buffer size_t sz = measureMsgPack(tmp); - JsonDbVector bytes{JsonDbAllocator(_cfg.usePSRAMBuffers)}; - bytes.resize(sz); - size_t written = serializeMsgPack(tmp, bytes.data(), bytes.size()); + DocumentRecord record(_cfg.usePSRAMBuffers); + record.meta.id = DocId(id); + record.msgpack.resize(sz); + size_t written = serializeMsgPack(tmp, record.msgpack.data(), record.msgpack.size()); if (written != sz) return setLastError({DbStatusCode::IoError, "serialize msgpack failed"}); - // Write file atomically - std::string finalPath = dir + "/" + std::string(id) + ".mp"; - std::string tmpPath = finalPath + ".tmp"; - { - FrLock fs(g_fsMutex); - File f = _fs->open(tmpPath.c_str(), FILE_WRITE); - if (!f) - return setLastError({DbStatusCode::IoError, "open for write failed"}); - WriteBufferingStream bufferedFile(f, 256); - size_t w = bufferedFile.write(bytes.data(), bytes.size()); - bufferedFile.flush(); - f.close(); - if (w != bytes.size()) { - _fs->remove(tmpPath.c_str()); - return setLastError({DbStatusCode::IoError, "write failed"}); - } - if (!_fs->rename(tmpPath.c_str(), finalPath.c_str())) { - _fs->remove(tmpPath.c_str()); - return setLastError({DbStatusCode::IoError, "rename failed"}); - } + JsonObjectConst meta = obj["_meta"].as(); + if (!meta.isNull()) { + record.meta.createdAtMs = meta["createdAtMs"] | nowUtcMs(); + record.meta.updatedAtMs = meta["updatedAtMs"] | record.meta.createdAtMs; + record.meta.revision = meta["revision"] | 1U; + record.meta.flags = meta["flags"] | static_cast(0); + } else { + record.meta.createdAtMs = nowUtcMs(); + record.meta.updatedAtMs = record.meta.createdAtMs; + record.meta.revision = 1; + record.meta.flags = 0; } + record.meta.dirty = false; + record.meta.removed = false; + auto stWrite = store.write(dir, record); + if (!stWrite.ok()) + return setLastError(stWrite); } } @@ -1662,8 +1850,8 @@ DbStatus ESPJsonDB::restoreFromSnapshot(const JsonDocument &snapshot) { void ESPJsonDB::refreshDiagFromFs() { if (!_fs) return; - StringUint32Map perCol{ - std::less{}, StringUint32Map::allocator_type(_cfg.usePSRAMBuffers)}; + DbRuntime::StringUint32Map perCol{ + std::less{}, DbRuntime::StringUint32Map::allocator_type(_cfg.usePSRAMBuffers)}; uint32_t colCount = 0; { FrLock fs(g_fsMutex); @@ -1688,7 +1876,7 @@ void ESPJsonDB::refreshDiagFromFs() { } f.close(); - // Count .mp files in collection dir + // Count persisted record files std::string dirPath = _baseDir; if (!dirPath.empty() && dirPath.back() != '/') dirPath += '/'; @@ -1707,7 +1895,7 @@ void ESPJsonDB::refreshDiagFromFs() { String fn = df.name(); df.close(); std::string n = fn.c_str(); - if (n.size() >= 3 && n.substr(n.size() - 3) == ".mp") + if (n.size() >= 4 && n.substr(n.size() - 4) == ".jdb") ++cnt; } colDir.close(); diff --git a/src/esp_jsondb/db.h b/src/esp_jsondb/db.h index 114ac6b..95bca5a 100644 --- a/src/esp_jsondb/db.h +++ b/src/esp_jsondb/db.h @@ -14,29 +14,32 @@ #include #include "collection/collection.h" +#include "files/file_store.h" #include "utils/dbTypes.h" #include "utils/fr_mutex.h" #include "utils/jsondb_allocator.h" #include "utils/schema.h" #include +struct DbRuntime; + class ESPJsonDB { public: + ESPJsonDB(); ~ESPJsonDB(); DbStatus init(const char *baseDir = "/db", const ESPJsonDBConfig &cfg = {}); void deinit(); - bool isInitialized() const { - return _initialized.load(std::memory_order_acquire); - } + bool isInitialized() const; + DbStatus configureCollection(const std::string &name, const CollectionConfig &cfg); DbStatus registerSchema(const std::string &name, const Schema &s); - DbStatus unRegisterSchema(const std::string &name); + DbStatus unregisterSchema(const std::string &name); DbStatus dropCollection(const std::string &name); // Drop all collections and documents (clears base directory) DbStatus dropAll(); // Returns collection names tracked in memory (preloaded + runtime-created) - std::vector getAllCollectionName(); + std::vector listCollectionNames(); // Change sync configuration; restarts autosync task if needed DbStatus changeConfig(const ESPJsonDBConfig &cfg); @@ -141,66 +144,20 @@ class ESPJsonDB { DbStatus syncNow(); // Retrieve last error or success status - DbStatus lastError() const { - return _lastError; - } + DbStatus lastError() const; // Allow other components to update diagnostics/error state - DbStatus recordStatus(const DbStatus &st) { - return setLastError(st); - } + DbStatus recordStatus(const DbStatus &st); // Diagnostics: number of collections, doc counts, and config - JsonDocument getDiag(); + JsonDocument getDiagnostics(); // Backup/restore - JsonDocument getSnapshot(); + JsonDocument getSnapshot(SnapshotMode mode = SnapshotMode::OnDiskOnly); DbStatus restoreFromSnapshot(const JsonDocument &snapshot); - // Generic file-bytes helpers under /_files. - DbStatus writeFileStream( - const std::string &relativePath, - Stream &in, - size_t bytesToWrite, - const ESPJsonDBFileOptions &opts = {} - ); - DbStatus writeFileStream( - const std::string &relativePath, - const DbFileUploadPullCb &pullCb, - const ESPJsonDBFileOptions &opts = {} - ); - DbStatus writeFileFromPath( - const std::string &relativePath, - const std::string &sourceFsPath, - const ESPJsonDBFileOptions &opts = {} - ); - DbStatus writeFile( - const std::string &relativePath, const uint8_t *data, size_t size, bool overwrite = true - ); - DbStatus - writeTextFile(const std::string &relativePath, const std::string &text, bool overwrite = true); - - DbResult - readFileStream(const std::string &relativePath, Stream &out, size_t chunkSize = 512); - DbResult> readFile(const std::string &relativePath); - DbResult readTextFile(const std::string &relativePath); - DbResult getFileInfo(const std::string &relativePath); - DbResult listFiles(const std::string &relativePrefix = "", bool recursive = true); - - DbStatus removeFile(const std::string &relativePath); - DbResult fileExists(const std::string &relativePath); - DbResult fileSize(const std::string &relativePath); - - // Non-blocking chunked file upload worker API. - // The pull callback runs on a background task and must fill up to `requested` bytes. - DbResult writeFileStreamAsync( - const std::string &relativePath, - const DbFileUploadPullCb &pullCb, - const ESPJsonDBFileOptions &opts = {}, - const DbFileUploadDoneCb &doneCb = {} - ); - DbStatus cancelFileUpload(uint32_t uploadId); - DbResult getFileUploadState(uint32_t uploadId); + FileStore &files(); + const FileStore &files() const; // Emit an event to registered listeners void emitEvent(DBEventType ev); @@ -213,41 +170,6 @@ class ESPJsonDB { void noteDocumentDeleted(const std::string &collectionName, uint32_t count = 1); private: - using CollectionMap = JsonDbMap>; - using SchemaMap = JsonDbMap; - using StringBoolMap = JsonDbMap; - using StringUint32Map = JsonDbMap; - using StringVector = JsonDbVector; - using UploadIdDeque = JsonDbDeque; - using EventCallbackVector = JsonDbVector>; - using ErrorCallbackVector = JsonDbVector>; - using SyncStatusCallbackVector = JsonDbVector>; - - std::string _baseDir; - ESPJsonDBConfig _cfg; - CollectionMap _cols; - SchemaMap _schemas; - StringVector _colsToDelete; - EventCallbackVector _eventCbs; - ErrorCallbackVector _errorCbs; - SyncStatusCallbackVector _syncStatusCbs; - fs::FS *_fs = &LittleFS; // active filesystem - FrMutex _mu; // guards _cols, _schemas, _colsToDelete - - // Tracks most recent status for diagnostics/debugging - DbStatus _lastError{DbStatusCode::Ok, ""}; - DBSyncStatus _lastSyncStatus{}; - - struct DiagCache { - StringUint32Map docsPerCollection; - uint32_t collections = 0; - uint32_t lastRefreshMs = 0; // millis when refreshed from FS - }; - - DiagCache _diagCache; // cached diagnostics; read without touching FS - bool _diagCachePrimed = false; // true once runtime counters are initialized - std::atomic _initialized{false}; - // sync task static void syncTaskThunk(void *arg); void syncTaskLoop(); @@ -255,13 +177,7 @@ class ESPJsonDB { void startSyncTaskUnlocked(); void stopSyncTaskUnlocked(); - // Update last error/status helper - DbStatus setLastError(const DbStatus &st) { - _lastError = st; - if (!st.ok()) - emitError(st); - return st; - } + DbStatus setLastError(const DbStatus &st); void emitSyncStatus(const DBSyncStatus &status); void emitSyncStatus( DBSyncStage stage, @@ -279,7 +195,6 @@ class ESPJsonDB { DbStatus removeCollectionDir(const std::string &name); bool isReservedName(const std::string &name) const; std::string fileRootDir() const; - DbStatus normalizeFilePath(const std::string &rawRelativePath, std::string &normalized) const; void rebuildDelayedCollectionStateFromConfigLocked(); DbStatus maybeRunDelayedPreload(bool triggeredByPeriodic, bool emitStatus, DBSyncSource statusSource); @@ -288,28 +203,6 @@ class ESPJsonDB { const std::string &name, bool markDelayedHandled, bool *insertedOut = nullptr ); bool collectionDirExistsOnFs(const std::string &name) const; - - // async file upload task - struct FileUploadJob { - uint32_t id = 0; - std::string relativePath; - std::string normalizedPath; - ESPJsonDBFileOptions opts{}; - DbFileUploadPullCb pullCb{}; - DbFileUploadDoneCb doneCb{}; - DbFileUploadState state = DbFileUploadState::Queued; - DbStatus finalStatus{DbStatusCode::Ok, ""}; - size_t bytesWritten = 0; - bool cancelRequested = false; - bool terminalTracked = false; - }; - static void fileUploadTaskThunk(void *arg); - void fileUploadTaskLoop(); - void startFileUploadTaskUnlocked(); - void stopFileUploadTaskUnlocked(bool cancelPending); - DbStatus runFileUploadJob(const std::shared_ptr &job, size_t &bytesWritten); - bool isUploadTerminal(DbFileUploadState state) const; - void trackTerminalUploadLocked(const std::shared_ptr &job); bool createTask(TaskFunction_t entry, const char *name, TaskHandle_t &outHandle); void stopTask( TaskHandle_t &taskHandle, std::atomic &stopRequested, std::atomic &taskExited @@ -320,24 +213,7 @@ class ESPJsonDB { void refreshDiagFromFs(); DbStatus preloadCollectionsFromFs(bool emitStatus, DBSyncSource statusSource); - // FreeRTOS task handles for autosync and async uploads - TaskHandle_t _syncTask = nullptr; - TaskHandle_t _fileUploadTask = nullptr; - std::atomic _syncStopRequested{false}; - std::atomic _syncTaskExited{true}; - std::atomic _syncKickRequested{false}; - std::atomic _syncRequestSeq{0}; - std::atomic _syncCompletedSeq{0}; - StringBoolMap _pendingDelayedCollections; - bool _delayedPreloadPhaseCompleted = true; - bool _dropAllRequested = false; - std::atomic _fileUploadStopRequested{false}; - std::atomic _fileUploadTaskExited{true}; - uint32_t _nextUploadId = 1; - UploadIdDeque _uploadQueue; - JsonDbMap> _uploadJobs; - UploadIdDeque _terminalUploadOrder; - static constexpr size_t kMaxRetainedTerminalUploads = 64; + std::unique_ptr _rt; }; template diff --git a/src/esp_jsondb/db_files.cpp b/src/esp_jsondb/db_files.cpp deleted file mode 100644 index 2b03510..0000000 --- a/src/esp_jsondb/db_files.cpp +++ /dev/null @@ -1,686 +0,0 @@ -#include "db.h" - -#include - -#include - -#include "utils/fs_utils.h" -#include "utils/jsondb_allocator.h" - -namespace { - -std::string parentDirOf(const std::string &path) { - auto pos = path.find_last_of('/'); - if (pos == std::string::npos || pos == 0) { - return "/"; - } - return path.substr(0, pos); -} - -std::string fileNameOf(const std::string &path) { - auto pos = path.find_last_of('/'); - if (pos == std::string::npos) - return path; - return path.substr(pos + 1); -} - -struct FileEntryInfo { - std::string path; - bool isDirectory = false; - size_t size = 0; -}; - -DbStatus statFileEntry( - fs::FS &filesystem, - const std::string &absolutePath, - const std::string &relativePath, - FileEntryInfo &out -) { - FrLock fs(g_fsMutex); - if (!filesystem.exists(absolutePath.c_str())) { - return {DbStatusCode::NotFound, "file not found"}; - } - File file = filesystem.open(absolutePath.c_str(), FILE_READ); - if (!file) { - return {DbStatusCode::IoError, "open file info failed"}; - } - out.path = relativePath; - out.isDirectory = file.isDirectory(); - out.size = out.isDirectory ? 0 : file.size(); - file.close(); - return {DbStatusCode::Ok, ""}; -} - -void appendFileInfoJson(JsonArray entries, const FileEntryInfo &info) { - JsonObject entry = entries.add(); - entry["path"] = info.path.c_str(); - entry["name"] = fileNameOf(info.path).c_str(); - entry["exists"] = true; - entry["isDirectory"] = info.isDirectory; - entry["size"] = info.size; -} - -DbStatus collectDirectoryEntries( - fs::FS &filesystem, - const std::string &absoluteDir, - const std::string &relativeDir, - bool recursive, - std::vector &entries -) { - std::vector> pendingDirs; - { - FrLock fs(g_fsMutex); - File dir = filesystem.open(absoluteDir.c_str(), FILE_READ); - if (!dir || !dir.isDirectory()) { - if (dir) - dir.close(); - return {DbStatusCode::NotFound, "file not found"}; - } - for (File child = dir.openNextFile(); child; child = dir.openNextFile()) { - const bool isDirectory = child.isDirectory(); - String rawName = child.name(); - child.close(); - std::string segment = rawName.c_str(); - auto slash = segment.find_last_of('/'); - if (slash != std::string::npos) - segment = segment.substr(slash + 1); - if (segment.empty()) - continue; - - FileEntryInfo info; - info.path = relativeDir.empty() ? segment : joinPath(relativeDir, segment); - info.isDirectory = isDirectory; - info.size = 0; - if (!isDirectory) { - const std::string childPath = joinPath(absoluteDir, segment); - File childFile = filesystem.open(childPath.c_str(), FILE_READ); - if (!childFile) { - dir.close(); - return {DbStatusCode::IoError, "open child file info failed"}; - } - info.size = childFile.size(); - childFile.close(); - } - entries.push_back(info); - - if (recursive && isDirectory) { - pendingDirs.emplace_back(joinPath(absoluteDir, segment), info.path); - } - } - dir.close(); - } - - for (const auto &pending : pendingDirs) { - auto st = collectDirectoryEntries(filesystem, pending.first, pending.second, true, entries); - if (!st.ok()) - return st; - } - - return {DbStatusCode::Ok, ""}; -} - -DbStatus writeFromPullCb( - fs::FS &filesystem, - const std::string &finalPath, - bool usePSRAMBuffers, - const ESPJsonDBFileOptions &opts, - const DbFileUploadPullCb &pullCb, - size_t &totalWritten -) { - totalWritten = 0; - if (!pullCb) { - return {DbStatusCode::InvalidArgument, "upload callback is required"}; - } - - const size_t chunkSize = opts.chunkSize < 32 ? 32 : opts.chunkSize; - JsonDbVector buffer{JsonDbAllocator(usePSRAMBuffers)}; - buffer.resize(chunkSize); - - const std::string parentDir = parentDirOf(finalPath); - const std::string tmpPath = finalPath + ".tmp"; - - FrLock fs(g_fsMutex); - if (!fsEnsureDir(filesystem, parentDir)) { - return {DbStatusCode::IoError, "mkdir file parent failed"}; - } - if (!opts.overwrite && filesystem.exists(finalPath.c_str())) { - return {DbStatusCode::AlreadyExists, "file already exists"}; - } - if (filesystem.exists(tmpPath.c_str())) { - filesystem.remove(tmpPath.c_str()); - } - - File file = filesystem.open(tmpPath.c_str(), FILE_WRITE); - if (!file) { - return {DbStatusCode::IoError, "open file for write failed"}; - } - WriteBufferingStream buffered(file, chunkSize); - - auto fail = [&](DbStatus st) { - buffered.flush(); - file.close(); - filesystem.remove(tmpPath.c_str()); - return st; - }; - - for (;;) { - size_t produced = 0; - bool eof = false; - auto st = pullCb(chunkSize, buffer.data(), produced, eof); - if (!st.ok()) - return fail(st); - if (produced > chunkSize) { - return fail({DbStatusCode::InvalidArgument, "upload callback produced too many bytes"}); - } - if (produced > 0) { - size_t written = buffered.write(buffer.data(), produced); - if (written != produced) { - return fail({DbStatusCode::IoError, "file write failed"}); - } - totalWritten += written; - } - if (eof) - break; - if (produced == 0) { - return fail( - {DbStatusCode::InvalidArgument, "upload callback produced no bytes without eof"} - ); - } - } - - buffered.flush(); - file.close(); - - if (opts.overwrite && filesystem.exists(finalPath.c_str()) && - !filesystem.remove(finalPath.c_str())) { - filesystem.remove(tmpPath.c_str()); - return {DbStatusCode::IoError, "remove old file failed"}; - } - if (!filesystem.rename(tmpPath.c_str(), finalPath.c_str())) { - filesystem.remove(tmpPath.c_str()); - return {DbStatusCode::IoError, "rename file failed"}; - } - - return {DbStatusCode::Ok, ""}; -} - -} // namespace - -DbStatus -ESPJsonDB::normalizeFilePath(const std::string &rawRelativePath, std::string &normalized) const { - normalized.clear(); - if (rawRelativePath.empty()) { - return {DbStatusCode::InvalidArgument, "file path is empty"}; - } - - std::string segment; - segment.reserve(rawRelativePath.size()); - - auto flushSegment = [&](void) -> DbStatus { - if (segment.empty() || segment == ".") { - segment.clear(); - return {DbStatusCode::Ok, ""}; - } - if (segment == "..") { - segment.clear(); - return {DbStatusCode::InvalidArgument, "path traversal is not allowed"}; - } - if (segment.find(':') != std::string::npos) { - segment.clear(); - return {DbStatusCode::InvalidArgument, "invalid file path segment"}; - } - if (!normalized.empty()) { - normalized.push_back('/'); - } - normalized += segment; - segment.clear(); - return {DbStatusCode::Ok, ""}; - }; - - for (char c : rawRelativePath) { - if (c == '\\') - c = '/'; - if (c == '/') { - auto st = flushSegment(); - if (!st.ok()) - return st; - continue; - } - segment.push_back(c); - } - auto st = flushSegment(); - if (!st.ok()) - return st; - - if (normalized.empty()) { - return {DbStatusCode::InvalidArgument, "file path resolves to empty"}; - } - - return {DbStatusCode::Ok, ""}; -} - -DbStatus ESPJsonDB::writeFileStream( - const std::string &relativePath, - Stream &in, - size_t bytesToWrite, - const ESPJsonDBFileOptions &opts -) { - auto ready = ensureReady(); - if (!ready.ok()) - return setLastError(ready); - - std::string normalized; - auto nst = normalizeFilePath(relativePath, normalized); - if (!nst.ok()) - return setLastError(nst); - - const std::string finalPath = joinPath(fileRootDir(), normalized); - size_t remaining = bytesToWrite; - DbFileUploadPullCb pullCb = - [&in, - &remaining](size_t requested, uint8_t *buffer, size_t &produced, bool &eof) -> DbStatus { - if (!buffer) { - return {DbStatusCode::InvalidArgument, "buffer is null"}; - } - if (remaining == 0) { - produced = 0; - eof = true; - return {DbStatusCode::Ok, ""}; - } - size_t want = std::min(requested, remaining); - produced = in.readBytes(reinterpret_cast(buffer), want); - if (produced == 0) { - eof = false; - return {DbStatusCode::IoError, "stream ended before expected size"}; - } - remaining -= produced; - eof = (remaining == 0); - return {DbStatusCode::Ok, ""}; - }; - - size_t totalWritten = 0; - auto st = writeFromPullCb(*_fs, finalPath, _cfg.usePSRAMBuffers, opts, pullCb, totalWritten); - if (!st.ok()) - return setLastError(st); - if (totalWritten != bytesToWrite) { - return setLastError({DbStatusCode::IoError, "written size mismatch"}); - } - - return setLastError({DbStatusCode::Ok, ""}); -} - -DbStatus ESPJsonDB::writeFileStream( - const std::string &relativePath, - const DbFileUploadPullCb &pullCb, - const ESPJsonDBFileOptions &opts -) { - auto ready = ensureReady(); - if (!ready.ok()) - return setLastError(ready); - - std::string normalized; - auto nst = normalizeFilePath(relativePath, normalized); - if (!nst.ok()) - return setLastError(nst); - - const std::string finalPath = joinPath(fileRootDir(), normalized); - size_t totalWritten = 0; - auto st = writeFromPullCb(*_fs, finalPath, _cfg.usePSRAMBuffers, opts, pullCb, totalWritten); - return setLastError(st); -} - -DbStatus ESPJsonDB::writeFileFromPath( - const std::string &relativePath, - const std::string &sourceFsPath, - const ESPJsonDBFileOptions &opts -) { - if (sourceFsPath.empty()) { - return setLastError({DbStatusCode::InvalidArgument, "source file path is empty"}); - } - auto ready = ensureReady(); - if (!ready.ok()) - return setLastError(ready); - - File source; - { - FrLock fs(g_fsMutex); - source = _fs->open(sourceFsPath.c_str(), FILE_READ); - } - if (!source) { - return setLastError({DbStatusCode::NotFound, "source file not found"}); - } - - const size_t bytesToWrite = source.size(); - auto st = writeFileStream(relativePath, source, bytesToWrite, opts); - source.close(); - return st; -} - -DbStatus ESPJsonDB::writeFile( - const std::string &relativePath, const uint8_t *data, size_t size, bool overwrite -) { - if (size > 0 && data == nullptr) { - return setLastError({DbStatusCode::InvalidArgument, "file data is null"}); - } - - auto ready = ensureReady(); - if (!ready.ok()) - return setLastError(ready); - - std::string normalized; - auto nst = normalizeFilePath(relativePath, normalized); - if (!nst.ok()) - return setLastError(nst); - - const std::string finalPath = joinPath(fileRootDir(), normalized); - const std::string parentDir = parentDirOf(finalPath); - const std::string tmpPath = finalPath + ".tmp"; - - FrLock fs(g_fsMutex); - if (!fsEnsureDir(*_fs, parentDir)) { - return setLastError({DbStatusCode::IoError, "mkdir file parent failed"}); - } - if (!overwrite && _fs->exists(finalPath.c_str())) { - return setLastError({DbStatusCode::AlreadyExists, "file already exists"}); - } - if (_fs->exists(tmpPath.c_str())) { - _fs->remove(tmpPath.c_str()); - } - - File f = _fs->open(tmpPath.c_str(), FILE_WRITE); - if (!f) { - return setLastError({DbStatusCode::IoError, "open file for write failed"}); - } - WriteBufferingStream buffered(f, 256); - - size_t written = 0; - if (size > 0) { - written = buffered.write(data, size); - } - buffered.flush(); - f.close(); - if (written != size) { - _fs->remove(tmpPath.c_str()); - return setLastError({DbStatusCode::IoError, "file write failed"}); - } - - if (overwrite && _fs->exists(finalPath.c_str()) && !_fs->remove(finalPath.c_str())) { - _fs->remove(tmpPath.c_str()); - return setLastError({DbStatusCode::IoError, "remove old file failed"}); - } - if (!_fs->rename(tmpPath.c_str(), finalPath.c_str())) { - _fs->remove(tmpPath.c_str()); - return setLastError({DbStatusCode::IoError, "rename file failed"}); - } - - return setLastError({DbStatusCode::Ok, ""}); -} - -DbStatus -ESPJsonDB::writeTextFile(const std::string &relativePath, const std::string &text, bool overwrite) { - return writeFile( - relativePath, - reinterpret_cast(text.data()), - text.size(), - overwrite - ); -} - -DbResult -ESPJsonDB::readFileStream(const std::string &relativePath, Stream &out, size_t chunkSize) { - DbResult res{}; - auto ready = ensureReady(); - if (!ready.ok()) { - res.status = setLastError(ready); - return res; - } - - std::string normalized; - auto nst = normalizeFilePath(relativePath, normalized); - if (!nst.ok()) { - res.status = setLastError(nst); - return res; - } - - if (chunkSize < 32) - chunkSize = 32; - JsonDbVector buffer{JsonDbAllocator(_cfg.usePSRAMBuffers)}; - buffer.resize(chunkSize); - const std::string path = joinPath(fileRootDir(), normalized); - - FrLock fs(g_fsMutex); - File f = _fs->open(path.c_str(), FILE_READ); - if (!f) { - res.status = setLastError({DbStatusCode::NotFound, "file not found"}); - return res; - } - - size_t total = 0; - while (true) { - size_t readBytes = f.read(buffer.data(), buffer.size()); - if (readBytes == 0) - break; - size_t written = out.write(buffer.data(), readBytes); - if (written != readBytes) { - f.close(); - res.status = setLastError({DbStatusCode::IoError, "output stream write failed"}); - return res; - } - total += written; - } - f.close(); - - res.status = setLastError({DbStatusCode::Ok, ""}); - res.value = total; - return res; -} - -DbResult> ESPJsonDB::readFile(const std::string &relativePath) { - DbResult> res{}; - auto ready = ensureReady(); - if (!ready.ok()) { - res.status = setLastError(ready); - return res; - } - - std::string normalized; - auto nst = normalizeFilePath(relativePath, normalized); - if (!nst.ok()) { - res.status = setLastError(nst); - return res; - } - - const std::string path = joinPath(fileRootDir(), normalized); - - FrLock fs(g_fsMutex); - File f = _fs->open(path.c_str(), FILE_READ); - if (!f) { - res.status = setLastError({DbStatusCode::NotFound, "file not found"}); - return res; - } - - size_t sz = f.size(); - res.value.resize(sz); - size_t readBytes = 0; - if (sz > 0) { - readBytes = f.read(res.value.data(), sz); - } - f.close(); - if (readBytes != sz) { - res.value.clear(); - res.status = setLastError({DbStatusCode::IoError, "file read failed"}); - return res; - } - - res.status = setLastError({DbStatusCode::Ok, ""}); - return res; -} - -DbResult ESPJsonDB::readTextFile(const std::string &relativePath) { - DbResult res{}; - auto fr = readFile(relativePath); - if (!fr.status.ok()) { - res.status = fr.status; - return res; - } - res.value.assign(fr.value.begin(), fr.value.end()); - res.status = setLastError({DbStatusCode::Ok, ""}); - return res; -} - -DbResult ESPJsonDB::getFileInfo(const std::string &relativePath) { - DbResult res{}; - auto ready = ensureReady(); - if (!ready.ok()) { - res.status = setLastError(ready); - return res; - } - - std::string normalized; - auto nst = normalizeFilePath(relativePath, normalized); - if (!nst.ok()) { - res.status = setLastError(nst); - return res; - } - - FileEntryInfo info; - const auto st = statFileEntry(*_fs, joinPath(fileRootDir(), normalized), normalized, info); - if (!st.ok()) { - res.status = setLastError(st); - return res; - } - - res.value["path"] = info.path.c_str(); - res.value["name"] = fileNameOf(info.path).c_str(); - res.value["exists"] = true; - res.value["isDirectory"] = info.isDirectory; - res.value["size"] = info.size; - res.status = setLastError({DbStatusCode::Ok, ""}); - return res; -} - -DbResult -ESPJsonDB::listFiles(const std::string &relativePrefix, bool recursive) { - DbResult res{}; - auto ready = ensureReady(); - if (!ready.ok()) { - res.status = setLastError(ready); - return res; - } - - std::string normalizedPrefix; - if (!relativePrefix.empty()) { - auto nst = normalizeFilePath(relativePrefix, normalizedPrefix); - if (!nst.ok()) { - res.status = setLastError(nst); - return res; - } - } - - const std::string rootPath = fileRootDir(); - const std::string targetPath = - normalizedPrefix.empty() ? rootPath : joinPath(rootPath, normalizedPrefix); - - FileEntryInfo targetInfo; - auto st = statFileEntry(*_fs, targetPath, normalizedPrefix, targetInfo); - if (!st.ok()) { - res.status = setLastError(st); - return res; - } - - std::vector entries; - if (targetInfo.isDirectory) { - st = collectDirectoryEntries(*_fs, targetPath, normalizedPrefix, recursive, entries); - if (!st.ok()) { - res.status = setLastError(st); - return res; - } - } else { - entries.push_back(targetInfo); - } - - std::sort(entries.begin(), entries.end(), [](const FileEntryInfo &lhs, const FileEntryInfo &rhs) { - return lhs.path < rhs.path; - }); - - res.value["prefix"] = normalizedPrefix.c_str(); - res.value["recursive"] = recursive; - JsonArray entriesJson = res.value["entries"].to(); - for (const auto &entry : entries) { - appendFileInfoJson(entriesJson, entry); - } - - res.status = setLastError({DbStatusCode::Ok, ""}); - return res; -} - -DbStatus ESPJsonDB::removeFile(const std::string &relativePath) { - auto ready = ensureReady(); - if (!ready.ok()) - return setLastError(ready); - - std::string normalized; - auto nst = normalizeFilePath(relativePath, normalized); - if (!nst.ok()) - return setLastError(nst); - - const std::string path = joinPath(fileRootDir(), normalized); - FrLock fs(g_fsMutex); - if (!_fs->exists(path.c_str())) { - return setLastError({DbStatusCode::NotFound, "file not found"}); - } - if (!_fs->remove(path.c_str())) { - return setLastError({DbStatusCode::IoError, "file remove failed"}); - } - return setLastError({DbStatusCode::Ok, ""}); -} - -DbResult ESPJsonDB::fileExists(const std::string &relativePath) { - DbResult res{}; - auto ready = ensureReady(); - if (!ready.ok()) { - res.status = setLastError(ready); - return res; - } - - std::string normalized; - auto nst = normalizeFilePath(relativePath, normalized); - if (!nst.ok()) { - res.status = setLastError(nst); - return res; - } - - const std::string path = joinPath(fileRootDir(), normalized); - FrLock fs(g_fsMutex); - res.value = _fs->exists(path.c_str()); - res.status = setLastError({DbStatusCode::Ok, ""}); - return res; -} - -DbResult ESPJsonDB::fileSize(const std::string &relativePath) { - DbResult res{}; - auto ready = ensureReady(); - if (!ready.ok()) { - res.status = setLastError(ready); - return res; - } - - std::string normalized; - auto nst = normalizeFilePath(relativePath, normalized); - if (!nst.ok()) { - res.status = setLastError(nst); - return res; - } - - const std::string path = joinPath(fileRootDir(), normalized); - FrLock fs(g_fsMutex); - File f = _fs->open(path.c_str(), FILE_READ); - if (!f) { - res.status = setLastError({DbStatusCode::NotFound, "file not found"}); - return res; - } - res.value = f.size(); - f.close(); - res.status = setLastError({DbStatusCode::Ok, ""}); - return res; -} diff --git a/src/esp_jsondb/db_files_async.cpp b/src/esp_jsondb/db_files_async.cpp deleted file mode 100644 index 927bd1c..0000000 --- a/src/esp_jsondb/db_files_async.cpp +++ /dev/null @@ -1,386 +0,0 @@ -#include "db.h" - -#include - -#include "utils/fs_utils.h" -#include "utils/jsondb_allocator.h" - -namespace { - -std::string parentDirForAsyncUpload(const std::string &path) { - auto pos = path.find_last_of('/'); - if (pos == std::string::npos || pos == 0) { - return "/"; - } - return path.substr(0, pos); -} - -} // namespace - -bool ESPJsonDB::isUploadTerminal(DbFileUploadState state) const { - return state == DbFileUploadState::Completed || state == DbFileUploadState::Failed || - state == DbFileUploadState::Cancelled; -} - -void ESPJsonDB::trackTerminalUploadLocked(const std::shared_ptr &job) { - if (!job || !isUploadTerminal(job->state) || job->terminalTracked) - return; - - job->terminalTracked = true; - _terminalUploadOrder.push_back(job->id); - - while (_terminalUploadOrder.size() > kMaxRetainedTerminalUploads) { - const uint32_t expiredId = _terminalUploadOrder.front(); - _terminalUploadOrder.pop_front(); - - auto it = _uploadJobs.find(expiredId); - if (it == _uploadJobs.end()) - continue; - if (!it->second || isUploadTerminal(it->second->state)) { - _uploadJobs.erase(it); - } - } -} - -DbResult ESPJsonDB::writeFileStreamAsync( - const std::string &relativePath, - const DbFileUploadPullCb &pullCb, - const ESPJsonDBFileOptions &opts, - const DbFileUploadDoneCb &doneCb -) { - DbResult res{}; - auto ready = ensureReady(); - if (!ready.ok()) { - res.status = setLastError(ready); - return res; - } - if (!pullCb) { - res.status = setLastError({DbStatusCode::InvalidArgument, "upload callback is required"}); - return res; - } - - std::string normalized; - auto nst = normalizeFilePath(relativePath, normalized); - if (!nst.ok()) { - res.status = setLastError(nst); - return res; - } - - auto job = std::make_shared(); - job->relativePath = relativePath; - job->normalizedPath = normalized; - job->opts = opts; - if (job->opts.chunkSize < 32) - job->opts.chunkSize = 32; - job->pullCb = pullCb; - job->doneCb = doneCb; - - { - FrLock lk(_mu); - job->id = _nextUploadId++; - _uploadJobs[job->id] = job; - _uploadQueue.push_back(job->id); - startFileUploadTaskUnlocked(); - if (_fileUploadTask == nullptr) { - _uploadQueue.erase( - std::remove(_uploadQueue.begin(), _uploadQueue.end(), job->id), - _uploadQueue.end() - ); - _uploadJobs.erase(job->id); - res.status = setLastError({DbStatusCode::Busy, "upload worker start failed"}); - return res; - } - } - - res.status = setLastError({DbStatusCode::Ok, ""}); - res.value = job->id; - return res; -} - -DbStatus ESPJsonDB::cancelFileUpload(uint32_t uploadId) { - DbFileUploadDoneCb doneCb; - DbStatus doneStatus{DbStatusCode::Busy, "upload cancelled"}; - size_t bytesWritten = 0; - bool triggerDone = false; - - { - FrLock lk(_mu); - auto it = _uploadJobs.find(uploadId); - if (it == _uploadJobs.end()) { - return setLastError({DbStatusCode::NotFound, "upload not found"}); - } - auto &job = it->second; - if (!job) { - return setLastError({DbStatusCode::NotFound, "upload not found"}); - } - if (isUploadTerminal(job->state)) { - return setLastError({DbStatusCode::Ok, ""}); - } - - job->cancelRequested = true; - if (job->state == DbFileUploadState::Queued) { - _uploadQueue.erase( - std::remove(_uploadQueue.begin(), _uploadQueue.end(), uploadId), - _uploadQueue.end() - ); - job->state = DbFileUploadState::Cancelled; - job->finalStatus = doneStatus; - trackTerminalUploadLocked(job); - doneCb = job->doneCb; - bytesWritten = job->bytesWritten; - triggerDone = true; - } - } - - if (triggerDone && doneCb) { - doneCb(uploadId, doneStatus, bytesWritten); - } - return setLastError({DbStatusCode::Ok, ""}); -} - -DbResult ESPJsonDB::getFileUploadState(uint32_t uploadId) { - DbResult res{}; - auto ready = ensureReady(); - if (!ready.ok()) { - res.status = setLastError(ready); - return res; - } - - { - FrLock lk(_mu); - auto it = _uploadJobs.find(uploadId); - if (it == _uploadJobs.end() || !it->second) { - res.status = setLastError({DbStatusCode::NotFound, "upload not found"}); - return res; - } - res.value = it->second->state; - } - - res.status = setLastError({DbStatusCode::Ok, ""}); - return res; -} - -void ESPJsonDB::fileUploadTaskThunk(void *arg) { - auto *self = static_cast(arg); - self->fileUploadTaskLoop(); -} - -void ESPJsonDB::startFileUploadTaskUnlocked() { - if (_fileUploadTask != nullptr) - return; - _fileUploadStopRequested.store(false, std::memory_order_release); - _fileUploadTaskExited.store(false, std::memory_order_release); - TaskHandle_t handle = nullptr; - if (createTask(fileUploadTaskThunk, "db.file.upload", handle)) { - _fileUploadTask = handle; - } else { - _fileUploadTaskExited.store(true, std::memory_order_release); - } -} - -void ESPJsonDB::stopFileUploadTaskUnlocked(bool cancelPending) { - if (cancelPending) { - for (auto &kv : _uploadJobs) { - auto &job = kv.second; - if (!job || isUploadTerminal(job->state)) - continue; - job->cancelRequested = true; - job->state = DbFileUploadState::Cancelled; - job->finalStatus = {DbStatusCode::Busy, "upload cancelled"}; - } - _uploadQueue.clear(); - } - if (_fileUploadTask) { - stopTask(_fileUploadTask, _fileUploadStopRequested, _fileUploadTaskExited); - } -} - -DbStatus -ESPJsonDB::runFileUploadJob(const std::shared_ptr &job, size_t &bytesWritten) { - bytesWritten = 0; - if (!job || !job->pullCb) { - return {DbStatusCode::InvalidArgument, "upload callback is required"}; - } - - const size_t chunkSize = job->opts.chunkSize < 32 ? 32 : job->opts.chunkSize; - JsonDbVector buffer{JsonDbAllocator(_cfg.usePSRAMBuffers)}; - buffer.resize(chunkSize); - - const std::string finalPath = joinPath(fileRootDir(), job->normalizedPath); - const std::string parentDir = parentDirForAsyncUpload(finalPath); - const std::string tmpPath = finalPath + ".tmp"; - - { - FrLock fs(g_fsMutex); - if (!fsEnsureDir(*_fs, parentDir)) { - return {DbStatusCode::IoError, "mkdir file parent failed"}; - } - if (!job->opts.overwrite && _fs->exists(finalPath.c_str())) { - return {DbStatusCode::AlreadyExists, "file already exists"}; - } - if (_fs->exists(tmpPath.c_str())) { - _fs->remove(tmpPath.c_str()); - } - } - - File f; - { - FrLock fs(g_fsMutex); - f = _fs->open(tmpPath.c_str(), FILE_WRITE); - } - if (!f) { - return {DbStatusCode::IoError, "open file for write failed"}; - } - - auto cleanupTmp = [&]() { - FrLock fs(g_fsMutex); - f.close(); - if (_fs->exists(tmpPath.c_str())) { - _fs->remove(tmpPath.c_str()); - } - }; - - for (;;) { - bool cancelled = false; - { - FrLock lk(_mu); - auto it = _uploadJobs.find(job->id); - cancelled = (it == _uploadJobs.end() || !it->second || it->second->cancelRequested); - } - if (cancelled) { - cleanupTmp(); - return {DbStatusCode::Busy, "upload cancelled"}; - } - - size_t produced = 0; - bool eof = false; - auto st = job->pullCb(chunkSize, buffer.data(), produced, eof); - if (!st.ok()) { - cleanupTmp(); - return st; - } - if (produced > chunkSize) { - cleanupTmp(); - return {DbStatusCode::InvalidArgument, "upload callback produced too many bytes"}; - } - - if (produced > 0) { - size_t written = 0; - { - FrLock fs(g_fsMutex); - written = f.write(buffer.data(), produced); - } - if (written != produced) { - cleanupTmp(); - return {DbStatusCode::IoError, "file write failed"}; - } - bytesWritten += written; - } - - if (eof) { - break; - } - if (produced == 0) { - vTaskDelay(pdMS_TO_TICKS(1)); - } - } - - { - FrLock fs(g_fsMutex); - f.flush(); - f.close(); - if (job->opts.overwrite && _fs->exists(finalPath.c_str()) && - !_fs->remove(finalPath.c_str())) { - if (_fs->exists(tmpPath.c_str())) { - _fs->remove(tmpPath.c_str()); - } - return {DbStatusCode::IoError, "remove old file failed"}; - } - if (!_fs->rename(tmpPath.c_str(), finalPath.c_str())) { - if (_fs->exists(tmpPath.c_str())) { - _fs->remove(tmpPath.c_str()); - } - return {DbStatusCode::IoError, "rename file failed"}; - } - } - - return {DbStatusCode::Ok, ""}; -} - -void ESPJsonDB::fileUploadTaskLoop() { - while (!_fileUploadStopRequested.load(std::memory_order_acquire)) { - std::shared_ptr job; - { - FrLock lk(_mu); - if (!_uploadQueue.empty()) { - auto id = _uploadQueue.front(); - _uploadQueue.pop_front(); - auto it = _uploadJobs.find(id); - if (it != _uploadJobs.end()) { - job = it->second; - if (job && job->state == DbFileUploadState::Queued && !job->cancelRequested) { - job->state = DbFileUploadState::Running; - } - } - } - } - - if (!job) { - vTaskDelay(pdMS_TO_TICKS(20)); - continue; - } - - if (job->cancelRequested) { - DbFileUploadDoneCb doneCb; - { - FrLock lk(_mu); - job->state = DbFileUploadState::Cancelled; - job->finalStatus = {DbStatusCode::Busy, "upload cancelled"}; - trackTerminalUploadLocked(job); - doneCb = job->doneCb; - } - if (doneCb) { - doneCb(job->id, job->finalStatus, job->bytesWritten); - } - continue; - } - - size_t bytesWritten = 0; - auto st = runFileUploadJob(job, bytesWritten); - - DbFileUploadDoneCb doneCb; - DbStatus finalStatus = st; - DbFileUploadState finalState = DbFileUploadState::Failed; - { - FrLock lk(_mu); - auto it = _uploadJobs.find(job->id); - if (it != _uploadJobs.end() && it->second) { - auto &j = it->second; - j->bytesWritten = bytesWritten; - if (j->cancelRequested) { - finalState = DbFileUploadState::Cancelled; - finalStatus = {DbStatusCode::Busy, "upload cancelled"}; - } else if (st.ok()) { - finalState = DbFileUploadState::Completed; - finalStatus = st; - } else { - finalState = DbFileUploadState::Failed; - finalStatus = st; - } - j->state = finalState; - j->finalStatus = finalStatus; - trackTerminalUploadLocked(j); - doneCb = j->doneCb; - } - } - - if (!finalStatus.ok() && finalState != DbFileUploadState::Cancelled) { - setLastError(finalStatus); - } - if (doneCb) { - doneCb(job->id, finalStatus, bytesWritten); - } - } - _fileUploadTaskExited.store(true, std::memory_order_release); - vTaskDelete(nullptr); -} diff --git a/src/esp_jsondb/db_runtime.h b/src/esp_jsondb/db_runtime.h new file mode 100644 index 0000000..8b187f6 --- /dev/null +++ b/src/esp_jsondb/db_runtime.h @@ -0,0 +1,84 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "files/file_store.h" +#include "utils/dbTypes.h" +#include "utils/fr_mutex.h" +#include "utils/jsondb_allocator.h" +#include "utils/schema.h" + +class Collection; +class ESPJsonDB; +struct FileStoreImpl; + +struct DbRuntime { + using CollectionMap = JsonDbMap>; + using SchemaMap = JsonDbMap; + using StringBoolMap = JsonDbMap; + using CollectionConfigMap = JsonDbMap; + using StringUint32Map = JsonDbMap; + using StringVector = JsonDbVector; + using EventCallbackVector = JsonDbVector>; + using ErrorCallbackVector = JsonDbVector>; + using SyncStatusCallbackVector = JsonDbVector>; + + struct DiagCache { + StringUint32Map docsPerCollection; + uint32_t collections = 0; + uint32_t lastRefreshMs = 0; + }; + + CollectionMap cols; + SchemaMap schemas; + CollectionConfigMap collectionConfigs; + StringVector colsToDelete; + EventCallbackVector eventCbs; + ErrorCallbackVector errorCbs; + SyncStatusCallbackVector syncStatusCbs; + StringBoolMap pendingDelayedCollections; + std::string baseDir; + ESPJsonDBConfig cfg{}; + fs::FS *fs = &LittleFS; + FrMutex mu; + DbStatus lastError{DbStatusCode::Ok, ""}; + DBSyncStatus lastSyncStatus{}; + DiagCache diagCache; + bool diagCachePrimed = false; + std::atomic initialized{false}; + TaskHandle_t syncTask = nullptr; + std::atomic syncStopRequested{false}; + std::atomic syncTaskExited{true}; + std::atomic syncKickRequested{false}; + std::atomic syncRequestSeq{0}; + std::atomic syncCompletedSeq{0}; + bool delayedPreloadPhaseCompleted = true; + bool dropAllRequested = false; + ESPJsonDB *owner = nullptr; + std::unique_ptr fileStoreImpl; + std::unique_ptr fileStore; + + explicit DbRuntime(bool usePSRAMBuffers = false); + ~DbRuntime(); + + DbStatus ensureReady() const; + DbStatus recordStatus(const DbStatus &st); + void emitEvent(DBEventType ev); + void emitError(const DbStatus &st); + void noteDocumentCreated(const std::string &collectionName, uint32_t count = 1); + void noteDocumentDeleted(const std::string &collectionName, uint32_t count = 1); + std::string fileRootDir() const; + bool createTask(TaskFunction_t entry, const char *name, void *arg, TaskHandle_t &outHandle); + void stopTask( + TaskHandle_t &taskHandle, std::atomic &stopRequested, std::atomic &taskExited + ); + static uint32_t stackBytesToWords(uint32_t stackBytes); +}; diff --git a/src/esp_jsondb/document/document.cpp b/src/esp_jsondb/document/document.cpp index 89309c9..9a60801 100644 --- a/src/esp_jsondb/document/document.cpp +++ b/src/esp_jsondb/document/document.cpp @@ -10,19 +10,83 @@ DocView::DocView( FrMutex *mu, ESPJsonDB *db, std::function &)> commitSink, + std::function decodeAcquire, + std::function decodeRelease, + std::function pinAcquire, + std::function pinRelease, + bool pinHeld, bool usePSRAMBuffers ) : _rec(std::move(rec)), _schema(schema), _mu(mu), _db(db), _commitSink(std::move(commitSink)), - _usePSRAMBuffers(usePSRAMBuffers) + _decodeAcquire(std::move(decodeAcquire)), _decodeRelease(std::move(decodeRelease)), + _pinRelease(std::move(pinRelease)), _usePSRAMBuffers(usePSRAMBuffers), _pinHeld(pinHeld) #if ESP_JSONDB_HAS_JSONDOC_ALLOCATOR , _docAllocator(usePSRAMBuffers) #endif { + if (_rec && !_pinHeld && pinAcquire) { + auto st = pinAcquire(); + _pinHeld = st.ok(); + recordStatus(st); + } +} + +DocView::DocView(DocView &&other) noexcept + : _rec(std::move(other._rec)), _schema(other._schema), _doc(std::move(other._doc)), + _dirtyLocally(other._dirtyLocally), _mu(other._mu), _db(other._db), + _commitSink(std::move(other._commitSink)), _decodeAcquire(std::move(other._decodeAcquire)), + _decodeRelease(std::move(other._decodeRelease)), _pinRelease(std::move(other._pinRelease)), + _usePSRAMBuffers(other._usePSRAMBuffers), _decodeReserved(other._decodeReserved), + _pinHeld(other._pinHeld) +#if ESP_JSONDB_HAS_JSONDOC_ALLOCATOR + , + _docAllocator(other._usePSRAMBuffers) +#endif +{ + other._decodeReserved = false; + other._pinHeld = false; +} + +DocView &DocView::operator=(DocView &&other) noexcept { + if (this == &other) + return *this; + releaseResources(); + _rec = std::move(other._rec); + _schema = other._schema; + _doc = std::move(other._doc); + _dirtyLocally = other._dirtyLocally; + _mu = other._mu; + _db = other._db; + _commitSink = std::move(other._commitSink); + _decodeAcquire = std::move(other._decodeAcquire); + _decodeRelease = std::move(other._decodeRelease); + _pinRelease = std::move(other._pinRelease); + _usePSRAMBuffers = other._usePSRAMBuffers; + _decodeReserved = other._decodeReserved; + _pinHeld = other._pinHeld; +#if ESP_JSONDB_HAS_JSONDOC_ALLOCATOR + _docAllocator.setUsePSRAMBuffers(_usePSRAMBuffers); +#endif + other._decodeReserved = false; + other._pinHeld = false; + return *this; } DocView::~DocView() { - // no auto-commit by default; discard decoded state + releaseResources(); +} + +void DocView::releaseResources() { + _doc.reset(); + if (_decodeReserved && _decodeRelease) { + _decodeRelease(); + } + if (_pinHeld && _pinRelease) { + _pinRelease(); + } + _decodeReserved = false; + _pinHeld = false; } DbStatus DocView::decode() { @@ -31,6 +95,12 @@ DbStatus DocView::decode() { guard = std::make_unique(*_mu); if (_doc) return recordStatus({DbStatusCode::Ok, ""}); + if (_decodeAcquire) { + auto st = _decodeAcquire(); + if (!st.ok()) + return recordStatus(st); + _decodeReserved = true; + } #if ESP_JSONDB_HAS_JSONDOC_ALLOCATOR _docAllocator.setUsePSRAMBuffers(_usePSRAMBuffers); _doc = std::make_unique(&_docAllocator); @@ -50,6 +120,10 @@ DbStatus DocView::decode() { err = deserializeMsgPack(*_doc, _rec->msgpack.data(), _rec->msgpack.size()); if (err) { _doc.reset(); + if (_decodeReserved && _decodeRelease) { + _decodeRelease(); + } + _decodeReserved = false; return recordStatus({DbStatusCode::Corrupted, "msgpack decode failed"}); } } @@ -126,7 +200,8 @@ DbStatus DocView::encode() { if (written != sz) { return recordStatus({DbStatusCode::IoError, "serialize msgpack size mismatch"}); } - _rec->meta.updatedAt = nowUtcMs(); + _rec->meta.updatedAtMs = nowUtcMs(); + _rec->meta.revision = static_cast(_rec->meta.revision + 1U); _rec->meta.dirty = true; _dirtyLocally = false; return recordStatus({DbStatusCode::Ok, ""}); @@ -209,7 +284,13 @@ DbStatus DocView::commit() { } void DocView::discard() { - _doc.reset(); + if (_doc) { + _doc.reset(); + if (_decodeReserved && _decodeRelease) { + _decodeRelease(); + } + _decodeReserved = false; + } _dirtyLocally = false; } @@ -225,20 +306,28 @@ DocRef DocView::getRef(const char *field) const { DocView DocView::populate(const char *field, uint8_t maxDepth) const { if (maxDepth == 0) { recordStatus({DbStatusCode::InvalidArgument, "max depth reached"}); - return DocView(nullptr, nullptr, nullptr, _db, nullptr, _usePSRAMBuffers); + return DocView( + nullptr, nullptr, nullptr, _db, nullptr, nullptr, nullptr, nullptr, nullptr, false, _usePSRAMBuffers + ); } auto ref = getRef(field); if (!ref.valid()) { recordStatus({DbStatusCode::InvalidArgument, "field not DocRef"}); - return DocView(nullptr, nullptr, nullptr, _db, nullptr, _usePSRAMBuffers); + return DocView( + nullptr, nullptr, nullptr, _db, nullptr, nullptr, nullptr, nullptr, nullptr, false, _usePSRAMBuffers + ); } if (!_db) { recordStatus({DbStatusCode::InvalidArgument, "database context unavailable"}); - return DocView(nullptr, nullptr, nullptr, _db, nullptr, _usePSRAMBuffers); + return DocView( + nullptr, nullptr, nullptr, _db, nullptr, nullptr, nullptr, nullptr, nullptr, false, _usePSRAMBuffers + ); } auto fr = _db->findById(ref.collection, ref.id); if (!fr.status.ok()) - return DocView(nullptr, nullptr, nullptr, _db, nullptr, _usePSRAMBuffers); + return DocView( + nullptr, nullptr, nullptr, _db, nullptr, nullptr, nullptr, nullptr, nullptr, false, _usePSRAMBuffers + ); if (maxDepth > 1) { for (auto kv : fr.value.asObjectConst()) { auto nested = docRefFromJson(kv.value()); diff --git a/src/esp_jsondb/document/document.h b/src/esp_jsondb/document/document.h index c27984f..1531568 100644 --- a/src/esp_jsondb/document/document.h +++ b/src/esp_jsondb/document/document.h @@ -59,11 +59,13 @@ class JsonDbDocAllocator : public ArduinoJson::Allocator { * The database does not manage or check time synchronization. */ struct DocumentMeta { - uint32_t createdAt = 0; // UTC milliseconds - uint32_t updatedAt = 0; // UTC milliseconds - DocId id; // 24-hex ObjectId - bool dirty = false; // needs flush to FS - bool removed = false; // logically deleted; DocView::commit should fail + DocId id; // 24-hex ObjectId + uint64_t createdAtMs = 0; // UTC milliseconds + uint64_t updatedAtMs = 0; // UTC milliseconds + uint32_t revision = 0; + uint16_t flags = 0; + bool dirty = false; // needs flush to FS + bool removed = false; // logically deleted; DocView::commit should fail }; // Internal storage unit (owned by Collection) @@ -74,6 +76,8 @@ struct DocumentRecord { DocumentMeta meta; JsonDbVector msgpack; // authoritative source + uint32_t pinCount = 0; + uint64_t lastAccessSeq = 0; // Optional decoded cache; created on demand and freed when view // destroyed Decoding/encoding uses ArduinoJson. }; @@ -89,15 +93,20 @@ class DocView { FrMutex *mu = nullptr, ESPJsonDB *db = nullptr, std::function &)> commitSink = nullptr, + std::function decodeAcquire = nullptr, + std::function decodeRelease = nullptr, + std::function pinAcquire = nullptr, + std::function pinRelease = nullptr, + bool pinHeld = false, bool usePSRAMBuffers = false ); - ~DocView(); // optional auto-commit if enabled + ~DocView(); // non-copyable, movable DocView(const DocView &) = delete; DocView &operator=(const DocView &) = delete; - DocView(DocView &&) noexcept = default; - DocView &operator=(DocView &&) noexcept = default; + DocView(DocView &&other) noexcept; + DocView &operator=(DocView &&other) noexcept; // Read-only or mutable variant access (ArduinoJson API) JsonVariant operator[](const char *key); @@ -135,13 +144,19 @@ class DocView { FrMutex *_mu = nullptr; // optional: used when called without external lock ESPJsonDB *_db = nullptr; std::function &)> _commitSink; + std::function _decodeAcquire; + std::function _decodeRelease; + std::function _pinRelease; bool _usePSRAMBuffers = false; + bool _decodeReserved = false; + bool _pinHeld = false; #if ESP_JSONDB_HAS_JSONDOC_ALLOCATOR JsonDbDocAllocator _docAllocator; #endif DbStatus decode(); DbStatus encode(); DbStatus recordStatus(const DbStatus &st) const; + void releaseResources(); }; template T DocView::getOr(const char *field, T def) const { diff --git a/src/esp_jsondb/files/file_store.cpp b/src/esp_jsondb/files/file_store.cpp new file mode 100644 index 0000000..6ccce1f --- /dev/null +++ b/src/esp_jsondb/files/file_store.cpp @@ -0,0 +1,1181 @@ +#include "file_store.h" + +#include "file_store_impl.h" + +#include "../db_runtime.h" +#include "../utils/fr_mutex.h" +#include "../utils/fs_utils.h" +#include "../utils/jsondb_allocator.h" + +#include + +#include + +extern FrMutex g_fsMutex; + +namespace { + +std::string parentDirOf(const std::string &path) { + auto pos = path.find_last_of('/'); + if (pos == std::string::npos || pos == 0) { + return "/"; + } + return path.substr(0, pos); +} + +std::string fileNameOf(const std::string &path) { + auto pos = path.find_last_of('/'); + if (pos == std::string::npos) + return path; + return path.substr(pos + 1); +} + +struct FileEntryInfo { + std::string path; + bool isDirectory = false; + size_t size = 0; +}; + +DbStatus statFileEntry( + fs::FS &filesystem, + const std::string &absolutePath, + const std::string &relativePath, + FileEntryInfo &out +) { + FrLock fs(g_fsMutex); + if (!filesystem.exists(absolutePath.c_str())) { + return {DbStatusCode::NotFound, "file not found"}; + } + File file = filesystem.open(absolutePath.c_str(), FILE_READ); + if (!file) { + return {DbStatusCode::IoError, "open file info failed"}; + } + out.path = relativePath; + out.isDirectory = file.isDirectory(); + out.size = out.isDirectory ? 0 : file.size(); + file.close(); + return {DbStatusCode::Ok, ""}; +} + +void appendFileInfoJson(JsonArray entries, const FileEntryInfo &info) { + JsonObject entry = entries.add(); + entry["path"] = info.path.c_str(); + entry["name"] = fileNameOf(info.path).c_str(); + entry["exists"] = true; + entry["isDirectory"] = info.isDirectory; + entry["size"] = info.size; +} + +DbStatus collectDirectoryEntries( + fs::FS &filesystem, + const std::string &absoluteDir, + const std::string &relativeDir, + bool recursive, + std::vector &entries +) { + std::vector> pendingDirs; + { + FrLock fs(g_fsMutex); + File dir = filesystem.open(absoluteDir.c_str(), FILE_READ); + if (!dir || !dir.isDirectory()) { + if (dir) + dir.close(); + return {DbStatusCode::NotFound, "file not found"}; + } + for (File child = dir.openNextFile(); child; child = dir.openNextFile()) { + const bool isDirectory = child.isDirectory(); + String rawName = child.name(); + child.close(); + std::string segment = rawName.c_str(); + auto slash = segment.find_last_of('/'); + if (slash != std::string::npos) + segment = segment.substr(slash + 1); + if (segment.empty()) + continue; + + FileEntryInfo info; + info.path = relativeDir.empty() ? segment : joinPath(relativeDir, segment); + info.isDirectory = isDirectory; + if (!isDirectory) { + const std::string childPath = joinPath(absoluteDir, segment); + File childFile = filesystem.open(childPath.c_str(), FILE_READ); + if (!childFile) { + dir.close(); + return {DbStatusCode::IoError, "open child file info failed"}; + } + info.size = childFile.size(); + childFile.close(); + } + entries.push_back(info); + + if (recursive && isDirectory) { + pendingDirs.emplace_back(joinPath(absoluteDir, segment), info.path); + } + } + dir.close(); + } + + for (const auto &pending : pendingDirs) { + auto st = collectDirectoryEntries(filesystem, pending.first, pending.second, true, entries); + if (!st.ok()) + return st; + } + + return {DbStatusCode::Ok, ""}; +} + +DbStatus writeFromPullCb( + fs::FS &filesystem, + const std::string &finalPath, + bool usePSRAMBuffers, + const ESPJsonDBFileOptions &opts, + const DbFileUploadPullCb &pullCb, + size_t &totalWritten +) { + totalWritten = 0; + if (!pullCb) { + return {DbStatusCode::InvalidArgument, "upload callback is required"}; + } + + const size_t chunkSize = opts.chunkSize < 32 ? 32 : opts.chunkSize; + JsonDbVector buffer{JsonDbAllocator(usePSRAMBuffers)}; + buffer.resize(chunkSize); + + const std::string parentDir = parentDirOf(finalPath); + const std::string tmpPath = finalPath + ".tmp"; + + FrLock fs(g_fsMutex); + if (!fsEnsureDir(filesystem, parentDir)) { + return {DbStatusCode::IoError, "mkdir file parent failed"}; + } + if (!opts.overwrite && filesystem.exists(finalPath.c_str())) { + return {DbStatusCode::AlreadyExists, "file already exists"}; + } + if (filesystem.exists(tmpPath.c_str())) { + filesystem.remove(tmpPath.c_str()); + } + + File file = filesystem.open(tmpPath.c_str(), FILE_WRITE); + if (!file) { + return {DbStatusCode::IoError, "open file for write failed"}; + } + WriteBufferingStream buffered(file, chunkSize); + + auto fail = [&](DbStatus st) { + buffered.flush(); + file.close(); + filesystem.remove(tmpPath.c_str()); + return st; + }; + + for (;;) { + size_t produced = 0; + bool eof = false; + auto st = pullCb(chunkSize, buffer.data(), produced, eof); + if (!st.ok()) + return fail(st); + if (produced > chunkSize) { + return fail({DbStatusCode::InvalidArgument, "upload callback produced too many bytes"}); + } + if (produced > 0) { + size_t written = buffered.write(buffer.data(), produced); + if (written != produced) { + return fail({DbStatusCode::IoError, "file write failed"}); + } + totalWritten += written; + } + if (eof) + break; + if (produced == 0) { + return fail( + {DbStatusCode::InvalidArgument, "upload callback produced no bytes without eof"} + ); + } + } + + buffered.flush(); + file.close(); + + if (opts.overwrite && filesystem.exists(finalPath.c_str()) && + !filesystem.remove(finalPath.c_str())) { + filesystem.remove(tmpPath.c_str()); + return {DbStatusCode::IoError, "remove old file failed"}; + } + if (!filesystem.rename(tmpPath.c_str(), finalPath.c_str())) { + filesystem.remove(tmpPath.c_str()); + return {DbStatusCode::IoError, "rename file failed"}; + } + + return {DbStatusCode::Ok, ""}; +} + +} // namespace + +FileStoreImpl::FileStoreImpl(DbRuntime &rt) + : _rt(&rt), uploadQueue(JsonDbAllocator(rt.cfg.usePSRAMBuffers)), + uploadJobs(std::less{}, UploadJobMap::allocator_type(rt.cfg.usePSRAMBuffers)), + terminalUploadOrder(JsonDbAllocator(rt.cfg.usePSRAMBuffers)) { +} + +DbStatus FileStoreImpl::normalizePath( + const std::string &rawRelativePath, std::string &normalized +) const { + normalized.clear(); + if (rawRelativePath.empty()) { + return {DbStatusCode::InvalidArgument, "file path is empty"}; + } + + std::string segment; + segment.reserve(rawRelativePath.size()); + + auto flushSegment = [&]() -> DbStatus { + if (segment.empty() || segment == ".") { + segment.clear(); + return {DbStatusCode::Ok, ""}; + } + if (segment == "..") { + segment.clear(); + return {DbStatusCode::InvalidArgument, "path traversal is not allowed"}; + } + if (segment.find(':') != std::string::npos) { + segment.clear(); + return {DbStatusCode::InvalidArgument, "invalid file path segment"}; + } + if (!normalized.empty()) { + normalized.push_back('/'); + } + normalized += segment; + segment.clear(); + return {DbStatusCode::Ok, ""}; + }; + + for (char c : rawRelativePath) { + if (c == '\\') + c = '/'; + if (c == '/') { + auto st = flushSegment(); + if (!st.ok()) + return st; + continue; + } + segment.push_back(c); + } + auto st = flushSegment(); + if (!st.ok()) + return st; + + if (normalized.empty()) { + return {DbStatusCode::InvalidArgument, "file path resolves to empty"}; + } + + return {DbStatusCode::Ok, ""}; +} + +DbStatus FileStoreImpl::writeFileStream( + const std::string &relativePath, + Stream &in, + size_t bytesToWrite, + const ESPJsonDBFileOptions &opts +) { + auto ready = _rt->ensureReady(); + if (!ready.ok()) + return _rt->recordStatus(ready); + + std::string normalized; + auto nst = normalizePath(relativePath, normalized); + if (!nst.ok()) + return _rt->recordStatus(nst); + + const std::string finalPath = joinPath(_rt->fileRootDir(), normalized); + size_t remaining = bytesToWrite; + DbFileUploadPullCb pullCb = + [&in, &remaining](size_t requested, uint8_t *buffer, size_t &produced, bool &eof) + -> DbStatus { + if (!buffer) { + return {DbStatusCode::InvalidArgument, "buffer is null"}; + } + if (remaining == 0) { + produced = 0; + eof = true; + return {DbStatusCode::Ok, ""}; + } + size_t want = std::min(requested, remaining); + produced = in.readBytes(reinterpret_cast(buffer), want); + if (produced == 0) { + eof = false; + return {DbStatusCode::IoError, "stream ended before expected size"}; + } + remaining -= produced; + eof = (remaining == 0); + return {DbStatusCode::Ok, ""}; + }; + + size_t totalWritten = 0; + auto st = writeFromPullCb(*_rt->fs, finalPath, _rt->cfg.usePSRAMBuffers, opts, pullCb, totalWritten); + if (!st.ok()) + return _rt->recordStatus(st); + if (totalWritten != bytesToWrite) { + return _rt->recordStatus({DbStatusCode::IoError, "written size mismatch"}); + } + + return _rt->recordStatus({DbStatusCode::Ok, ""}); +} + +DbStatus FileStoreImpl::writeFileStream( + const std::string &relativePath, + const DbFileUploadPullCb &pullCb, + const ESPJsonDBFileOptions &opts +) { + auto ready = _rt->ensureReady(); + if (!ready.ok()) + return _rt->recordStatus(ready); + + std::string normalized; + auto nst = normalizePath(relativePath, normalized); + if (!nst.ok()) + return _rt->recordStatus(nst); + + const std::string finalPath = joinPath(_rt->fileRootDir(), normalized); + size_t totalWritten = 0; + auto st = writeFromPullCb(*_rt->fs, finalPath, _rt->cfg.usePSRAMBuffers, opts, pullCb, totalWritten); + return _rt->recordStatus(st); +} + +DbStatus FileStoreImpl::writeFileFromPath( + const std::string &relativePath, + const std::string &sourceFsPath, + const ESPJsonDBFileOptions &opts +) { + if (sourceFsPath.empty()) { + return _rt->recordStatus({DbStatusCode::InvalidArgument, "source file path is empty"}); + } + auto ready = _rt->ensureReady(); + if (!ready.ok()) + return _rt->recordStatus(ready); + + File source; + { + FrLock fs(g_fsMutex); + source = _rt->fs->open(sourceFsPath.c_str(), FILE_READ); + } + if (!source) { + return _rt->recordStatus({DbStatusCode::NotFound, "source file not found"}); + } + + const size_t bytesToWrite = source.size(); + auto st = writeFileStream(relativePath, source, bytesToWrite, opts); + source.close(); + return st; +} + +DbStatus FileStoreImpl::writeFile( + const std::string &relativePath, const uint8_t *data, size_t size, bool overwrite +) { + if (size > 0 && data == nullptr) { + return _rt->recordStatus({DbStatusCode::InvalidArgument, "file data is null"}); + } + + auto ready = _rt->ensureReady(); + if (!ready.ok()) + return _rt->recordStatus(ready); + + std::string normalized; + auto nst = normalizePath(relativePath, normalized); + if (!nst.ok()) + return _rt->recordStatus(nst); + + const std::string finalPath = joinPath(_rt->fileRootDir(), normalized); + const std::string parentDir = parentDirOf(finalPath); + const std::string tmpPath = finalPath + ".tmp"; + + FrLock fs(g_fsMutex); + if (!fsEnsureDir(*_rt->fs, parentDir)) { + return _rt->recordStatus({DbStatusCode::IoError, "mkdir file parent failed"}); + } + if (!overwrite && _rt->fs->exists(finalPath.c_str())) { + return _rt->recordStatus({DbStatusCode::AlreadyExists, "file already exists"}); + } + if (_rt->fs->exists(tmpPath.c_str())) { + _rt->fs->remove(tmpPath.c_str()); + } + + File f = _rt->fs->open(tmpPath.c_str(), FILE_WRITE); + if (!f) { + return _rt->recordStatus({DbStatusCode::IoError, "open file for write failed"}); + } + WriteBufferingStream buffered(f, 256); + + size_t written = 0; + if (size > 0) { + written = buffered.write(data, size); + } + buffered.flush(); + f.close(); + if (written != size) { + _rt->fs->remove(tmpPath.c_str()); + return _rt->recordStatus({DbStatusCode::IoError, "file write failed"}); + } + + if (overwrite && _rt->fs->exists(finalPath.c_str()) && !_rt->fs->remove(finalPath.c_str())) { + _rt->fs->remove(tmpPath.c_str()); + return _rt->recordStatus({DbStatusCode::IoError, "remove old file failed"}); + } + if (!_rt->fs->rename(tmpPath.c_str(), finalPath.c_str())) { + _rt->fs->remove(tmpPath.c_str()); + return _rt->recordStatus({DbStatusCode::IoError, "rename file failed"}); + } + + return _rt->recordStatus({DbStatusCode::Ok, ""}); +} + +DbStatus +FileStoreImpl::writeTextFile(const std::string &relativePath, const std::string &text, bool overwrite) { + return writeFile( + relativePath, + reinterpret_cast(text.data()), + text.size(), + overwrite + ); +} + +DbResult +FileStoreImpl::readFileStream(const std::string &relativePath, Stream &out, size_t chunkSize) { + DbResult res{}; + auto ready = _rt->ensureReady(); + if (!ready.ok()) { + res.status = _rt->recordStatus(ready); + return res; + } + + std::string normalized; + auto nst = normalizePath(relativePath, normalized); + if (!nst.ok()) { + res.status = _rt->recordStatus(nst); + return res; + } + + if (chunkSize < 32) + chunkSize = 32; + JsonDbVector buffer{JsonDbAllocator(_rt->cfg.usePSRAMBuffers)}; + buffer.resize(chunkSize); + const std::string path = joinPath(_rt->fileRootDir(), normalized); + + FrLock fs(g_fsMutex); + File f = _rt->fs->open(path.c_str(), FILE_READ); + if (!f) { + res.status = _rt->recordStatus({DbStatusCode::NotFound, "file not found"}); + return res; + } + + size_t total = 0; + while (true) { + size_t readBytes = f.read(buffer.data(), buffer.size()); + if (readBytes == 0) + break; + size_t written = out.write(buffer.data(), readBytes); + if (written != readBytes) { + f.close(); + res.status = _rt->recordStatus({DbStatusCode::IoError, "output stream write failed"}); + return res; + } + total += written; + } + f.close(); + + res.status = _rt->recordStatus({DbStatusCode::Ok, ""}); + res.value = total; + return res; +} + +DbResult> FileStoreImpl::readFile(const std::string &relativePath) { + DbResult> res{}; + auto ready = _rt->ensureReady(); + if (!ready.ok()) { + res.status = _rt->recordStatus(ready); + return res; + } + + std::string normalized; + auto nst = normalizePath(relativePath, normalized); + if (!nst.ok()) { + res.status = _rt->recordStatus(nst); + return res; + } + + const std::string path = joinPath(_rt->fileRootDir(), normalized); + + FrLock fs(g_fsMutex); + File f = _rt->fs->open(path.c_str(), FILE_READ); + if (!f) { + res.status = _rt->recordStatus({DbStatusCode::NotFound, "file not found"}); + return res; + } + + size_t sz = f.size(); + res.value.resize(sz); + size_t readBytes = 0; + if (sz > 0) { + readBytes = f.read(res.value.data(), sz); + } + f.close(); + if (readBytes != sz) { + res.value.clear(); + res.status = _rt->recordStatus({DbStatusCode::IoError, "file read failed"}); + return res; + } + + res.status = _rt->recordStatus({DbStatusCode::Ok, ""}); + return res; +} + +DbResult FileStoreImpl::readTextFile(const std::string &relativePath) { + DbResult res{}; + auto fr = readFile(relativePath); + if (!fr.status.ok()) { + res.status = fr.status; + return res; + } + res.value.assign(fr.value.begin(), fr.value.end()); + res.status = _rt->recordStatus({DbStatusCode::Ok, ""}); + return res; +} + +DbResult FileStoreImpl::getFileInfo(const std::string &relativePath) { + DbResult res; + auto ready = _rt->ensureReady(); + if (!ready.ok()) { + res.status = _rt->recordStatus(ready); + return res; + } + + std::string normalized; + auto nst = normalizePath(relativePath, normalized); + if (!nst.ok()) { + res.status = _rt->recordStatus(nst); + return res; + } + + FileEntryInfo info; + const auto st = statFileEntry(*_rt->fs, joinPath(_rt->fileRootDir(), normalized), normalized, info); + if (!st.ok()) { + res.status = _rt->recordStatus(st); + return res; + } + + res.value["path"] = info.path.c_str(); + res.value["name"] = fileNameOf(info.path).c_str(); + res.value["exists"] = true; + res.value["isDirectory"] = info.isDirectory; + res.value["size"] = info.size; + res.status = _rt->recordStatus({DbStatusCode::Ok, ""}); + return res; +} + +DbResult FileStoreImpl::listFiles(const std::string &relativePrefix, bool recursive) { + DbResult res; + auto ready = _rt->ensureReady(); + if (!ready.ok()) { + res.status = _rt->recordStatus(ready); + return res; + } + + std::string normalizedPrefix; + if (!relativePrefix.empty()) { + auto nst = normalizePath(relativePrefix, normalizedPrefix); + if (!nst.ok()) { + res.status = _rt->recordStatus(nst); + return res; + } + } + + const std::string rootPath = _rt->fileRootDir(); + const std::string targetPath = + normalizedPrefix.empty() ? rootPath : joinPath(rootPath, normalizedPrefix); + + FileEntryInfo targetInfo; + auto st = statFileEntry(*_rt->fs, targetPath, normalizedPrefix, targetInfo); + if (!st.ok()) { + res.status = _rt->recordStatus(st); + return res; + } + + std::vector entries; + if (targetInfo.isDirectory) { + st = collectDirectoryEntries(*_rt->fs, targetPath, normalizedPrefix, recursive, entries); + if (!st.ok()) { + res.status = _rt->recordStatus(st); + return res; + } + } else { + entries.push_back(targetInfo); + } + + std::sort(entries.begin(), entries.end(), [](const FileEntryInfo &lhs, const FileEntryInfo &rhs) { + return lhs.path < rhs.path; + }); + + res.value["prefix"] = normalizedPrefix.c_str(); + res.value["recursive"] = recursive; + JsonArray entriesJson = res.value["entries"].to(); + for (const auto &entry : entries) { + appendFileInfoJson(entriesJson, entry); + } + + res.status = _rt->recordStatus({DbStatusCode::Ok, ""}); + return res; +} + +DbStatus FileStoreImpl::removeFile(const std::string &relativePath) { + auto ready = _rt->ensureReady(); + if (!ready.ok()) + return _rt->recordStatus(ready); + + std::string normalized; + auto nst = normalizePath(relativePath, normalized); + if (!nst.ok()) + return _rt->recordStatus(nst); + + const std::string path = joinPath(_rt->fileRootDir(), normalized); + FrLock fs(g_fsMutex); + if (!_rt->fs->exists(path.c_str())) { + return _rt->recordStatus({DbStatusCode::NotFound, "file not found"}); + } + if (!_rt->fs->remove(path.c_str())) { + return _rt->recordStatus({DbStatusCode::IoError, "file remove failed"}); + } + return _rt->recordStatus({DbStatusCode::Ok, ""}); +} + +DbResult FileStoreImpl::fileExists(const std::string &relativePath) { + DbResult res{}; + auto ready = _rt->ensureReady(); + if (!ready.ok()) { + res.status = _rt->recordStatus(ready); + return res; + } + + std::string normalized; + auto nst = normalizePath(relativePath, normalized); + if (!nst.ok()) { + res.status = _rt->recordStatus(nst); + return res; + } + + const std::string path = joinPath(_rt->fileRootDir(), normalized); + FrLock fs(g_fsMutex); + res.value = _rt->fs->exists(path.c_str()); + res.status = _rt->recordStatus({DbStatusCode::Ok, ""}); + return res; +} + +DbResult FileStoreImpl::fileSize(const std::string &relativePath) { + DbResult res{}; + auto ready = _rt->ensureReady(); + if (!ready.ok()) { + res.status = _rt->recordStatus(ready); + return res; + } + + std::string normalized; + auto nst = normalizePath(relativePath, normalized); + if (!nst.ok()) { + res.status = _rt->recordStatus(nst); + return res; + } + + const std::string path = joinPath(_rt->fileRootDir(), normalized); + FrLock fs(g_fsMutex); + File f = _rt->fs->open(path.c_str(), FILE_READ); + if (!f) { + res.status = _rt->recordStatus({DbStatusCode::NotFound, "file not found"}); + return res; + } + res.value = f.size(); + f.close(); + res.status = _rt->recordStatus({DbStatusCode::Ok, ""}); + return res; +} + +bool FileStoreImpl::isUploadTerminal(DbFileUploadState state) const { + return state == DbFileUploadState::Completed || state == DbFileUploadState::Failed || + state == DbFileUploadState::Cancelled; +} + +void FileStoreImpl::trackTerminalUploadLocked(const std::shared_ptr &job) { + if (!job || !isUploadTerminal(job->state) || job->terminalTracked) + return; + + job->terminalTracked = true; + terminalUploadOrder.push_back(job->id); + + while (terminalUploadOrder.size() > kMaxRetainedTerminalUploads) { + const uint32_t expiredId = terminalUploadOrder.front(); + terminalUploadOrder.pop_front(); + + auto it = uploadJobs.find(expiredId); + if (it == uploadJobs.end()) + continue; + if (!it->second || isUploadTerminal(it->second->state)) { + uploadJobs.erase(it); + } + } +} + +DbResult FileStoreImpl::writeFileStreamAsync( + const std::string &relativePath, + const DbFileUploadPullCb &pullCb, + const ESPJsonDBFileOptions &opts, + const DbFileUploadDoneCb &doneCb +) { + DbResult res{}; + auto ready = _rt->ensureReady(); + if (!ready.ok()) { + res.status = _rt->recordStatus(ready); + return res; + } + if (!pullCb) { + res.status = _rt->recordStatus({DbStatusCode::InvalidArgument, "upload callback is required"}); + return res; + } + + std::string normalized; + auto nst = normalizePath(relativePath, normalized); + if (!nst.ok()) { + res.status = _rt->recordStatus(nst); + return res; + } + + auto job = std::make_shared(); + job->relativePath = relativePath; + job->normalizedPath = normalized; + job->opts = opts; + if (job->opts.chunkSize < 32) + job->opts.chunkSize = 32; + job->pullCb = pullCb; + job->doneCb = doneCb; + + { + FrLock lk(_rt->mu); + job->id = nextUploadId++; + uploadJobs[job->id] = job; + uploadQueue.push_back(job->id); + startTaskUnlocked(); + if (taskHandle == nullptr) { + uploadQueue.erase( + std::remove(uploadQueue.begin(), uploadQueue.end(), job->id), + uploadQueue.end() + ); + uploadJobs.erase(job->id); + res.status = _rt->recordStatus({DbStatusCode::Busy, "upload worker start failed"}); + return res; + } + } + + res.status = _rt->recordStatus({DbStatusCode::Ok, ""}); + res.value = job->id; + return res; +} + +DbStatus FileStoreImpl::cancelUpload(uint32_t uploadId) { + DbFileUploadDoneCb doneCb; + DbStatus doneStatus{DbStatusCode::Busy, "upload cancelled"}; + size_t bytesWritten = 0; + bool triggerDone = false; + + { + FrLock lk(_rt->mu); + auto it = uploadJobs.find(uploadId); + if (it == uploadJobs.end()) { + return _rt->recordStatus({DbStatusCode::NotFound, "upload not found"}); + } + auto &job = it->second; + if (!job) { + return _rt->recordStatus({DbStatusCode::NotFound, "upload not found"}); + } + if (isUploadTerminal(job->state)) { + return _rt->recordStatus({DbStatusCode::Ok, ""}); + } + + job->cancelRequested = true; + if (job->state == DbFileUploadState::Queued) { + uploadQueue.erase( + std::remove(uploadQueue.begin(), uploadQueue.end(), uploadId), + uploadQueue.end() + ); + job->state = DbFileUploadState::Cancelled; + job->finalStatus = doneStatus; + trackTerminalUploadLocked(job); + doneCb = job->doneCb; + bytesWritten = job->bytesWritten; + triggerDone = true; + } + } + + if (triggerDone && doneCb) { + doneCb(uploadId, doneStatus, bytesWritten); + } + return _rt->recordStatus({DbStatusCode::Ok, ""}); +} + +DbResult FileStoreImpl::getUploadState(uint32_t uploadId) { + DbResult res{}; + auto ready = _rt->ensureReady(); + if (!ready.ok()) { + res.status = _rt->recordStatus(ready); + return res; + } + + { + FrLock lk(_rt->mu); + auto it = uploadJobs.find(uploadId); + if (it == uploadJobs.end() || !it->second) { + res.status = _rt->recordStatus({DbStatusCode::NotFound, "upload not found"}); + return res; + } + res.value = it->second->state; + } + + res.status = _rt->recordStatus({DbStatusCode::Ok, ""}); + return res; +} + +void FileStoreImpl::taskThunk(void *arg) { + auto *self = static_cast(arg); + self->taskLoop(); +} + +void FileStoreImpl::startTaskUnlocked() { + if (taskHandle != nullptr) + return; + stopRequested.store(false, std::memory_order_release); + taskExited.store(false, std::memory_order_release); + TaskHandle_t handle = nullptr; + if (_rt->createTask(taskThunk, "db.file.upload", this, handle)) { + taskHandle = handle; + } else { + taskExited.store(true, std::memory_order_release); + } +} + +void FileStoreImpl::stopTask(bool cancelPending) { + if (cancelPending) { + for (auto &kv : uploadJobs) { + auto &job = kv.second; + if (!job || isUploadTerminal(job->state)) + continue; + job->cancelRequested = true; + job->state = DbFileUploadState::Cancelled; + job->finalStatus = {DbStatusCode::Busy, "upload cancelled"}; + } + uploadQueue.clear(); + } + if (taskHandle) { + _rt->stopTask(taskHandle, stopRequested, taskExited); + } + uploadJobs.clear(); + terminalUploadOrder.clear(); + nextUploadId = 1; +} + +DbStatus FileStoreImpl::runUploadJob(const std::shared_ptr &job, size_t &bytesWritten) { + bytesWritten = 0; + if (!job || !job->pullCb) { + return {DbStatusCode::InvalidArgument, "upload callback is required"}; + } + + const size_t chunkSize = job->opts.chunkSize < 32 ? 32 : job->opts.chunkSize; + JsonDbVector buffer{JsonDbAllocator(_rt->cfg.usePSRAMBuffers)}; + buffer.resize(chunkSize); + + const std::string finalPath = joinPath(_rt->fileRootDir(), job->normalizedPath); + const std::string parentDir = parentDirOf(finalPath); + const std::string tmpPath = finalPath + ".tmp"; + + { + FrLock fs(g_fsMutex); + if (!fsEnsureDir(*_rt->fs, parentDir)) { + return {DbStatusCode::IoError, "mkdir file parent failed"}; + } + if (!job->opts.overwrite && _rt->fs->exists(finalPath.c_str())) { + return {DbStatusCode::AlreadyExists, "file already exists"}; + } + if (_rt->fs->exists(tmpPath.c_str())) { + _rt->fs->remove(tmpPath.c_str()); + } + } + + File f; + { + FrLock fs(g_fsMutex); + f = _rt->fs->open(tmpPath.c_str(), FILE_WRITE); + } + if (!f) { + return {DbStatusCode::IoError, "open file for write failed"}; + } + + auto cleanupTmp = [&]() { + FrLock fs(g_fsMutex); + f.close(); + if (_rt->fs->exists(tmpPath.c_str())) { + _rt->fs->remove(tmpPath.c_str()); + } + }; + + for (;;) { + bool cancelled = false; + { + FrLock lk(_rt->mu); + auto it = uploadJobs.find(job->id); + cancelled = (it == uploadJobs.end() || !it->second || it->second->cancelRequested); + } + if (cancelled) { + cleanupTmp(); + return {DbStatusCode::Busy, "upload cancelled"}; + } + + size_t produced = 0; + bool eof = false; + auto st = job->pullCb(chunkSize, buffer.data(), produced, eof); + if (!st.ok()) { + cleanupTmp(); + return st; + } + if (produced > chunkSize) { + cleanupTmp(); + return {DbStatusCode::InvalidArgument, "upload callback produced too many bytes"}; + } + + if (produced > 0) { + size_t written = 0; + { + FrLock fs(g_fsMutex); + written = f.write(buffer.data(), produced); + } + if (written != produced) { + cleanupTmp(); + return {DbStatusCode::IoError, "file write failed"}; + } + bytesWritten += written; + } + + if (eof) { + break; + } + if (produced == 0) { + vTaskDelay(pdMS_TO_TICKS(1)); + } + } + + { + FrLock fs(g_fsMutex); + f.flush(); + f.close(); + if (job->opts.overwrite && _rt->fs->exists(finalPath.c_str()) && + !_rt->fs->remove(finalPath.c_str())) { + if (_rt->fs->exists(tmpPath.c_str())) { + _rt->fs->remove(tmpPath.c_str()); + } + return {DbStatusCode::IoError, "remove old file failed"}; + } + if (!_rt->fs->rename(tmpPath.c_str(), finalPath.c_str())) { + if (_rt->fs->exists(tmpPath.c_str())) { + _rt->fs->remove(tmpPath.c_str()); + } + return {DbStatusCode::IoError, "rename file failed"}; + } + } + + return {DbStatusCode::Ok, ""}; +} + +void FileStoreImpl::taskLoop() { + while (!stopRequested.load(std::memory_order_acquire)) { + std::shared_ptr job; + { + FrLock lk(_rt->mu); + if (!uploadQueue.empty()) { + auto id = uploadQueue.front(); + uploadQueue.pop_front(); + auto it = uploadJobs.find(id); + if (it != uploadJobs.end()) { + job = it->second; + if (job && job->state == DbFileUploadState::Queued && !job->cancelRequested) { + job->state = DbFileUploadState::Running; + } + } + } + } + + if (!job) { + vTaskDelay(pdMS_TO_TICKS(20)); + continue; + } + + if (job->cancelRequested) { + DbFileUploadDoneCb doneCb; + { + FrLock lk(_rt->mu); + job->state = DbFileUploadState::Cancelled; + job->finalStatus = {DbStatusCode::Busy, "upload cancelled"}; + trackTerminalUploadLocked(job); + doneCb = job->doneCb; + } + if (doneCb) { + doneCb(job->id, job->finalStatus, job->bytesWritten); + } + continue; + } + + size_t bytesWritten = 0; + auto st = runUploadJob(job, bytesWritten); + + DbFileUploadDoneCb doneCb; + DbStatus finalStatus = st; + DbFileUploadState finalState = DbFileUploadState::Failed; + { + FrLock lk(_rt->mu); + auto it = uploadJobs.find(job->id); + if (it != uploadJobs.end() && it->second) { + auto &j = it->second; + j->bytesWritten = bytesWritten; + if (j->cancelRequested) { + finalState = DbFileUploadState::Cancelled; + finalStatus = {DbStatusCode::Busy, "upload cancelled"}; + } else if (st.ok()) { + finalState = DbFileUploadState::Completed; + } + j->state = finalState; + j->finalStatus = finalStatus; + trackTerminalUploadLocked(j); + doneCb = j->doneCb; + } + } + + if (!finalStatus.ok() && finalState != DbFileUploadState::Cancelled) { + _rt->recordStatus(finalStatus); + } + if (doneCb) { + doneCb(job->id, finalStatus, bytesWritten); + } + } + taskExited.store(true, std::memory_order_release); + vTaskDelete(nullptr); +} + +DbStatus FileStore::writeFileStream( + const std::string &relativePath, + Stream &in, + size_t bytesToWrite, + const ESPJsonDBFileOptions &opts +) { + if (!_impl) + return {DbStatusCode::NotInitialized, "file store not initialized"}; + return _impl->writeFileStream(relativePath, in, bytesToWrite, opts); +} + +DbStatus FileStore::writeFileStream( + const std::string &relativePath, + const DbFileUploadPullCb &pullCb, + const ESPJsonDBFileOptions &opts +) { + if (!_impl) + return {DbStatusCode::NotInitialized, "file store not initialized"}; + return _impl->writeFileStream(relativePath, pullCb, opts); +} + +DbStatus FileStore::writeFileFromPath( + const std::string &relativePath, const std::string &sourceFsPath, const ESPJsonDBFileOptions &opts +) { + if (!_impl) + return {DbStatusCode::NotInitialized, "file store not initialized"}; + return _impl->writeFileFromPath(relativePath, sourceFsPath, opts); +} + +DbStatus FileStore::writeFile( + const std::string &relativePath, const uint8_t *data, size_t size, bool overwrite +) { + if (!_impl) + return {DbStatusCode::NotInitialized, "file store not initialized"}; + return _impl->writeFile(relativePath, data, size, overwrite); +} + +DbStatus FileStore::writeTextFile( + const std::string &relativePath, const std::string &text, bool overwrite +) { + if (!_impl) + return {DbStatusCode::NotInitialized, "file store not initialized"}; + return _impl->writeTextFile(relativePath, text, overwrite); +} + +DbResult +FileStore::readFileStream(const std::string &relativePath, Stream &out, size_t chunkSize) { + if (!_impl) + return {{DbStatusCode::NotInitialized, "file store not initialized"}, 0}; + return _impl->readFileStream(relativePath, out, chunkSize); +} + +DbResult> FileStore::readFile(const std::string &relativePath) { + if (!_impl) + return {{DbStatusCode::NotInitialized, "file store not initialized"}, {}}; + return _impl->readFile(relativePath); +} + +DbResult FileStore::readTextFile(const std::string &relativePath) { + if (!_impl) + return {{DbStatusCode::NotInitialized, "file store not initialized"}, {}}; + return _impl->readTextFile(relativePath); +} + +DbResult FileStore::getFileInfo(const std::string &relativePath) { + if (!_impl) + return {{DbStatusCode::NotInitialized, "file store not initialized"}, JsonDocument{}}; + return _impl->getFileInfo(relativePath); +} + +DbResult FileStore::listFiles(const std::string &relativePrefix, bool recursive) { + if (!_impl) + return {{DbStatusCode::NotInitialized, "file store not initialized"}, JsonDocument{}}; + return _impl->listFiles(relativePrefix, recursive); +} + +DbStatus FileStore::removeFile(const std::string &relativePath) { + if (!_impl) + return {DbStatusCode::NotInitialized, "file store not initialized"}; + return _impl->removeFile(relativePath); +} + +DbResult FileStore::fileExists(const std::string &relativePath) { + if (!_impl) + return {{DbStatusCode::NotInitialized, "file store not initialized"}, false}; + return _impl->fileExists(relativePath); +} + +DbResult FileStore::fileSize(const std::string &relativePath) { + if (!_impl) + return {{DbStatusCode::NotInitialized, "file store not initialized"}, 0}; + return _impl->fileSize(relativePath); +} + +DbResult FileStore::writeFileStreamAsync( + const std::string &relativePath, + const DbFileUploadPullCb &pullCb, + const ESPJsonDBFileOptions &opts, + const DbFileUploadDoneCb &doneCb +) { + if (!_impl) + return {{DbStatusCode::NotInitialized, "file store not initialized"}, 0}; + return _impl->writeFileStreamAsync(relativePath, pullCb, opts, doneCb); +} + +DbStatus FileStore::cancelUpload(uint32_t uploadId) { + if (!_impl) + return {DbStatusCode::NotInitialized, "file store not initialized"}; + return _impl->cancelUpload(uploadId); +} + +DbResult FileStore::getUploadState(uint32_t uploadId) { + if (!_impl) + return {{DbStatusCode::NotInitialized, "file store not initialized"}, + DbFileUploadState::Failed}; + return _impl->getUploadState(uploadId); +} diff --git a/src/esp_jsondb/files/file_store.h b/src/esp_jsondb/files/file_store.h new file mode 100644 index 0000000..ef9c3c5 --- /dev/null +++ b/src/esp_jsondb/files/file_store.h @@ -0,0 +1,65 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include + +#include "../utils/dbTypes.h" + +struct FileStoreImpl; + +class FileStore { + public: + FileStore() = default; + explicit FileStore(FileStoreImpl *impl) : _impl(impl) { + } + void bind(FileStoreImpl *impl) { + _impl = impl; + } + + DbStatus writeFileStream( + const std::string &relativePath, + Stream &in, + size_t bytesToWrite, + const ESPJsonDBFileOptions &opts = {} + ); + DbStatus writeFileStream( + const std::string &relativePath, + const DbFileUploadPullCb &pullCb, + const ESPJsonDBFileOptions &opts = {} + ); + DbStatus writeFileFromPath( + const std::string &relativePath, + const std::string &sourceFsPath, + const ESPJsonDBFileOptions &opts = {} + ); + DbStatus writeFile( + const std::string &relativePath, const uint8_t *data, size_t size, bool overwrite = true + ); + DbStatus + writeTextFile(const std::string &relativePath, const std::string &text, bool overwrite = true); + DbResult + readFileStream(const std::string &relativePath, Stream &out, size_t chunkSize = 512); + DbResult> readFile(const std::string &relativePath); + DbResult readTextFile(const std::string &relativePath); + DbResult getFileInfo(const std::string &relativePath); + DbResult listFiles(const std::string &relativePrefix = "", bool recursive = true); + DbStatus removeFile(const std::string &relativePath); + DbResult fileExists(const std::string &relativePath); + DbResult fileSize(const std::string &relativePath); + DbResult writeFileStreamAsync( + const std::string &relativePath, + const DbFileUploadPullCb &pullCb, + const ESPJsonDBFileOptions &opts = {}, + const DbFileUploadDoneCb &doneCb = {} + ); + DbStatus cancelUpload(uint32_t uploadId); + DbResult getUploadState(uint32_t uploadId); + + private: + FileStoreImpl *_impl = nullptr; +}; diff --git a/src/esp_jsondb/files/file_store_impl.h b/src/esp_jsondb/files/file_store_impl.h new file mode 100644 index 0000000..41b87f6 --- /dev/null +++ b/src/esp_jsondb/files/file_store_impl.h @@ -0,0 +1,91 @@ +#pragma once + +#include + +#include +#include +#include + +#include "../utils/dbTypes.h" +#include "../utils/jsondb_allocator.h" + +struct DbRuntime; + +struct FileStoreImpl { + struct FileUploadJob { + uint32_t id = 0; + std::string relativePath; + std::string normalizedPath; + ESPJsonDBFileOptions opts{}; + DbFileUploadPullCb pullCb{}; + DbFileUploadDoneCb doneCb{}; + DbFileUploadState state = DbFileUploadState::Queued; + DbStatus finalStatus{DbStatusCode::Ok, ""}; + size_t bytesWritten = 0; + bool cancelRequested = false; + bool terminalTracked = false; + }; + + static constexpr size_t kMaxRetainedTerminalUploads = 64; + using UploadIdDeque = JsonDbDeque; + using UploadJobMap = JsonDbMap>; + + explicit FileStoreImpl(DbRuntime &rt); + + DbStatus normalizePath(const std::string &rawRelativePath, std::string &normalized) const; + + DbStatus writeFileStream( + const std::string &relativePath, + Stream &in, + size_t bytesToWrite, + const ESPJsonDBFileOptions &opts + ); + DbStatus writeFileStream( + const std::string &relativePath, + const DbFileUploadPullCb &pullCb, + const ESPJsonDBFileOptions &opts + ); + DbStatus writeFileFromPath( + const std::string &relativePath, + const std::string &sourceFsPath, + const ESPJsonDBFileOptions &opts + ); + DbStatus writeFile( + const std::string &relativePath, const uint8_t *data, size_t size, bool overwrite + ); + DbStatus writeTextFile(const std::string &relativePath, const std::string &text, bool overwrite); + DbResult readFileStream(const std::string &relativePath, Stream &out, size_t chunkSize); + DbResult> readFile(const std::string &relativePath); + DbResult readTextFile(const std::string &relativePath); + DbResult getFileInfo(const std::string &relativePath); + DbResult listFiles(const std::string &relativePrefix, bool recursive); + DbStatus removeFile(const std::string &relativePath); + DbResult fileExists(const std::string &relativePath); + DbResult fileSize(const std::string &relativePath); + DbResult writeFileStreamAsync( + const std::string &relativePath, + const DbFileUploadPullCb &pullCb, + const ESPJsonDBFileOptions &opts, + const DbFileUploadDoneCb &doneCb + ); + DbStatus cancelUpload(uint32_t uploadId); + DbResult getUploadState(uint32_t uploadId); + + void stopTask(bool cancelPending); + + bool isUploadTerminal(DbFileUploadState state) const; + void trackTerminalUploadLocked(const std::shared_ptr &job); + static void taskThunk(void *arg); + void taskLoop(); + void startTaskUnlocked(); + DbStatus runUploadJob(const std::shared_ptr &job, size_t &bytesWritten); + + DbRuntime *_rt = nullptr; + TaskHandle_t taskHandle = nullptr; + std::atomic stopRequested{false}; + std::atomic taskExited{true}; + uint32_t nextUploadId = 1; + UploadIdDeque uploadQueue; + UploadJobMap uploadJobs; + UploadIdDeque terminalUploadOrder; +}; diff --git a/src/esp_jsondb/storage/doc_codec.cpp b/src/esp_jsondb/storage/doc_codec.cpp new file mode 100644 index 0000000..7c8ea60 --- /dev/null +++ b/src/esp_jsondb/storage/doc_codec.cpp @@ -0,0 +1,177 @@ +#include "doc_codec.h" + +#include + +namespace { +constexpr uint8_t kMagic[4] = {'J', 'D', 'B', '2'}; +constexpr uint16_t kVersionWithDuplicatedFlags = 1; +constexpr uint16_t kVersionPrefixFlagsOnly = 2; +constexpr uint32_t kHeaderSizeV1 = 24 + 8 + 8 + 4 + 4 + 2; +constexpr uint32_t kHeaderSizeV2 = 24 + 8 + 8 + 4 + 4; +constexpr uint32_t kPrefixSize = 4 + 2 + 2 + 4 + 4; + +void appendU16(JsonDbVector &out, uint16_t value) { + out.push_back(static_cast(value & 0xFFu)); + out.push_back(static_cast((value >> 8) & 0xFFu)); +} + +void appendU32(JsonDbVector &out, uint32_t value) { + out.push_back(static_cast(value & 0xFFu)); + out.push_back(static_cast((value >> 8) & 0xFFu)); + out.push_back(static_cast((value >> 16) & 0xFFu)); + out.push_back(static_cast((value >> 24) & 0xFFu)); +} + +void appendU64(JsonDbVector &out, uint64_t value) { + for (uint8_t shift = 0; shift < 8; ++shift) { + out.push_back(static_cast((value >> (shift * 8U)) & 0xFFu)); + } +} + +bool readU16(const uint8_t *data, size_t size, size_t &offset, uint16_t &value) { + if (offset + 2 > size) + return false; + value = static_cast(data[offset]) | + (static_cast(data[offset + 1]) << 8); + offset += 2; + return true; +} + +bool readU32(const uint8_t *data, size_t size, size_t &offset, uint32_t &value) { + if (offset + 4 > size) + return false; + value = static_cast(data[offset]) | + (static_cast(data[offset + 1]) << 8) | + (static_cast(data[offset + 2]) << 16) | + (static_cast(data[offset + 3]) << 24); + offset += 4; + return true; +} + +bool readU64(const uint8_t *data, size_t size, size_t &offset, uint64_t &value) { + if (offset + 8 > size) + return false; + value = 0; + for (uint8_t shift = 0; shift < 8; ++shift) { + value |= static_cast(data[offset + shift]) << (shift * 8U); + } + offset += 8; + return true; +} +} // namespace + +uint32_t DocCodec::crc32(const uint8_t *data, size_t size) { + uint32_t crc = 0xFFFFFFFFu; + for (size_t i = 0; i < size; ++i) { + crc ^= static_cast(data[i]); + for (uint8_t bit = 0; bit < 8; ++bit) { + const uint32_t mask = static_cast(-(static_cast(crc & 1u))); + crc = (crc >> 1) ^ (0xEDB88320u & mask); + } + } + return ~crc; +} + +DbStatus DocCodec::encodeRecord( + const RecordHeader &header, const JsonDbVector &payload, JsonDbVector &out +) { + if (!header.id.valid()) { + return {DbStatusCode::InvalidArgument, "record id is invalid"}; + } + + const uint32_t payloadCrc = crc32(payload.data(), payload.size()); + out.clear(); + out.reserve(kPrefixSize + kHeaderSizeV2 + payload.size() + sizeof(uint32_t)); + out.insert(out.end(), kMagic, kMagic + sizeof(kMagic)); + appendU16(out, kVersionPrefixFlagsOnly); + appendU16(out, header.flags); + appendU32(out, kHeaderSizeV2); + appendU32(out, static_cast(payload.size())); + + for (size_t idx = 0; idx < DocId::kHexLength; ++idx) { + out.push_back(static_cast(header.id.c_str()[idx])); + } + appendU64(out, header.createdAtMs); + appendU64(out, header.updatedAtMs); + appendU32(out, header.revision); + appendU32(out, payloadCrc); + + out.insert(out.end(), payload.begin(), payload.end()); + appendU32(out, payloadCrc); + return {DbStatusCode::Ok, ""}; +} + +DbStatus DocCodec::decodeRecord( + const uint8_t *data, + size_t size, + RecordHeader &header, + JsonDbVector &payload, + bool usePSRAMBuffers +) { + payload = JsonDbVector(JsonDbAllocator(usePSRAMBuffers)); + if (!data || size < (kPrefixSize + kHeaderSizeV2 + sizeof(uint32_t))) { + return {DbStatusCode::CorruptionDetected, "record too small"}; + } + if (std::memcmp(data, kMagic, sizeof(kMagic)) != 0) { + return {DbStatusCode::CorruptionDetected, "record magic mismatch"}; + } + + size_t offset = sizeof(kMagic); + uint16_t version = 0; + uint16_t prefixFlags = 0; + uint32_t headerSize = 0; + uint32_t payloadSize = 0; + if (!readU16(data, size, offset, version) || !readU16(data, size, offset, prefixFlags) || + !readU32(data, size, offset, headerSize) || !readU32(data, size, offset, payloadSize)) { + return {DbStatusCode::CorruptionDetected, "record header truncated"}; + } + const bool isV1 = version == kVersionWithDuplicatedFlags && headerSize == kHeaderSizeV1; + const bool isV2 = version == kVersionPrefixFlagsOnly && headerSize == kHeaderSizeV2; + if (!isV1 && !isV2) { + return {DbStatusCode::SchemaMismatch, "unsupported record version"}; + } + if (size != (kPrefixSize + headerSize + payloadSize + sizeof(uint32_t))) { + return {DbStatusCode::CorruptionDetected, "record size mismatch"}; + } + + char idBuffer[DocId::kStorageLength]; + if (offset + DocId::kHexLength > size) { + return {DbStatusCode::CorruptionDetected, "record id truncated"}; + } + std::memcpy(idBuffer, data + offset, DocId::kHexLength); + idBuffer[DocId::kHexLength] = '\0'; + offset += DocId::kHexLength; + if (!header.id.assign(idBuffer)) { + return {DbStatusCode::CorruptionDetected, "record id invalid"}; + } + if (!readU64(data, size, offset, header.createdAtMs) || + !readU64(data, size, offset, header.updatedAtMs) || + !readU32(data, size, offset, header.revision) || + !readU32(data, size, offset, header.payloadCrc32)) { + return {DbStatusCode::CorruptionDetected, "record header payload truncated"}; + } + if (isV1) { + uint16_t legacyFlags = 0; + if (!readU16(data, size, offset, legacyFlags)) { + return {DbStatusCode::CorruptionDetected, "record header payload truncated"}; + } + } + + payload.resize(payloadSize); + if (payloadSize > 0) { + std::memcpy(payload.data(), data + offset, payloadSize); + } + offset += payloadSize; + + uint32_t trailerCrc = 0; + if (!readU32(data, size, offset, trailerCrc)) { + return {DbStatusCode::CorruptionDetected, "record crc truncated"}; + } + + const uint32_t actualCrc = crc32(payload.data(), payload.size()); + if (actualCrc != header.payloadCrc32 || actualCrc != trailerCrc) { + return {DbStatusCode::CorruptionDetected, "record crc mismatch"}; + } + header.flags = prefixFlags; + return {DbStatusCode::Ok, ""}; +} diff --git a/src/esp_jsondb/storage/doc_codec.h b/src/esp_jsondb/storage/doc_codec.h new file mode 100644 index 0000000..5faa491 --- /dev/null +++ b/src/esp_jsondb/storage/doc_codec.h @@ -0,0 +1,36 @@ +#pragma once + +#include + +#include +#include + +#include "../document/document.h" +#include "../utils/dbTypes.h" +#include "../utils/jsondb_allocator.h" + +struct RecordHeader { + DocId id; + uint64_t createdAtMs = 0; + uint64_t updatedAtMs = 0; + uint32_t revision = 0; + uint32_t payloadCrc32 = 0; + uint16_t flags = 0; +}; + +class DocCodec { + public: + static constexpr const char *kRecordExtension = ".jdb"; + + static uint32_t crc32(const uint8_t *data, size_t size); + static DbStatus encodeRecord( + const RecordHeader &header, const JsonDbVector &payload, JsonDbVector &out + ); + static DbStatus decodeRecord( + const uint8_t *data, + size_t size, + RecordHeader &header, + JsonDbVector &payload, + bool usePSRAMBuffers + ); +}; diff --git a/src/esp_jsondb/storage/record_store.cpp b/src/esp_jsondb/storage/record_store.cpp new file mode 100644 index 0000000..d7b01d7 --- /dev/null +++ b/src/esp_jsondb/storage/record_store.cpp @@ -0,0 +1,167 @@ +#include "record_store.h" + +#include + +#include + +#include "../storage/doc_codec.h" +#include "../utils/fr_mutex.h" +#include "../utils/fs_utils.h" +#include "../utils/jsondb_allocator.h" + +namespace { +std::string recordPathFor(const std::string &collectionDir, const std::string &id) { + return joinPath(collectionDir, id + DocCodec::kRecordExtension); +} +} // namespace + +DbStatus RecordStore::write(const std::string &collectionDir, const DocumentRecord &record) { + if (!_fs) { + return {DbStatusCode::IoError, "filesystem not ready"}; + } + if (!record.meta.id.valid()) { + return {DbStatusCode::InvalidArgument, "record id is invalid"}; + } + + JsonDbVector encoded{JsonDbAllocator(_usePSRAMBuffers)}; + RecordHeader header; + header.id = record.meta.id; + header.createdAtMs = record.meta.createdAtMs; + header.updatedAtMs = record.meta.updatedAtMs; + header.revision = record.meta.revision; + header.flags = record.meta.flags; + auto encodeStatus = DocCodec::encodeRecord(header, record.msgpack, encoded); + if (!encodeStatus.ok()) + return encodeStatus; + + const std::string finalPath = recordPathFor(collectionDir, record.meta.id.c_str()); + const std::string tmpPath = finalPath + ".tmp"; + + FrLock fs(g_fsMutex); + if (!fsEnsureDir(*_fs, collectionDir)) { + return {DbStatusCode::IoError, "mkdir failed"}; + } + File file = _fs->open(tmpPath.c_str(), FILE_WRITE); + if (!file) { + return {DbStatusCode::IoError, "open for write failed"}; + } + WriteBufferingStream buffered(file, 256); + const size_t written = buffered.write(encoded.data(), encoded.size()); + buffered.flush(); + file.close(); + if (written != encoded.size()) { + _fs->remove(tmpPath.c_str()); + return {DbStatusCode::IoError, "write failed"}; + } + if (_fs->exists(finalPath.c_str()) && !_fs->remove(finalPath.c_str())) { + _fs->remove(tmpPath.c_str()); + return {DbStatusCode::IoError, "replace old record failed"}; + } + if (!_fs->rename(tmpPath.c_str(), finalPath.c_str())) { + _fs->remove(tmpPath.c_str()); + return {DbStatusCode::IoError, "rename failed"}; + } + return {DbStatusCode::Ok, ""}; +} + +DbResult> +RecordStore::read(const std::string &collectionDir, const std::string &id) const { + DbResult> result{}; + if (!_fs) { + result.status = {DbStatusCode::IoError, "filesystem not ready"}; + return result; + } + + const std::string path = recordPathFor(collectionDir, id); + JsonDbVector encoded{JsonDbAllocator(_usePSRAMBuffers)}; + { + FrLock fs(g_fsMutex); + File file = _fs->open(path.c_str(), FILE_READ); + if (!file) { + result.status = {DbStatusCode::NotFound, "file not found"}; + return result; + } + const size_t size = file.size(); + encoded.resize(size); + const size_t readSize = file.read(encoded.data(), size); + file.close(); + if (readSize != size) { + result.status = {DbStatusCode::IoError, "read failed"}; + return result; + } + } + + auto record = + std::allocate_shared(JsonDbAllocator(_usePSRAMBuffers), _usePSRAMBuffers); + RecordHeader header; + auto decodeStatus = + DocCodec::decodeRecord(encoded.data(), encoded.size(), header, record->msgpack, _usePSRAMBuffers); + if (!decodeStatus.ok()) { + result.status = decodeStatus; + return result; + } + record->meta.id = header.id; + record->meta.createdAtMs = header.createdAtMs; + record->meta.updatedAtMs = header.updatedAtMs; + record->meta.revision = header.revision; + record->meta.flags = header.flags; + record->meta.dirty = false; + record->meta.removed = false; + result.status = {DbStatusCode::Ok, ""}; + result.value = std::move(record); + return result; +} + +JsonDbVector RecordStore::listIds(const std::string &collectionDir) const { + JsonDbVector ids{JsonDbAllocator(_usePSRAMBuffers)}; + if (!_fs) + return ids; + + FrLock fs(g_fsMutex); + if (!_fs->exists(collectionDir.c_str())) + return ids; + File dir = _fs->open(collectionDir.c_str()); + if (!dir || !dir.isDirectory()) { + if (dir) + dir.close(); + return ids; + } + for (File file = dir.openNextFile(); file; file = dir.openNextFile()) { + if (file.isDirectory()) { + file.close(); + continue; + } + String rawName = file.name(); + file.close(); + std::string name = rawName.c_str(); + const auto slash = name.find_last_of('/'); + if (slash != std::string::npos) + name = name.substr(slash + 1); + if (name.size() <= std::strlen(DocCodec::kRecordExtension)) + continue; + if (name.substr(name.size() - std::strlen(DocCodec::kRecordExtension)) != + DocCodec::kRecordExtension) + continue; + DocId parsed; + if (parsed.assign(name.substr(0, name.size() - std::strlen(DocCodec::kRecordExtension)))) { + ids.push_back(parsed); + } + } + dir.close(); + return ids; +} + +DbStatus RecordStore::remove(const std::string &collectionDir, const DocId &id) const { + if (!_fs) { + return {DbStatusCode::IoError, "filesystem not ready"}; + } + const std::string path = recordPathFor(collectionDir, id.c_str()); + FrLock fs(g_fsMutex); + if (!_fs->exists(path.c_str())) { + return {DbStatusCode::NotFound, "file not found"}; + } + if (!_fs->remove(path.c_str())) { + return {DbStatusCode::IoError, "remove failed"}; + } + return {DbStatusCode::Ok, ""}; +} diff --git a/src/esp_jsondb/storage/record_store.h b/src/esp_jsondb/storage/record_store.h new file mode 100644 index 0000000..b04f732 --- /dev/null +++ b/src/esp_jsondb/storage/record_store.h @@ -0,0 +1,26 @@ +#pragma once + +#include + +#include +#include + +#include "../document/document.h" +#include "../utils/dbTypes.h" +#include "../utils/jsondb_allocator.h" + +class RecordStore { + public: + RecordStore(fs::FS &fs, bool usePSRAMBuffers = false) : _fs(&fs), _usePSRAMBuffers(usePSRAMBuffers) { + } + + DbStatus write(const std::string &collectionDir, const DocumentRecord &record); + DbResult> + read(const std::string &collectionDir, const std::string &id) const; + JsonDbVector listIds(const std::string &collectionDir) const; + DbStatus remove(const std::string &collectionDir, const DocId &id) const; + + private: + fs::FS *_fs = nullptr; + bool _usePSRAMBuffers = false; +}; diff --git a/src/esp_jsondb/utils/dbTypes.h b/src/esp_jsondb/utils/dbTypes.h index 29d18a2..06d51c6 100644 --- a/src/esp_jsondb/utils/dbTypes.h +++ b/src/esp_jsondb/utils/dbTypes.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -18,9 +19,25 @@ enum class DbStatusCode : uint8_t { InvalidArgument, ValidationFailed, IoError, - Corrupted, + CorruptionDetected, Busy, - Unknown + NotInitialized, + Conflict, + Timeout, + Unsupported, + SchemaMismatch, + Unknown, + Corrupted = CorruptionDetected +}; + +enum class SnapshotMode : uint8_t { InMemoryConsistent = 0, OnDiskOnly }; + +enum class CollectionLoadPolicy : uint8_t { Eager = 0, Lazy, Delayed }; + +struct CollectionConfig { + CollectionLoadPolicy loadPolicy = CollectionLoadPolicy::Eager; + size_t maxDecodedViews = 0; + size_t maxRecordsInMemory = 0; }; struct ESPJsonDBConfig { @@ -29,15 +46,13 @@ struct ESPJsonDBConfig { UBaseType_t priority = 2; BaseType_t coreId = tskNO_AFFINITY; bool autosync = true; - bool coldSync = false; - bool cacheEnabled = true; // must remain true; false is rejected at runtime - std::vector delayedCollectionSyncArray; fs::FS *fs = nullptr; // optional external filesystem handle bool initFileSystem = true; bool formatOnFail = true; uint8_t maxOpenFiles = 10; const char *partitionLabel = "spiffs"; bool usePSRAMBuffers = false; // Prefer PSRAM for internal byte buffers when available + CollectionLoadPolicy defaultLoadPolicy = CollectionLoadPolicy::Eager; }; struct ESPJsonDBFileOptions { @@ -126,8 +141,13 @@ static constexpr const char *kDbStatusCodeDescriptions[] = { "Invalid argument", "Validation failed", "I/O error", - "Corrupted", + "Corruption detected", "Busy", + "Not initialized", + "Conflict", + "Timeout", + "Unsupported", + "Schema mismatch", "Unknown", }; diff --git a/src/esp_jsondb/utils/schema.h b/src/esp_jsondb/utils/schema.h index 176e9ae..b96484f 100644 --- a/src/esp_jsondb/utils/schema.h +++ b/src/esp_jsondb/utils/schema.h @@ -3,16 +3,16 @@ #include #include -#include -#include +#include #include #include #include +#include #include struct ValidationError { - bool valid; - const char *message; // lifetime must be static or managed externally + bool valid = true; + const char *message = ""; }; using ValidateFn = std::function; @@ -21,20 +21,138 @@ using PostLoadFn = std::function; enum class FieldType { String, - Int, + Int32, + Int64, + UInt32, + UInt64, Float, + Double, Bool, Object, Array, + Int = Int32, }; +struct EmptyObjectTag {}; +struct EmptyArrayTag {}; + +using JsonDefaultValue = std::variant< + std::monostate, + std::string, + int32_t, + int64_t, + uint32_t, + uint64_t, + float, + double, + bool, + EmptyObjectTag, + EmptyArrayTag>; + struct SchemaField { - const char *name; - FieldType type; - const char *defaultValue = nullptr; - bool unique = false; // enforce per-collection uniqueness when true + const char *name = nullptr; + FieldType type = FieldType::String; + bool required = false; + bool unique = false; + bool hasDefault = false; + JsonDefaultValue defaultValue{}; + + SchemaField() = default; + + SchemaField(const char *fieldName, FieldType fieldType) + : name(fieldName), type(fieldType) { + } + + SchemaField(const char *fieldName, FieldType fieldType, const char *defaultString) + : name(fieldName), type(fieldType), hasDefault(defaultString != nullptr), + defaultValue(defaultString ? JsonDefaultValue(std::string(defaultString)) + : JsonDefaultValue(std::monostate{})) { + } + + SchemaField(const char *fieldName, FieldType fieldType, const char *defaultString, bool uniqueFlag) + : name(fieldName), type(fieldType), unique(uniqueFlag), + hasDefault(defaultString != nullptr), + defaultValue(defaultString ? JsonDefaultValue(std::string(defaultString)) + : JsonDefaultValue(std::monostate{})) { + } + + SchemaField(const char *fieldName, FieldType fieldType, JsonDefaultValue value, bool uniqueFlag = false) + : name(fieldName), type(fieldType), unique(uniqueFlag), hasDefault(true), + defaultValue(std::move(value)) { + } }; +inline bool schemaFieldTypeMatches(JsonVariantConst value, FieldType type) { + switch (type) { + case FieldType::String: + return value.is() || value.is() || value.is(); + case FieldType::Int32: + return value.is() || value.is(); + case FieldType::Int64: + return value.is(); + case FieldType::UInt32: + return value.is(); + case FieldType::UInt64: + return value.is(); + case FieldType::Float: + return value.is(); + case FieldType::Double: + return value.is() || value.is(); + case FieldType::Bool: + return value.is(); + case FieldType::Object: + return value.is(); + case FieldType::Array: + return value.is(); + } + return false; +} + +inline void schemaApplyDefaultValue(JsonObject obj, const SchemaField &field) { + if (!field.name || !field.hasDefault) + return; + + if (std::holds_alternative(field.defaultValue)) { + obj[field.name] = std::get(field.defaultValue).c_str(); + return; + } + if (std::holds_alternative(field.defaultValue)) { + obj[field.name] = std::get(field.defaultValue); + return; + } + if (std::holds_alternative(field.defaultValue)) { + obj[field.name] = std::get(field.defaultValue); + return; + } + if (std::holds_alternative(field.defaultValue)) { + obj[field.name] = std::get(field.defaultValue); + return; + } + if (std::holds_alternative(field.defaultValue)) { + obj[field.name] = std::get(field.defaultValue); + return; + } + if (std::holds_alternative(field.defaultValue)) { + obj[field.name] = std::get(field.defaultValue); + return; + } + if (std::holds_alternative(field.defaultValue)) { + obj[field.name] = std::get(field.defaultValue); + return; + } + if (std::holds_alternative(field.defaultValue)) { + obj[field.name] = std::get(field.defaultValue); + return; + } + if (std::holds_alternative(field.defaultValue)) { + obj[field.name].to(); + return; + } + if (std::holds_alternative(field.defaultValue)) { + obj[field.name].to(); + } +} + struct Schema { std::vector fields; PreSaveFn preSave{}; @@ -48,73 +166,37 @@ struct Schema { } inline void applyDefaults(JsonObject obj) const { - for (const auto &f : fields) { - JsonVariant v = obj[f.name]; - if (!v && f.defaultValue) { - switch (f.type) { - case FieldType::String: - obj[f.name] = f.defaultValue; - break; - case FieldType::Int: - obj[f.name] = atoi(f.defaultValue); - break; - case FieldType::Float: - obj[f.name] = atof(f.defaultValue); - break; - case FieldType::Bool: - obj[f.name] = - (strcmp(f.defaultValue, "true") == 0 || strcmp(f.defaultValue, "1") == 0); - break; - case FieldType::Object: - obj[f.name].to(); - break; - case FieldType::Array: - obj[f.name].to(); - break; - } + for (const auto &field : fields) { + JsonVariant value = obj[field.name]; + if (value.isNull() && field.hasDefault) { + schemaApplyDefaultValue(obj, field); } } } - inline bool validateTypes(JsonObjectConst obj) const { - for (const auto &f : fields) { - JsonVariantConst v = obj[f.name]; - if (!v.isNull()) { - switch (f.type) { - case FieldType::String: - if (!v.is() && !v.is() && !v.is()) - return false; - break; - case FieldType::Int: - if (!v.is()) - return false; - break; - case FieldType::Float: - if (!v.is()) - return false; - break; - case FieldType::Bool: - if (!v.is()) - return false; - break; - case FieldType::Object: - if (!v.is()) - return false; - break; - case FieldType::Array: - if (!v.is()) - return false; - break; + inline ValidationError validateFields(JsonObjectConst obj) const { + for (const auto &field : fields) { + if (!field.name || !*field.name) + continue; + JsonVariantConst value = obj[field.name]; + if (value.isNull()) { + if (field.required) { + return {false, "schema: required field missing"}; } + continue; + } + if (!schemaFieldTypeMatches(value, field.type)) { + return {false, "schema: invalid type"}; } } - return true; + return {true, ""}; } inline ValidationError runPreSave(JsonObject &o) const { applyDefaults(o); - if (!validateTypes(o)) - return {false, "schema: invalid type"}; + auto fieldStatus = validateFields(o); + if (!fieldStatus.valid) + return fieldStatus; if (preSave) return preSave(o); if (validate) @@ -123,8 +205,9 @@ struct Schema { } inline ValidationError runValidate(const JsonObjectConst &o) const { - if (!validateTypes(o)) - return {false, "schema: invalid type"}; + auto fieldStatus = validateFields(o); + if (!fieldStatus.valid) + return fieldStatus; if (validate) return validate(o); return {true, ""}; diff --git a/src/esp_jsondb/utils/time_utils.h b/src/esp_jsondb/utils/time_utils.h index 7bb1c0c..5ac6623 100644 --- a/src/esp_jsondb/utils/time_utils.h +++ b/src/esp_jsondb/utils/time_utils.h @@ -1,12 +1,19 @@ #pragma once #include +#include #include -inline uint32_t nowUtcMs() { - time_t s = time(nullptr); - if (s < 0) - s = 0; - uint64_t ms = static_cast(s) * 1000ULL; - return static_cast(ms); +inline uint64_t nowUtcMs() { + timeval tv{}; + if (gettimeofday(&tv, nullptr) == 0) { + const uint64_t seconds = tv.tv_sec < 0 ? 0ULL : static_cast(tv.tv_sec); + const uint64_t micros = tv.tv_usec < 0 ? 0ULL : static_cast(tv.tv_usec); + return seconds * 1000ULL + (micros / 1000ULL); + } + + time_t seconds = time(nullptr); + if (seconds < 0) + seconds = 0; + return static_cast(seconds) * 1000ULL; } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt new file mode 100644 index 0000000..7abdc59 --- /dev/null +++ b/test/CMakeLists.txt @@ -0,0 +1,8 @@ +add_library(espjsondb_test_harness INTERFACE) + +target_include_directories( + espjsondb_test_harness + INTERFACE + ${CMAKE_CURRENT_LIST_DIR}/.. + ${CMAKE_CURRENT_LIST_DIR}/../src +) diff --git a/test/collectionTests.cpp b/test/collectionTests.cpp index a1e55ce..f1452e8 100644 --- a/test/collectionTests.cpp +++ b/test/collectionTests.cpp @@ -136,7 +136,7 @@ void DbTester::dropAllRemovesBaseDirTest() { return; } - auto fileWrite = db.writeTextFile("drop_all_cleanup/payload.txt", "cleanup"); + auto fileWrite = db.files().writeTextFile("drop_all_cleanup/payload.txt", "cleanup"); if (!fileWrite.ok()) { ESP_LOGE(DB_TESTER_TAG, "dropAllRemovesBaseDirTest writeTextFile failed: %s", fileWrite.message); return; @@ -171,3 +171,111 @@ void DbTester::dropAllRemovesBaseDirTest() { ESP_LOGI(DB_TESTER_TAG, "dropAll baseDir cleanup test passed"); } + +void DbTester::collectionBudgetEnforcementTest() { + ESPJsonDB budgetDb; + ESPJsonDBConfig cfg; + cfg.autosync = false; + + auto initStatus = budgetDb.init("/test_budget_db", cfg); + if (!initStatus.ok()) { + ESP_LOGE(DB_TESTER_TAG, "collectionBudgetEnforcementTest init failed: %s", initStatus.message); + return; + } + + (void)budgetDb.dropAll(); + const std::string collection = "budget_docs"; + + JsonDocument first; + first["index"] = 1; + JsonDocument second; + second["index"] = 2; + + auto firstCreate = budgetDb.create(collection, first.as()); + auto secondCreate = budgetDb.create(collection, second.as()); + if (!firstCreate.status.ok() || !secondCreate.status.ok()) { + ESP_LOGE(DB_TESTER_TAG, "collectionBudgetEnforcementTest seed create failed"); + budgetDb.deinit(); + return; + } + + auto syncStatus = budgetDb.syncNow(); + if (!syncStatus.ok()) { + ESP_LOGE(DB_TESTER_TAG, "collectionBudgetEnforcementTest sync failed: %s", syncStatus.message); + budgetDb.deinit(); + return; + } + budgetDb.deinit(); + + initStatus = budgetDb.init("/test_budget_db", cfg); + if (!initStatus.ok()) { + ESP_LOGE(DB_TESTER_TAG, "collectionBudgetEnforcementTest re-init failed: %s", initStatus.message); + return; + } + + auto residentCfgStatus = budgetDb.configureCollection( + collection, CollectionConfig{CollectionLoadPolicy::Lazy, 0, 1}); + if (!residentCfgStatus.ok()) { + ESP_LOGE( + DB_TESTER_TAG, + "collectionBudgetEnforcementTest resident configure failed: %s", + residentCfgStatus.message + ); + budgetDb.deinit(); + return; + } + + auto firstView = budgetDb.findById(collection, firstCreate.value); + if (!firstView.status.ok()) { + ESP_LOGE(DB_TESTER_TAG, "collectionBudgetEnforcementTest first lazy load failed: %s", firstView.status.message); + budgetDb.deinit(); + return; + } + auto secondView = budgetDb.findById(collection, secondCreate.value); + if (secondView.status.code != DbStatusCode::Busy) { + ESP_LOGE(DB_TESTER_TAG, "collectionBudgetEnforcementTest expected Busy for resident budget"); + budgetDb.deinit(); + return; + } + + budgetDb.deinit(); + initStatus = budgetDb.init("/test_budget_db", cfg); + if (!initStatus.ok()) { + ESP_LOGE(DB_TESTER_TAG, "collectionBudgetEnforcementTest third init failed: %s", initStatus.message); + return; + } + + auto decodeCfgStatus = budgetDb.configureCollection( + collection, CollectionConfig{CollectionLoadPolicy::Eager, 1, 0}); + if (!decodeCfgStatus.ok()) { + ESP_LOGE( + DB_TESTER_TAG, + "collectionBudgetEnforcementTest decode configure failed: %s", + decodeCfgStatus.message + ); + budgetDb.deinit(); + return; + } + + auto decodedFirst = budgetDb.findById(collection, firstCreate.value); + auto decodedSecond = budgetDb.findById(collection, secondCreate.value); + if (!decodedFirst.status.ok() || !decodedSecond.status.ok()) { + ESP_LOGE(DB_TESTER_TAG, "collectionBudgetEnforcementTest eager find failed"); + budgetDb.deinit(); + return; + } + if (decodedFirst.value.asObject().isNull()) { + ESP_LOGE(DB_TESTER_TAG, "collectionBudgetEnforcementTest first decode unexpectedly failed"); + budgetDb.deinit(); + return; + } + if (!decodedSecond.value.asObject().isNull() || + budgetDb.lastError().code != DbStatusCode::Busy) { + ESP_LOGE(DB_TESTER_TAG, "collectionBudgetEnforcementTest expected Busy for decode budget"); + budgetDb.deinit(); + return; + } + + budgetDb.deinit(); + ESP_LOGI(DB_TESTER_TAG, "Collection budget enforcement test passed"); +} diff --git a/test/dbTest.cpp b/test/dbTest.cpp index aef2eb0..69ede85 100644 --- a/test/dbTest.cpp +++ b/test/dbTest.cpp @@ -34,6 +34,9 @@ void DbTester::run() { refPopulateTest(); idLifecycleRoundTripTest(); snapshotRestoreIdLifecycleTest(); + docCodecCompatibilityTest(); + optimisticConflictTest(); + collectionBudgetEnforcementTest(); documentFileDeletionOnSyncTest(); fileStorageTest(); fileMetadataDiscoveryTest(); @@ -86,7 +89,7 @@ void DbTester::dbErrorHandler(const DbStatus &st) { } void DbTester::printDBDiag() { - JsonDocument diagDoc = db.getDiag(); + JsonDocument diagDoc = db.getDiagnostics(); ESP_LOGI(DB_TESTER_TAG, "DB Diagnostics"); serializeJsonPretty(diagDoc, Serial); ESP_LOGI(DB_TESTER_TAG, ""); diff --git a/test/dbTest.h b/test/dbTest.h index d999eee..c1d618b 100644 --- a/test/dbTest.h +++ b/test/dbTest.h @@ -24,6 +24,9 @@ class DbTester { void refPopulateTest(); void idLifecycleRoundTripTest(); void snapshotRestoreIdLifecycleTest(); + void docCodecCompatibilityTest(); + void optimisticConflictTest(); + void collectionBudgetEnforcementTest(); void documentFileDeletionOnSyncTest(); void fileStorageTest(); void fileMetadataDiscoveryTest(); diff --git a/test/delayedSyncTests.cpp b/test/delayedSyncTests.cpp index 8f082e2..db03138 100644 --- a/test/delayedSyncTests.cpp +++ b/test/delayedSyncTests.cpp @@ -53,14 +53,16 @@ void DbTester::delayedCollectionAccessBeforeAutosyncTickTest() { ESPJsonDBConfig cfg; cfg.autosync = true; cfg.intervalMs = 60000; // Keep autosync tick away so first access path is deterministic. - cfg.delayedCollectionSyncArray = {"delayed_access"}; + (void)db.configureCollection( + "delayed_access", CollectionConfig{CollectionLoadPolicy::Delayed, 0, 0} + ); auto initStatus = db.init("/test_db", cfg); if (!initStatus.ok()) { ESP_LOGE(DB_TESTER_TAG, "re-init failed for delayed access test: %s", initStatus.message); return; } - auto namesBefore = db.getAllCollectionName(); + auto namesBefore = db.listCollectionNames(); if (hasCollectionName(namesBefore, "delayed_access")) { ESP_LOGE(DB_TESTER_TAG, "delayed_access was preloaded but should have been deferred"); return; @@ -82,7 +84,7 @@ void DbTester::delayedCollectionAccessBeforeAutosyncTickTest() { return; } - auto namesAfter = db.getAllCollectionName(); + auto namesAfter = db.listCollectionNames(); if (!hasCollectionName(namesAfter, "delayed_access")) { ESP_LOGE(DB_TESTER_TAG, "delayed_access not tracked after first access load"); return; @@ -123,14 +125,16 @@ void DbTester::delayedCollectionSyncNowFallbackTest() { db.deinit(); ESPJsonDBConfig cfg; cfg.autosync = false; - cfg.delayedCollectionSyncArray = {"delayed_syncnow"}; + (void)db.configureCollection( + "delayed_syncnow", CollectionConfig{CollectionLoadPolicy::Delayed, 0, 0} + ); auto initStatus = db.init("/test_db", cfg); if (!initStatus.ok()) { ESP_LOGE(DB_TESTER_TAG, "re-init failed for syncNow fallback test: %s", initStatus.message); return; } - auto namesBefore = db.getAllCollectionName(); + auto namesBefore = db.listCollectionNames(); if (hasCollectionName(namesBefore, "delayed_syncnow")) { ESP_LOGE(DB_TESTER_TAG, "delayed_syncnow was preloaded but should wait for syncNow"); return; @@ -142,7 +146,7 @@ void DbTester::delayedCollectionSyncNowFallbackTest() { return; } - auto namesAfter = db.getAllCollectionName(); + auto namesAfter = db.listCollectionNames(); if (!hasCollectionName(namesAfter, "delayed_syncnow")) { ESP_LOGE(DB_TESTER_TAG, "delayed_syncnow was not loaded by syncNow fallback path"); return; @@ -187,7 +191,9 @@ void DbTester::delayedCollectionDropBeforeLoadTest() { db.deinit(); ESPJsonDBConfig cfg; cfg.autosync = false; - cfg.delayedCollectionSyncArray = {"drop_before_load"}; + (void)db.configureCollection( + "drop_before_load", CollectionConfig{CollectionLoadPolicy::Delayed, 0, 0} + ); auto initStatus = db.init("/test_db", cfg); if (!initStatus.ok()) { ESP_LOGE(DB_TESTER_TAG, "re-init failed for delayed drop test: %s", initStatus.message); @@ -246,14 +252,16 @@ void DbTester::delayedCollectionConfigNormalizationTest() { db.deinit(); ESPJsonDBConfig cfg; cfg.autosync = false; - cfg.delayedCollectionSyncArray = {"dup_collection", "dup_collection", "", "_files"}; + (void)db.configureCollection( + "dup_collection", CollectionConfig{CollectionLoadPolicy::Delayed, 0, 0} + ); auto initStatus = db.init("/test_db", cfg); if (!initStatus.ok()) { ESP_LOGE(DB_TESTER_TAG, "re-init failed for delayed normalization test: %s", initStatus.message); return; } - auto namesBefore = db.getAllCollectionName(); + auto namesBefore = db.listCollectionNames(); if (hasCollectionName(namesBefore, "dup_collection")) { ESP_LOGE(DB_TESTER_TAG, "dup_collection was preloaded but should have been deferred"); return; @@ -269,7 +277,7 @@ void DbTester::delayedCollectionConfigNormalizationTest() { return; } - auto namesAfter = db.getAllCollectionName(); + auto namesAfter = db.listCollectionNames(); if (!hasCollectionName(namesAfter, "dup_collection")) { ESP_LOGE(DB_TESTER_TAG, "dup_collection did not load after syncNow in normalization test"); return; diff --git a/test/documentTests.cpp b/test/documentTests.cpp index 26e3982..245b36d 100644 --- a/test/documentTests.cpp +++ b/test/documentTests.cpp @@ -1,6 +1,12 @@ #include "dbTest.h" +#include "../src/esp_jsondb/storage/doc_codec.h" +#include "../src/esp_jsondb/utils/objectId.h" namespace { +constexpr uint8_t kLegacyMagic[4] = {'J', 'D', 'B', '2'}; +constexpr uint16_t kLegacyVersion = 1; +constexpr uint32_t kLegacyHeaderSize = 24 + 8 + 8 + 4 + 4 + 2; + bool isHex24(const std::string &id) { if (id.size() != 24) return false; @@ -19,7 +25,48 @@ std::string collectionDirPath(const std::string &collection) { } std::string documentPath(const std::string &collection, const std::string &id) { - return collectionDirPath(collection) + "/" + id + ".mp"; + return collectionDirPath(collection) + "/" + id + ".jdb"; +} + +void appendU16(JsonDbVector &out, uint16_t value) { + out.push_back(static_cast(value & 0xFFu)); + out.push_back(static_cast((value >> 8) & 0xFFu)); +} + +void appendU32(JsonDbVector &out, uint32_t value) { + out.push_back(static_cast(value & 0xFFu)); + out.push_back(static_cast((value >> 8) & 0xFFu)); + out.push_back(static_cast((value >> 16) & 0xFFu)); + out.push_back(static_cast((value >> 24) & 0xFFu)); +} + +void appendU64(JsonDbVector &out, uint64_t value) { + for (uint8_t shift = 0; shift < 8; ++shift) { + out.push_back(static_cast((value >> (shift * 8U)) & 0xFFu)); + } +} + +JsonDbVector encodeLegacyRecord( + const RecordHeader &header, const JsonDbVector &payload +) { + JsonDbVector encoded{JsonDbAllocator(false)}; + const uint32_t payloadCrc = DocCodec::crc32(payload.data(), payload.size()); + encoded.insert(encoded.end(), kLegacyMagic, kLegacyMagic + sizeof(kLegacyMagic)); + appendU16(encoded, kLegacyVersion); + appendU16(encoded, header.flags); + appendU32(encoded, kLegacyHeaderSize); + appendU32(encoded, static_cast(payload.size())); + for (size_t idx = 0; idx < DocId::kHexLength; ++idx) { + encoded.push_back(static_cast(header.id.c_str()[idx])); + } + appendU64(encoded, header.createdAtMs); + appendU64(encoded, header.updatedAtMs); + appendU32(encoded, header.revision); + appendU32(encoded, payloadCrc); + appendU16(encoded, header.flags); + encoded.insert(encoded.end(), payload.begin(), payload.end()); + appendU32(encoded, payloadCrc); + return encoded; } } // namespace @@ -111,7 +158,7 @@ void DbTester::idLifecycleRoundTripTest() { return; } - auto updateStatus = db.updateById(collection, id, [](DocView &doc) { doc["value"] = 2; }); + auto updateStatus = db.updateById(collection, id, [](DocView &doc) { doc["value"].set(2); }); if (!updateStatus.ok()) { ESP_LOGE(DB_TESTER_TAG, "idLifecycleRoundTripTest updateById failed: %s", updateStatus.message); return; @@ -180,7 +227,7 @@ void DbTester::snapshotRestoreIdLifecycleTest() { return; } - auto snapshot = db.getSnapshot(); + auto snapshot = db.getSnapshot(SnapshotMode::OnDiskOnly); JsonArrayConst arr = snapshot["collections"][collection.c_str()].as(); if (arr.isNull() || arr.size() != ids.size()) { ESP_LOGE(DB_TESTER_TAG, "snapshotRestoreIdLifecycleTest snapshot collection missing"); @@ -192,6 +239,12 @@ void DbTester::snapshotRestoreIdLifecycleTest() { ESP_LOGE(DB_TESTER_TAG, "snapshotRestoreIdLifecycleTest snapshot _id format invalid"); return; } + JsonObjectConst meta = obj["_meta"].as(); + if (meta.isNull() || meta["createdAtMs"].isNull() || meta["updatedAtMs"].isNull() || + meta["revision"].isNull()) { + ESP_LOGE(DB_TESTER_TAG, "snapshotRestoreIdLifecycleTest snapshot metadata missing"); + return; + } } JsonDocument badSnapshot; @@ -233,11 +286,115 @@ void DbTester::snapshotRestoreIdLifecycleTest() { ESP_LOGE(DB_TESTER_TAG, "snapshotRestoreIdLifecycleTest restored payload mismatch"); return; } + if (findRes.value.meta().createdAtMs == 0 || findRes.value.meta().updatedAtMs == 0 || + findRes.value.meta().revision == 0) { + ESP_LOGE(DB_TESTER_TAG, "snapshotRestoreIdLifecycleTest restored metadata missing"); + return; + } } ESP_LOGI(DB_TESTER_TAG, "Snapshot restore ID lifecycle test passed"); } +void DbTester::docCodecCompatibilityTest() { + RecordHeader header; + header.id = ObjectId().toDocId(); + header.createdAtMs = 123456789ULL; + header.updatedAtMs = 123456999ULL; + header.revision = 7; + header.flags = 0x002A; + + JsonDocument payloadDoc; + payloadDoc["kind"] = "codec"; + payloadDoc["value"] = 42; + JsonDbVector payload{JsonDbAllocator(false)}; + const size_t payloadSize = measureMsgPack(payloadDoc); + payload.resize(payloadSize); + if (serializeMsgPack(payloadDoc, payload.data(), payload.size()) != payloadSize) { + ESP_LOGE(DB_TESTER_TAG, "docCodecCompatibilityTest payload serialization failed"); + return; + } + + JsonDbVector encoded{JsonDbAllocator(false)}; + auto encodeStatus = DocCodec::encodeRecord(header, payload, encoded); + if (!encodeStatus.ok()) { + ESP_LOGE(DB_TESTER_TAG, "docCodecCompatibilityTest encode failed: %s", encodeStatus.message); + return; + } + + RecordHeader decoded{}; + JsonDbVector decodedPayload{JsonDbAllocator(false)}; + auto decodeStatus = + DocCodec::decodeRecord(encoded.data(), encoded.size(), decoded, decodedPayload, false); + if (!decodeStatus.ok() || decoded.flags != header.flags || decoded.revision != header.revision || + decodedPayload != payload) { + ESP_LOGE(DB_TESTER_TAG, "docCodecCompatibilityTest v2 decode verification failed"); + return; + } + + auto legacy = encodeLegacyRecord(header, payload); + RecordHeader legacyDecoded{}; + JsonDbVector legacyPayload{JsonDbAllocator(false)}; + auto legacyStatus = + DocCodec::decodeRecord(legacy.data(), legacy.size(), legacyDecoded, legacyPayload, false); + if (!legacyStatus.ok() || legacyDecoded.flags != header.flags || legacyPayload != payload) { + ESP_LOGE(DB_TESTER_TAG, "docCodecCompatibilityTest legacy decode failed"); + return; + } + + legacy[4] = 0x63; + legacy[5] = 0x00; + RecordHeader badHeader{}; + JsonDbVector badPayload{JsonDbAllocator(false)}; + auto badStatus = + DocCodec::decodeRecord(legacy.data(), legacy.size(), badHeader, badPayload, false); + if (badStatus.code != DbStatusCode::SchemaMismatch) { + ESP_LOGE(DB_TESTER_TAG, "docCodecCompatibilityTest expected SchemaMismatch for bad version"); + return; + } + + ESP_LOGI(DB_TESTER_TAG, "DocCodec compatibility test passed"); +} + +void DbTester::optimisticConflictTest() { + auto clearStatus = db.dropAll(); + if (!clearStatus.ok()) { + ESP_LOGE(DB_TESTER_TAG, "optimisticConflictTest dropAll failed: %s", clearStatus.message); + return; + } + + JsonDocument seed; + seed["count"] = 0; + auto createRes = db.create("conflict_docs", seed.as()); + if (!createRes.status.ok()) { + ESP_LOGE(DB_TESTER_TAG, "optimisticConflictTest create failed: %s", createRes.status.message); + return; + } + + const std::string id = createRes.value; + auto updateStatus = db.updateById("conflict_docs", id, [&](DocView &doc) { + auto nested = + db.updateById("conflict_docs", id, [](DocView &inner) { inner["count"].set(2); }); + if (!nested.ok()) { + ESP_LOGE(DB_TESTER_TAG, "optimisticConflictTest nested update failed: %s", nested.message); + return; + } + doc["count"].set(1); + }); + if (updateStatus.code != DbStatusCode::Conflict) { + ESP_LOGE(DB_TESTER_TAG, "optimisticConflictTest expected Conflict, got %s", updateStatus.message); + return; + } + + auto findRes = db.findById("conflict_docs", id); + if (!findRes.status.ok() || findRes.value["count"].as() != 2) { + ESP_LOGE(DB_TESTER_TAG, "optimisticConflictTest final document state mismatch"); + return; + } + + ESP_LOGI(DB_TESTER_TAG, "Optimistic conflict test passed"); +} + void DbTester::documentFileDeletionOnSyncTest() { const std::string collection = "sync_delete_files"; auto clearStatus = db.dropAll(); diff --git a/test/fileTests.cpp b/test/fileTests.cpp index 27da76b..cf548ad 100644 --- a/test/fileTests.cpp +++ b/test/fileTests.cpp @@ -18,24 +18,24 @@ void DbTester::fileStorageTest() { const std::string textPath = "docs/sample.txt"; const std::string textPayload = "ESPJsonDB file storage test"; - auto textWrite = db.writeTextFile(textPath, textPayload, true); + auto textWrite = db.files().writeTextFile(textPath, textPayload, true); if (!textWrite.ok()) { ESP_LOGE(DB_TESTER_TAG, "writeTextFile failed: %s", textWrite.message); return; } - auto exists = db.fileExists(textPath); + auto exists = db.files().fileExists(textPath); if (!exists.status.ok() || !exists.value) { ESP_LOGE(DB_TESTER_TAG, "fileExists failed for text payload"); return; } - auto size = db.fileSize(textPath); + auto size = db.files().fileSize(textPath); if (!size.status.ok() || size.value != textPayload.size()) { ESP_LOGE(DB_TESTER_TAG, "fileSize failed for text payload"); return; } - auto textRead = db.readTextFile(textPath); + auto textRead = db.files().readTextFile(textPath); if (!textRead.status.ok() || textRead.value != textPayload) { ESP_LOGE(DB_TESTER_TAG, "readTextFile failed or content mismatch"); return; @@ -48,7 +48,7 @@ void DbTester::fileStorageTest() { } const std::string binPath = "bin/source.bin"; - auto binWrite = db.writeFile(binPath, binaryPayload.data(), binaryPayload.size(), true); + auto binWrite = db.files().writeFile(binPath, binaryPayload.data(), binaryPayload.size(), true); if (!binWrite.ok()) { ESP_LOGE(DB_TESTER_TAG, "writeFile failed: %s", binWrite.message); return; @@ -63,20 +63,20 @@ void DbTester::fileStorageTest() { ESPJsonDBFileOptions streamOpts; streamOpts.overwrite = true; streamOpts.chunkSize = 128; - auto streamWrite = db.writeFileStream("bin/copied.bin", src, src.size(), streamOpts); + auto streamWrite = db.files().writeFileStream("bin/copied.bin", src, src.size(), streamOpts); src.close(); if (!streamWrite.ok()) { ESP_LOGE(DB_TESTER_TAG, "writeFileStream failed: %s", streamWrite.message); return; } - auto copied = db.readFile("bin/copied.bin"); + auto copied = db.files().readFile("bin/copied.bin"); if (!copied.status.ok() || copied.value != binaryPayload) { ESP_LOGE(DB_TESTER_TAG, "readFile failed or binary mismatch"); return; } - auto copiedFromPath = db.writeFileFromPath( + auto copiedFromPath = db.files().writeFileFromPath( "bin/copied_from_path.bin", "/test_db/_files/bin/source.bin", streamOpts @@ -86,7 +86,7 @@ void DbTester::fileStorageTest() { return; } - auto copiedFromPathRead = db.readFile("bin/copied_from_path.bin"); + auto copiedFromPathRead = db.files().readFile("bin/copied_from_path.bin"); if (!copiedFromPathRead.status.ok() || copiedFromPathRead.value != binaryPayload) { ESP_LOGE(DB_TESTER_TAG, "writeFileFromPath readback mismatch"); return; @@ -118,26 +118,26 @@ void DbTester::fileStorageTest() { return {DbStatusCode::Ok, ""}; }; - auto syncCbWrite = db.writeFileStream("bin/copied_from_callback.bin", pullCb, streamOpts); + auto syncCbWrite = db.files().writeFileStream("bin/copied_from_callback.bin", pullCb, streamOpts); if (!syncCbWrite.ok()) { ESP_LOGE(DB_TESTER_TAG, "sync callback writeFileStream failed: %s", syncCbWrite.message); return; } - auto copiedFromCallback = db.readFile("bin/copied_from_callback.bin"); + auto copiedFromCallback = db.files().readFile("bin/copied_from_callback.bin"); if (!copiedFromCallback.status.ok() || copiedFromCallback.value != binaryPayload) { ESP_LOGE(DB_TESTER_TAG, "sync callback stream readback mismatch"); return; } auto sourceNotFound = - db.writeFileFromPath("bin/not_created.bin", "/test_db/_files/bin/missing.bin", streamOpts); + db.files().writeFileFromPath("bin/not_created.bin", "/test_db/_files/bin/missing.bin", streamOpts); if (sourceNotFound.ok() || sourceNotFound.code != DbStatusCode::NotFound) { ESP_LOGE(DB_TESTER_TAG, "writeFileFromPath missing-source check failed"); return; } - auto invalidProducer = db.writeFileStream( + auto invalidProducer = db.files().writeFileStream( "bin/invalid_callback.bin", [](size_t requested, uint8_t *, size_t &produced, bool &eof) -> DbStatus { produced = requested + 1; @@ -156,23 +156,23 @@ void DbTester::fileStorageTest() { ESP_LOGE(DB_TESTER_TAG, "Failed to open sink file for readFileStream"); return; } - auto streamRead = db.readFileStream(textPath, sink, 96); + auto streamRead = db.files().readFileStream(textPath, sink, 96); sink.close(); if (!streamRead.status.ok() || streamRead.value != textPayload.size()) { ESP_LOGE(DB_TESTER_TAG, "readFileStream failed: %s", streamRead.status.message); return; } - auto removeText = db.removeFile(textPath); + auto removeText = db.files().removeFile(textPath); if (!removeText.ok()) { ESP_LOGE(DB_TESTER_TAG, "removeFile failed: %s", removeText.message); return; } - (void)db.removeFile("bin/source.bin"); - (void)db.removeFile("bin/copied.bin"); - (void)db.removeFile("bin/copied_from_path.bin"); - (void)db.removeFile("bin/copied_from_callback.bin"); - (void)db.removeFile("bin/invalid_callback.bin"); + (void)db.files().removeFile("bin/source.bin"); + (void)db.files().removeFile("bin/copied.bin"); + (void)db.files().removeFile("bin/copied_from_path.bin"); + (void)db.files().removeFile("bin/copied_from_callback.bin"); + (void)db.files().removeFile("bin/invalid_callback.bin"); ESP_LOGI(DB_TESTER_TAG, "File storage test passed"); } @@ -184,9 +184,9 @@ void DbTester::fileMetadataDiscoveryTest() { return; } - auto topWrite = db.writeTextFile("top.txt", "root"); - auto infoWrite = db.writeTextFile("docs/info.txt", "metadata"); - auto nestedWrite = db.writeTextFile("docs/nested/child.txt", "nested"); + auto topWrite = db.files().writeTextFile("top.txt", "root"); + auto infoWrite = db.files().writeTextFile("docs/info.txt", "metadata"); + auto nestedWrite = db.files().writeTextFile("docs/nested/child.txt", "nested"); if (!topWrite.ok() || !infoWrite.ok() || !nestedWrite.ok()) { ESP_LOGE(DB_TESTER_TAG, "fileMetadataDiscoveryTest failed to seed files"); return; @@ -206,7 +206,7 @@ void DbTester::fileMetadataDiscoveryTest() { return; } - auto fileInfo = db.getFileInfo("docs/info.txt"); + auto fileInfo = db.files().getFileInfo("docs/info.txt"); if (!fileInfo.status.ok()) { ESP_LOGE(DB_TESTER_TAG, "fileMetadataDiscoveryTest getFileInfo(file) failed: %s", fileInfo.status.message); return; @@ -220,7 +220,7 @@ void DbTester::fileMetadataDiscoveryTest() { return; } - auto dirInfo = db.getFileInfo("docs"); + auto dirInfo = db.files().getFileInfo("docs"); if (!dirInfo.status.ok()) { ESP_LOGE(DB_TESTER_TAG, "fileMetadataDiscoveryTest getFileInfo(dir) failed: %s", dirInfo.status.message); return; @@ -234,7 +234,7 @@ void DbTester::fileMetadataDiscoveryTest() { return; } - auto topLevel = db.listFiles("", false); + auto topLevel = db.files().listFiles("", false); if (!topLevel.status.ok()) { ESP_LOGE(DB_TESTER_TAG, "fileMetadataDiscoveryTest top-level listFiles failed: %s", topLevel.status.message); return; @@ -250,7 +250,7 @@ void DbTester::fileMetadataDiscoveryTest() { return; } - auto docsRecursive = db.listFiles("docs", true); + auto docsRecursive = db.files().listFiles("docs", true); if (!docsRecursive.status.ok()) { ESP_LOGE(DB_TESTER_TAG, "fileMetadataDiscoveryTest recursive listFiles failed: %s", docsRecursive.status.message); return; @@ -270,33 +270,33 @@ void DbTester::fileMetadataDiscoveryTest() { } } - auto missingInfo = db.getFileInfo("missing.bin"); + auto missingInfo = db.files().getFileInfo("missing.bin"); if (missingInfo.status.code != DbStatusCode::NotFound) { ESP_LOGE(DB_TESTER_TAG, "fileMetadataDiscoveryTest expected NotFound for missing file info"); return; } - auto invalidInfo = db.getFileInfo("../escape"); + auto invalidInfo = db.files().getFileInfo("../escape"); if (invalidInfo.status.code != DbStatusCode::InvalidArgument) { ESP_LOGE(DB_TESTER_TAG, "fileMetadataDiscoveryTest expected InvalidArgument for invalid file info path"); return; } - auto invalidList = db.listFiles("../escape", true); + auto invalidList = db.files().listFiles("../escape", true); if (invalidList.status.code != DbStatusCode::InvalidArgument) { ESP_LOGE(DB_TESTER_TAG, "fileMetadataDiscoveryTest expected InvalidArgument for invalid listFiles path"); return; } - auto missingList = db.listFiles("missing_prefix", true); + auto missingList = db.files().listFiles("missing_prefix", true); if (missingList.status.code != DbStatusCode::NotFound) { ESP_LOGE(DB_TESTER_TAG, "fileMetadataDiscoveryTest expected NotFound for missing prefix"); return; } - (void)db.removeFile("top.txt"); - (void)db.removeFile("docs/info.txt"); - (void)db.removeFile("docs/nested/child.txt"); + (void)db.files().removeFile("top.txt"); + (void)db.files().removeFile("docs/info.txt"); + (void)db.files().removeFile("docs/nested/child.txt"); ESP_LOGI(DB_TESTER_TAG, "File metadata discovery test passed"); } @@ -348,7 +348,7 @@ void DbTester::asyncFileUploadTest() { ESPJsonDBFileOptions opts; opts.overwrite = true; opts.chunkSize = 96; - auto asyncRes = db.writeFileStreamAsync("async/payload.bin", pullCb, opts, doneCb); + auto asyncRes = db.files().writeFileStreamAsync("async/payload.bin", pullCb, opts, doneCb); if (!asyncRes.status.ok()) { ESP_LOGE(DB_TESTER_TAG, "writeFileStreamAsync failed: %s", asyncRes.status.message); return; @@ -361,7 +361,7 @@ void DbTester::asyncFileUploadTest() { } if (!done) { ESP_LOGE(DB_TESTER_TAG, "Async upload timed out"); - (void)db.cancelFileUpload(uploadId); + (void)db.files().cancelUpload(uploadId); return; } if (!doneOk || doneBytes != payload.size()) { @@ -369,19 +369,19 @@ void DbTester::asyncFileUploadTest() { return; } - auto stateRes = db.getFileUploadState(uploadId); + auto stateRes = db.files().getUploadState(uploadId); if (!stateRes.status.ok() || stateRes.value != DbFileUploadState::Completed) { ESP_LOGE(DB_TESTER_TAG, "Async upload state mismatch"); return; } - auto readBack = db.readFile("async/payload.bin"); + auto readBack = db.files().readFile("async/payload.bin"); if (!readBack.status.ok() || readBack.value != payload) { ESP_LOGE(DB_TESTER_TAG, "Async upload payload mismatch"); return; } - (void)db.removeFile("async/payload.bin"); + (void)db.files().removeFile("async/payload.bin"); ESP_LOGI(DB_TESTER_TAG, "Async file upload test passed"); } @@ -436,7 +436,7 @@ void DbTester::asyncFileUploadRetentionBoundTest() { }; const std::string path = "async/retention_" + std::to_string(i) + ".bin"; - auto asyncRes = db.writeFileStreamAsync(path, pullCb, opts, doneCb); + auto asyncRes = db.files().writeFileStreamAsync(path, pullCb, opts, doneCb); if (!asyncRes.status.ok()) { ESP_LOGE( DB_TESTER_TAG, @@ -461,7 +461,7 @@ void DbTester::asyncFileUploadRetentionBoundTest() { return; } - auto latestState = db.getFileUploadState(asyncRes.value); + auto latestState = db.files().getUploadState(asyncRes.value); if (!latestState.status.ok() || latestState.value != DbFileUploadState::Completed) { ESP_LOGE( DB_TESTER_TAG, @@ -471,16 +471,16 @@ void DbTester::asyncFileUploadRetentionBoundTest() { return; } - (void)db.removeFile(path); + (void)db.files().removeFile(path); } - auto oldestState = db.getFileUploadState(uploadIds.front()); + auto oldestState = db.files().getUploadState(uploadIds.front()); if (oldestState.status.code != DbStatusCode::NotFound) { ESP_LOGE(DB_TESTER_TAG, "Retention test expected oldest upload state to expire"); return; } - auto newestState = db.getFileUploadState(uploadIds.back()); + auto newestState = db.files().getUploadState(uploadIds.back()); if (!newestState.status.ok() || newestState.value != DbFileUploadState::Completed) { ESP_LOGE(DB_TESTER_TAG, "Retention test expected newest upload state to be retained"); return; @@ -549,7 +549,7 @@ void DbTester::asyncFileUploadQueueOrderTest() { } }; - auto asyncRes = db.writeFileStreamAsync(path, pullCb, opts, doneCb); + auto asyncRes = db.files().writeFileStreamAsync(path, pullCb, opts, doneCb); if (!asyncRes.status.ok()) { ESP_LOGE(DB_TESTER_TAG, "Queue order test upload start failed: %s", asyncRes.status.message); return; @@ -571,7 +571,7 @@ void DbTester::asyncFileUploadQueueOrderTest() { ESP_LOGE(DB_TESTER_TAG, "Queue order test completion order mismatch"); return; } - auto state = db.getFileUploadState(uploadIds[i]); + auto state = db.files().getUploadState(uploadIds[i]); if (!state.status.ok() || state.value != DbFileUploadState::Completed) { ESP_LOGE(DB_TESTER_TAG, "Queue order test upload state mismatch"); return; @@ -579,7 +579,7 @@ void DbTester::asyncFileUploadQueueOrderTest() { } for (const auto &path : uploadPaths) { - (void)db.removeFile(path); + (void)db.files().removeFile(path); } ESP_LOGI(DB_TESTER_TAG, "Async upload queue order test passed"); diff --git a/test/psramTests.cpp b/test/psramTests.cpp index 8484432..e95cac0 100644 --- a/test/psramTests.cpp +++ b/test/psramTests.cpp @@ -237,7 +237,7 @@ void DbTester::psramMemoryBenchmarkTest() { doneCount.fetch_add(1); }; - auto asyncRes = db.writeFileStreamAsync(path, pullCb, opts, doneCb); + auto asyncRes = db.files().writeFileStreamAsync(path, pullCb, opts, doneCb); if (!asyncRes.status.ok()) { ESP_LOGE(DB_TESTER_TAG, "benchmark async upload start failed: %s", asyncRes.status.message); return false; @@ -260,7 +260,7 @@ void DbTester::psramMemoryBenchmarkTest() { uploadStartInternal > minInternalFree ? (uploadStartInternal - minInternalFree) : 0; for (const auto &path : uploadPaths) { - (void)db.removeFile(path); + (void)db.files().removeFile(path); } (void)db.dropAll(); out.ok = true;