diff --git a/CMakeLists.txt b/CMakeLists.txt index 6b4238ec..f1c881ad 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,7 @@ endif() if(PJ_INSTALL_SDK) include(CMakePackageConfigHelpers) - set(PJ_PACKAGE_VERSION "0.8.0") + set(PJ_PACKAGE_VERSION "0.9.0") set(PJ_PACKAGE_CMAKE_DIR ${CMAKE_INSTALL_LIBDIR}/cmake/plotjuggler_sdk) install(EXPORT plotjuggler_sdkTargets diff --git a/conanfile.py b/conanfile.py index 82991499..ea93f537 100644 --- a/conanfile.py +++ b/conanfile.py @@ -6,7 +6,7 @@ plugin_sdk — umbrella for plugin authors (base + dialog SDK + parser SDK) plugin_host — umbrella for host loaders (data_source/parser/toolbox/dialog) -A consuming Conan recipe declares e.g. `plotjuggler_sdk/0.8.0` and then: +A consuming Conan recipe declares e.g. `plotjuggler_sdk/0.9.0` and then: find_package(plotjuggler_sdk REQUIRED COMPONENTS plugin_sdk) target_link_libraries(my_plugin PRIVATE plotjuggler_sdk::plugin_sdk) @@ -30,7 +30,7 @@ class PlotjugglerSdkConan(ConanFile): name = "plotjuggler_sdk" - version = "0.8.0" + version = "0.9.0" # Apache-2.0 covers the whole SDK (pj_base + pj_plugins). See LICENSE. license = "Apache-2.0" url = "https://github.com/PlotJuggler/plotjuggler_sdk" diff --git a/pj_base/CMakeLists.txt b/pj_base/CMakeLists.txt index d733fbee..190799e2 100644 --- a/pj_base/CMakeLists.txt +++ b/pj_base/CMakeLists.txt @@ -72,6 +72,7 @@ if(PJ_BUILD_TESTS) tests/expected_test.cpp tests/number_parse_test.cpp tests/plugin_data_api_test.cpp + tests/data_processors_api_test.cpp tests/settings_store_host_test.cpp tests/data_source_protocol_test.cpp # TODO: data_source_plugin_base_test.cpp and diff --git a/pj_base/include/pj_base/plugin_data_api.h b/pj_base/include/pj_base/plugin_data_api.h index d48648d2..71b17ca7 100644 --- a/pj_base/include/pj_base/plugin_data_api.h +++ b/pj_base/include/pj_base/plugin_data_api.h @@ -754,6 +754,85 @@ typedef struct { const PJ_colormap_registry_vtable_t* vtable; } PJ_colormap_registry_t; +/** + * Data-processor host service ("pj.data_processors.v1", protocol_version 1). + * + * Lets a toolbox plugin create catalog-resident "transform" nodes in the host + * by DATA only. Nothing executable crosses the boundary — only strings: + * + * - id : per-call stable key. The host namespaces it under the + * calling plugin. create() UPSERTS by id (idempotent Save). + * ids survive a session reload, so a plugin can re-edit its + * nodes after the host has replayed them from persisted + * recipes (see data_processor_config). + * - inputs : topic OR topic-field names ("pose/orientation" or + * "pose/orientation/x"); the host resolves them and exact-joins + * co-timestamped inputs. Field-level resolution is a host + * concern and needs no ABI change. + * - outputs : REQUIRED non-empty. This service creates named catalog + * topics (transforms). View-bound hidden-output "filters" are + * host-internal and are NOT created through this ABI. + * - script : the processor class source. Its first-line directive + * ("-- pj-script: ") selects the backend; an unsupported + * one is rejected by the host at create() time. BINARY-SAFE: + * script is a PJ_string_view_t {data,size}, NOT a C string, so + * it MAY carry a non-text blob — e.g. a future WASM module + * detected by the leading "\0asm" magic. Nothing on the path + * may treat it as NUL-terminated (no strlen). + * - params_json : forwarded VERBATIM to the backend's create(params); the host + * does not interpret its keys (translucent pass-through). Also + * a binary-safe PJ_string_view_t {data,size}. + * + * Per-plugin isolation: the host namespaces every id under the calling plugin, so + * list/remove/config only ever see or affect THAT plugin's nodes; one plugin can + * neither enumerate nor remove another's. + * + * FORWARD-COMPAT — the native door is WASM, not a C++ kernel. Because the script + * slot is a binary-safe blob the host owns and runs (today Luau; tomorrow a + * host-owned WASM/Python backend), a native processor needs NO new ABI: it ships + * as bytes through this same data-only surface, survives plugin unload (the host + * owns the payload + runtime), and is sandboxed. Adding a backend is purely + * additive. This is deliberately the OPPOSITE of crossing a C++ vtable across the + * DSO boundary (which dangles on unload). + * + * ABI-APPENDABLE: new slots may be added at the tail; struct_size gates read. + */ +typedef struct PJ_data_processors_host_vtable_t { + uint32_t protocol_version; // = 1 + uint32_t struct_size; // = sizeof(PJ_data_processors_host_vtable_t) + + /* [main-thread] Create or replace (upsert by id) a transform node. outputs + * MUST be non-empty. Transactional: on failure no partial node/topic is left + * behind. All string arguments are borrowed for the duration of the call. */ + bool (*create_data_processor)( + void* ctx, PJ_string_view_t id, const PJ_string_view_t* inputs, uint64_t input_count, + const PJ_string_view_t* outputs, uint64_t output_count, PJ_string_view_t script, PJ_string_view_t params_json, + PJ_error_t* out_error) PJ_NOEXCEPT; + + /* [main-thread] Remove a node by id. An unknown id is an error. */ + bool (*remove_data_processor)(void* ctx, PJ_string_view_t id, PJ_error_t* out_error) PJ_NOEXCEPT; + + /* [main-thread] Enumerate the ids of THIS plugin's live processors. + * Count-then-fill: pass capacity 0 to read *out_count, then call again with a + * buffer of that size. On success the first min(capacity, *out_count) entries + * of out_ids are filled and point into host storage valid only until the next + * call on this vtable. */ + bool (*list_data_processor_ids)( + void* ctx, PJ_string_view_t* out_ids, uint64_t capacity, uint64_t* out_count, PJ_error_t* out_error) PJ_NOEXCEPT; + + /* [main-thread] Read a node's full recipe as JSON + * {"inputs":[...],"outputs":[...],"params":{...}} for re-edit (e.g. after a + * session reload). *out_recipe_json is borrowed, valid only until the next + * call on this vtable. An unknown id is an error. */ + bool (*data_processor_config)( + void* ctx, PJ_string_view_t id, PJ_string_view_t* out_recipe_json, PJ_error_t* out_error) PJ_NOEXCEPT; +} PJ_data_processors_host_vtable_t; + +typedef struct { + void* ctx; + const PJ_data_processors_host_vtable_t* vtable; +} PJ_data_processors_host_t; + /** * Settings store service ("pj.settings.v1", protocol_version 1). * diff --git a/pj_base/include/pj_base/sdk/plugin_data_api.hpp b/pj_base/include/pj_base/sdk/plugin_data_api.hpp index 9ac88443..c6d6107c 100644 --- a/pj_base/include/pj_base/sdk/plugin_data_api.hpp +++ b/pj_base/include/pj_base/sdk/plugin_data_api.hpp @@ -1286,6 +1286,109 @@ class ColorMapRegistryView { PJ_colormap_registry_t registry_{}; }; +// --------------------------------------------------------------------------- +// DataProcessorsHostView — typed C++ view over PJ_data_processors_host_t +// --------------------------------------------------------------------------- + +/// C++ wrapper around PJ_data_processors_host_t for plugins that create +/// catalog-resident transform nodes in the host (see the C ABI doc-comment on +/// PJ_data_processors_host_vtable_t). Empty-constructible; `valid()` tells +/// whether the host exposed the service. Strings returned by `list()`/ +/// `recipeOf()` are copied into owned values, so they stay valid past the next +/// vtable call. +class DataProcessorsHostView { + public: + DataProcessorsHostView() = default; + explicit DataProcessorsHostView(PJ_data_processors_host_t host) : host_(host) {} + + [[nodiscard]] bool valid() const noexcept { + return host_.vtable != nullptr && host_.ctx != nullptr; + } + + /// Create or replace (upsert by id) a transform node. `outputs` must be + /// non-empty (the host rejects empty output lists). `params_json` is + /// forwarded verbatim to the script's create(params). + [[nodiscard]] Status createTransform( + std::string_view id, Span inputs, Span outputs, + std::string_view script, std::string_view params_json) const { + if (!valid() || host_.vtable->create_data_processor == nullptr) { + return unexpected("data processors host is not bound"); + } + if (outputs.empty()) { + return unexpected("data processors transform requires at least one output topic"); + } + std::vector in_abi; + in_abi.reserve(inputs.size()); + for (const auto& name : inputs) { + in_abi.push_back(toAbiString(name)); + } + std::vector out_abi; + out_abi.reserve(outputs.size()); + for (const auto& name : outputs) { + out_abi.push_back(toAbiString(name)); + } + PJ_error_t err{}; + if (!host_.vtable->create_data_processor( + host_.ctx, toAbiString(id), in_abi.data(), in_abi.size(), out_abi.data(), out_abi.size(), + toAbiString(script), toAbiString(params_json), &err)) { + return unexpected(errorToString(err)); + } + return okStatus(); + } + + /// Remove a previously created node by id. + [[nodiscard]] Status remove(std::string_view id) const { + if (!valid() || host_.vtable->remove_data_processor == nullptr) { + return unexpected("data processors host is not bound"); + } + PJ_error_t err{}; + if (!host_.vtable->remove_data_processor(host_.ctx, toAbiString(id), &err)) { + return unexpected(errorToString(err)); + } + return okStatus(); + } + + /// Enumerate the ids of this plugin's live processors (owned copies). + [[nodiscard]] Expected> list() const { + if (!valid() || host_.vtable->list_data_processor_ids == nullptr) { + return unexpected("data processors host is not bound"); + } + PJ_error_t err{}; + uint64_t count = 0; + if (!host_.vtable->list_data_processor_ids(host_.ctx, nullptr, 0, &count, &err)) { + return unexpected(errorToString(err)); + } + std::vector borrowed(count); + uint64_t filled = 0; + if (count != 0 && + !host_.vtable->list_data_processor_ids(host_.ctx, borrowed.data(), borrowed.size(), &filled, &err)) { + return unexpected(errorToString(err)); + } + std::vector ids; + ids.reserve(filled); + for (uint64_t i = 0; i < filled; ++i) { + ids.emplace_back(toStringView(borrowed[i])); + } + return ids; + } + + /// Read a node's full recipe JSON (owned copy) for re-edit. + [[nodiscard]] Expected recipeOf(std::string_view id) const { + if (!valid() || host_.vtable->data_processor_config == nullptr) { + return unexpected("data processors host is not bound"); + } + PJ_error_t err{}; + PJ_string_view_t recipe{}; + if (!host_.vtable->data_processor_config(host_.ctx, toAbiString(id), &recipe, &err)) { + return unexpected(errorToString(err)); + } + return std::string(toStringView(recipe)); + } + + private: + PJ_data_processors_host_t host_{}; +}; + // --------------------------------------------------------------------------- // SettingsView — typed C++ view over PJ_settings_store_t // --------------------------------------------------------------------------- diff --git a/pj_base/include/pj_base/sdk/service_traits.hpp b/pj_base/include/pj_base/sdk/service_traits.hpp index fffe99ca..4cc4eff0 100644 --- a/pj_base/include/pj_base/sdk/service_traits.hpp +++ b/pj_base/include/pj_base/sdk/service_traits.hpp @@ -158,6 +158,19 @@ struct ColorMapRegistryService { static_assert(detail::isValidServiceName(kName), "kName must match the pj naming rule"); }; +/// Optional service for plugins that create catalog-resident transform nodes in +/// the host by data (script + input/output names + params JSON). Nothing +/// executable crosses the boundary; the host owns execution. See the C ABI +/// doc-comment on PJ_data_processors_host_vtable_t. +struct DataProcessorsHostService { + static constexpr const char* kName = "pj.data_processors.v1"; + static constexpr uint32_t kMinVersion = 1; + using Raw = PJ_data_processors_host_t; + using Vtable = PJ_data_processors_host_vtable_t; + using View = DataProcessorsHostView; + static_assert(detail::isValidServiceName(kName), "kName must match the pj naming rule"); +}; + /// Optional QSettings-like key/value persistence exposed to any plugin family. /// Host-backed (QSettings in the GUI app, JSON in a headless host); keys are /// namespaced per plugin by the host. diff --git a/pj_base/tests/data_processors_api_test.cpp b/pj_base/tests/data_processors_api_test.cpp new file mode 100644 index 00000000..b77deda6 --- /dev/null +++ b/pj_base/tests/data_processors_api_test.cpp @@ -0,0 +1,228 @@ +// Copyright 2026 Davide Faconti +// SPDX-License-Identifier: Apache-2.0 + +#include + +#include +#include +#include +#include + +#include "pj_base/plugin_data_api.h" +#include "pj_base/sdk/plugin_data_api.hpp" +#include "pj_base/span.hpp" + +namespace PJ { +namespace { + +// Fake host: records the last create/remove call and serves list/config from +// host-owned storage (so the borrowed-string lifetime contract can be tested). +struct FakeDataProcessorsHost { + bool create_called = false; + bool create_should_fail = false; + std::string last_id; + std::vector last_inputs; + std::vector last_outputs; + std::string last_script; + std::string last_params; + + std::string removed_id; + + std::vector stored_ids; // host storage the listed views point into + std::string recipe_storage; // host storage the recipe view points into +}; + +bool dpCreate( + void* ctx, PJ_string_view_t id, const PJ_string_view_t* inputs, uint64_t input_count, + const PJ_string_view_t* outputs, uint64_t output_count, PJ_string_view_t script, PJ_string_view_t params_json, + PJ_error_t* out_error) noexcept { + auto* self = static_cast(ctx); + if (self->create_should_fail) { + if (out_error != nullptr) { + sdk::fillError(out_error, 1, "data_processors", "create boom"); + } + return false; + } + self->create_called = true; + self->last_id = std::string(sdk::toStringView(id)); + self->last_inputs.clear(); + for (uint64_t i = 0; i < input_count; ++i) { + self->last_inputs.emplace_back(sdk::toStringView(inputs[i])); + } + self->last_outputs.clear(); + for (uint64_t i = 0; i < output_count; ++i) { + self->last_outputs.emplace_back(sdk::toStringView(outputs[i])); + } + self->last_script = std::string(sdk::toStringView(script)); + self->last_params = std::string(sdk::toStringView(params_json)); + return true; +} + +bool dpRemove(void* ctx, PJ_string_view_t id, PJ_error_t* /*out_error*/) noexcept { + static_cast(ctx)->removed_id = std::string(sdk::toStringView(id)); + return true; +} + +bool dpList( + void* ctx, PJ_string_view_t* out_ids, uint64_t capacity, uint64_t* out_count, PJ_error_t* /*out_error*/) noexcept { + auto* self = static_cast(ctx); + *out_count = self->stored_ids.size(); + const uint64_t n = std::min(capacity, self->stored_ids.size()); + for (uint64_t i = 0; i < n; ++i) { + out_ids[i] = sdk::toAbiString(self->stored_ids[i]); + } + return true; +} + +bool dpConfig( + void* ctx, PJ_string_view_t /*id*/, PJ_string_view_t* out_recipe_json, PJ_error_t* /*out_error*/) noexcept { + out_recipe_json[0] = sdk::toAbiString(static_cast(ctx)->recipe_storage); + return true; +} + +PJ_data_processors_host_vtable_t makeVtable() { + return PJ_data_processors_host_vtable_t{ + .protocol_version = 1, + .struct_size = sizeof(PJ_data_processors_host_vtable_t), + .create_data_processor = dpCreate, + .remove_data_processor = dpRemove, + .list_data_processor_ids = dpList, + .data_processor_config = dpConfig, + }; +} + +TEST(DataProcessorsApiTest, CreateForwardsAllArgsIntact) { + FakeDataProcessorsHost host; + const auto vtable = makeVtable(); + sdk::DataProcessorsHostView view(PJ_data_processors_host_t{.ctx = &host, .vtable = &vtable}); + + const std::string_view inputs[] = {"pose/orientation/x", "pose/orientation/y"}; + const std::string_view outputs[] = {"pose/rpy/roll", "pose/rpy/pitch"}; + auto status = view.createTransform( + "quat_rpy", PJ::Span(inputs), PJ::Span(outputs), + "-- pj-script: lua\nreturn {}", R"({"window":10})"); + + ASSERT_TRUE(status) << status.error(); + EXPECT_TRUE(host.create_called); + EXPECT_EQ(host.last_id, "quat_rpy"); + ASSERT_EQ(host.last_inputs.size(), 2u); + EXPECT_EQ(host.last_inputs[0], "pose/orientation/x"); + EXPECT_EQ(host.last_inputs[1], "pose/orientation/y"); + ASSERT_EQ(host.last_outputs.size(), 2u); + EXPECT_EQ(host.last_outputs[0], "pose/rpy/roll"); + EXPECT_EQ(host.last_outputs[1], "pose/rpy/pitch"); + EXPECT_EQ(host.last_script, "-- pj-script: lua\nreturn {}"); + EXPECT_EQ(host.last_params, R"({"window":10})"); +} + +TEST(DataProcessorsApiTest, CreateFailureSurfacesError) { + FakeDataProcessorsHost host; + host.create_should_fail = true; + const auto vtable = makeVtable(); + sdk::DataProcessorsHostView view(PJ_data_processors_host_t{.ctx = &host, .vtable = &vtable}); + + const std::string_view outputs[] = {"out"}; + auto status = view.createTransform( + "x", PJ::Span{}, PJ::Span(outputs), "s", "{}"); + + EXPECT_FALSE(status); + EXPECT_NE(status.error().find("create boom"), std::string::npos); + EXPECT_FALSE(host.create_called); +} + +TEST(DataProcessorsApiTest, RemoveForwardsId) { + FakeDataProcessorsHost host; + const auto vtable = makeVtable(); + sdk::DataProcessorsHostView view(PJ_data_processors_host_t{.ctx = &host, .vtable = &vtable}); + + ASSERT_TRUE(view.remove("quat_rpy")); + EXPECT_EQ(host.removed_id, "quat_rpy"); +} + +TEST(DataProcessorsApiTest, ListCountThenFillReturnsOwnedCopies) { + FakeDataProcessorsHost host; + host.stored_ids = {"alpha", "beta", "gamma"}; + const auto vtable = makeVtable(); + sdk::DataProcessorsHostView view(PJ_data_processors_host_t{.ctx = &host, .vtable = &vtable}); + + auto ids = view.list(); + ASSERT_TRUE(ids) << ids.error(); + ASSERT_EQ(ids->size(), 3u); + EXPECT_EQ((*ids)[0], "alpha"); + EXPECT_EQ((*ids)[2], "gamma"); + + // Owned copies: mutating host storage must not change the returned vector. + host.stored_ids[0] = "clobbered"; + EXPECT_EQ((*ids)[0], "alpha"); +} + +TEST(DataProcessorsApiTest, RecipeOfCopiesBorrowedJson) { + FakeDataProcessorsHost host; + host.recipe_storage = R"({"inputs":["a"],"outputs":["b"],"params":{}})"; + const auto vtable = makeVtable(); + sdk::DataProcessorsHostView view(PJ_data_processors_host_t{.ctx = &host, .vtable = &vtable}); + + auto recipe = view.recipeOf("id"); + ASSERT_TRUE(recipe) << recipe.error(); + const std::string expected = host.recipe_storage; + + // The returned string is an owned copy: clobbering the host buffer is invisible. + host.recipe_storage = "CLOBBERED"; + EXPECT_EQ(*recipe, expected); +} + +TEST(DataProcessorsApiTest, UnboundViewReportsNotBound) { + sdk::DataProcessorsHostView view; // default-constructed = not bound + EXPECT_FALSE(view.valid()); + + const std::string_view outputs[] = {"out"}; + auto status = view.createTransform( + "x", PJ::Span{}, PJ::Span(outputs), "s", "{}"); + + EXPECT_FALSE(status); + EXPECT_NE(status.error().find("not bound"), std::string::npos); +} + +// The view fails fast on an empty output list: this service creates NAMED catalog +// topics (transforms), so >= 1 output is mandatory. The guard lives in the view so +// a misuse never reaches the vtable (the host enforces it authoritatively too). +TEST(DataProcessorsApiTest, CreateRejectsEmptyOutputs) { + FakeDataProcessorsHost host; + const auto vtable = makeVtable(); + sdk::DataProcessorsHostView view(PJ_data_processors_host_t{.ctx = &host, .vtable = &vtable}); + + auto status = view.createTransform( + "x", PJ::Span{}, PJ::Span{}, "-- pj-script: lua\nreturn {}", + "{}"); + + EXPECT_FALSE(status); + EXPECT_NE(status.error().find("output"), std::string::npos); + EXPECT_FALSE(host.create_called); // rejected before the ABI call +} + +// Regression guard: `script` / `params_json` are PJ_string_view_t {data, size} -- +// binary-safe, NOT NUL-terminated. A payload carrying embedded NULs and the WASM +// "\0asm" magic round-trips byte-for-byte. This keeps the future WASM/Python +// backend door open: a script slot may carry a binary module, so nothing on the +// path may "optimize" the marshalling to strlen. +TEST(DataProcessorsApiTest, BinarySafePayloadRoundTrips) { + FakeDataProcessorsHost host; + const auto vtable = makeVtable(); + sdk::DataProcessorsHostView view(PJ_data_processors_host_t{.ctx = &host, .vtable = &vtable}); + + const char wasm_bytes[] = {'\0', 'a', 's', 'm', '\x01', '\0', '\0', '\0', 'X', 'Y'}; + const std::string blob(wasm_bytes, sizeof(wasm_bytes)); // 10 bytes, 3 embedded NULs + ASSERT_EQ(blob.size(), 10u); + + const std::string_view outputs[] = {"spectrum"}; + auto status = view.createTransform( + "fft", PJ::Span{}, PJ::Span(outputs), + std::string_view(blob.data(), blob.size()), "{}"); + + ASSERT_TRUE(status) << status.error(); + EXPECT_EQ(host.last_script.size(), blob.size()); // not truncated at the first NUL + EXPECT_EQ(host.last_script, blob); +} + +} // namespace +} // namespace PJ diff --git a/pj_plugins/docs/ARCHITECTURE.md b/pj_plugins/docs/ARCHITECTURE.md index b0964175..a8f874bb 100644 --- a/pj_plugins/docs/ARCHITECTURE.md +++ b/pj_plugins/docs/ARCHITECTURE.md @@ -143,7 +143,16 @@ service registry, error out-params, and typed borrowed-dialog patterns): (write hosts, runtime hosts, colormap, settings, etc.) under canonical reverse-DNS-style names (e.g. `"pj.source_write.v1"`, `"pj.runtime.v1"`, `"pj.toolbox_runtime.v1"`, `"pj.colormap.v1"`, - `"pj.settings.v1"`). Plugins acquire only the services they use. + `"pj.settings.v1"`, `"pj.data_processors.v1"`). Plugins acquire only the + services they use. `"pj.data_processors.v1"` (optional) lets a toolbox create + catalog-resident transform nodes in the host by data — a script plus + input/output names and a params JSON blob; nothing executable crosses the + boundary (the host owns execution). The script payload is **binary-safe** + (`PJ_string_view_t {data,size}`), so the native "door" is WASM bytes through + this same data-only surface (a future host-owned WASM/Python backend is purely + additive and survives plugin unload) — deliberately *not* a C++ kernel vtable + that would dangle on unload. The plugin sees a Qt-free + `sdk::DataProcessorsHostView` (`createTransform`/`remove`/`list`/`recipeOf`). `"pj.settings.v1"` (optional) is a QSettings-like key/value store any plugin family can use for persistent state — the plugin sees a Qt-free `sdk::SettingsView` (`setValue(key, v)` returns a `Status`; reads return an